1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2010 QLogic Corporation. All rights reserved.
  24  */
  25 
  26 #include <qlge.h>
  27 #include <sys/atomic.h>
  28 #include <sys/strsubr.h>
  29 #include <sys/pattr.h>
  30 #include <netinet/in.h>
  31 #include <netinet/ip.h>
  32 #include <netinet/ip6.h>
  33 #include <netinet/tcp.h>
  34 #include <netinet/udp.h>
  35 #include <inet/ip.h>
  36 
  37 
  38 
  39 /*
  40  * Local variables
  41  */
  42 static struct ether_addr ql_ether_broadcast_addr =
  43         {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
  44 static char version[] = "GLDv3 QLogic 81XX " VERSIONSTR;
  45 
  46 /*
  47  * Local function prototypes
  48  */
  49 static void ql_free_resources(qlge_t *);
  50 static void ql_fini_kstats(qlge_t *);
  51 static uint32_t ql_get_link_state(qlge_t *);
  52 static void ql_read_conf(qlge_t *);
  53 static int ql_alloc_phys(dev_info_t *, ddi_dma_handle_t *,
  54     ddi_device_acc_attr_t *, uint_t, ddi_acc_handle_t *,
  55     size_t, size_t, caddr_t *, ddi_dma_cookie_t *);
  56 static int ql_alloc_phys_rbuf(dev_info_t *, ddi_dma_handle_t *,
  57     ddi_device_acc_attr_t *, uint_t, ddi_acc_handle_t *,
  58     size_t, size_t, caddr_t *, ddi_dma_cookie_t *);
  59 static void ql_free_phys(ddi_dma_handle_t *, ddi_acc_handle_t *);
  60 static int ql_set_routing_reg(qlge_t *, uint32_t, uint32_t, int);
  61 static int ql_attach(dev_info_t *, ddi_attach_cmd_t);
  62 static int ql_detach(dev_info_t *, ddi_detach_cmd_t);
  63 static int ql_bringdown_adapter(qlge_t *);
  64 static int ql_bringup_adapter(qlge_t *);
  65 static int ql_asic_reset(qlge_t *);
  66 static void ql_wake_mpi_reset_soft_intr(qlge_t *);
  67 static void ql_stop_timer(qlge_t *qlge);
  68 static void ql_fm_fini(qlge_t *qlge);
  69 int ql_clean_outbound_rx_ring(struct rx_ring *rx_ring);
  70 
  71 /*
  72  * TX dma maping handlers allow multiple sscatter-gather lists
  73  */
  74 ddi_dma_attr_t  tx_mapping_dma_attr = {
  75         DMA_ATTR_V0,                    /* dma_attr_version */
  76         QL_DMA_LOW_ADDRESS,             /* low DMA address range */
  77         QL_DMA_HIGH_64BIT_ADDRESS,      /* high DMA address range */
  78         QL_DMA_XFER_COUNTER,            /* DMA counter register */
  79         QL_DMA_ADDRESS_ALIGNMENT,       /* DMA address alignment, default - 8 */
  80         QL_DMA_BURSTSIZES,              /* DMA burstsizes */
  81         QL_DMA_MIN_XFER_SIZE,           /* min effective DMA size */
  82         QL_DMA_MAX_XFER_SIZE,           /* max DMA xfer size */
  83         QL_DMA_SEGMENT_BOUNDARY,        /* segment boundary */
  84         QL_MAX_TX_DMA_HANDLES,          /* s/g list length */
  85         QL_DMA_GRANULARITY,             /* granularity of device */
  86         DDI_DMA_RELAXED_ORDERING        /* DMA transfer flags */
  87 };
  88 
  89 /*
  90  * Receive buffers and Request/Response queues do not allow scatter-gather lists
  91  */
  92 ddi_dma_attr_t  dma_attr = {
  93         DMA_ATTR_V0,                    /* dma_attr_version */
  94         QL_DMA_LOW_ADDRESS,             /* low DMA address range */
  95         QL_DMA_HIGH_64BIT_ADDRESS,      /* high DMA address range */
  96         QL_DMA_XFER_COUNTER,            /* DMA counter register */
  97         QL_DMA_ADDRESS_ALIGNMENT,       /* DMA address alignment, default - 8 */
  98         QL_DMA_BURSTSIZES,              /* DMA burstsizes */
  99         QL_DMA_MIN_XFER_SIZE,           /* min effective DMA size */
 100         QL_DMA_MAX_XFER_SIZE,           /* max DMA xfer size */
 101         QL_DMA_SEGMENT_BOUNDARY,        /* segment boundary */
 102         1,                              /* s/g list length, i.e no sg list */
 103         QL_DMA_GRANULARITY,             /* granularity of device */
 104         QL_DMA_XFER_FLAGS               /* DMA transfer flags */
 105 };
 106 /*
 107  * Receive buffers do not allow scatter-gather lists
 108  */
 109 ddi_dma_attr_t  dma_attr_rbuf = {
 110         DMA_ATTR_V0,                    /* dma_attr_version */
 111         QL_DMA_LOW_ADDRESS,             /* low DMA address range */
 112         QL_DMA_HIGH_64BIT_ADDRESS,      /* high DMA address range */
 113         QL_DMA_XFER_COUNTER,            /* DMA counter register */
 114         0x1,                            /* DMA address alignment, default - 8 */
 115         QL_DMA_BURSTSIZES,              /* DMA burstsizes */
 116         QL_DMA_MIN_XFER_SIZE,           /* min effective DMA size */
 117         QL_DMA_MAX_XFER_SIZE,           /* max DMA xfer size */
 118         QL_DMA_SEGMENT_BOUNDARY,        /* segment boundary */
 119         1,                              /* s/g list length, i.e no sg list */
 120         QL_DMA_GRANULARITY,             /* granularity of device */
 121         DDI_DMA_RELAXED_ORDERING        /* DMA transfer flags */
 122 };
 123 /*
 124  * DMA access attribute structure.
 125  */
 126 /* device register access from host */
 127 ddi_device_acc_attr_t ql_dev_acc_attr = {
 128         DDI_DEVICE_ATTR_V0,
 129         DDI_STRUCTURE_LE_ACC,
 130         DDI_STRICTORDER_ACC
 131 };
 132 
 133 /* host ring descriptors */
 134 ddi_device_acc_attr_t ql_desc_acc_attr = {
 135         DDI_DEVICE_ATTR_V0,
 136         DDI_NEVERSWAP_ACC,
 137         DDI_STRICTORDER_ACC
 138 };
 139 
 140 /* host ring buffer */
 141 ddi_device_acc_attr_t ql_buf_acc_attr = {
 142         DDI_DEVICE_ATTR_V0,
 143         DDI_NEVERSWAP_ACC,
 144         DDI_STRICTORDER_ACC
 145 };
 146 
 147 /*
 148  * Hash key table for Receive Side Scaling (RSS) support
 149  */
 150 const uint8_t key_data[] = {
 151         0x23, 0x64, 0xa1, 0xaa, 0x37, 0xc0, 0xed, 0x05, 0x2b, 0x36,
 152         0x50, 0x5c, 0x45, 0x1e, 0x7e, 0xc8, 0x5d, 0x2a, 0x54, 0x2f,
 153         0xe4, 0x3d, 0x0f, 0xbb, 0x91, 0xd9, 0x25, 0x60, 0xd4, 0xf8,
 154         0x12, 0xa0, 0x59, 0x4b, 0x9e, 0x8a, 0x51, 0xda, 0xcd, 0x49};
 155 
 156 /*
 157  * Shadow Registers:
 158  * Outbound queues have a consumer index that is maintained by the chip.
 159  * Inbound queues have a producer index that is maintained by the chip.
 160  * For lower overhead, these registers are "shadowed" to host memory
 161  * which allows the device driver to track the queue progress without
 162  * PCI reads. When an entry is placed on an inbound queue, the chip will
 163  * update the relevant index register and then copy the value to the
 164  * shadow register in host memory.
 165  * Currently, ql_read_sh_reg only read Inbound queues'producer index.
 166  */
 167 
 168 static inline unsigned int
 169 ql_read_sh_reg(qlge_t *qlge, struct rx_ring *rx_ring)
 170 {
 171         uint32_t rtn;
 172 
 173         /* re-synchronize shadow prod index dma buffer before reading */
 174         (void) ddi_dma_sync(qlge->host_copy_shadow_dma_attr.dma_handle,
 175             rx_ring->prod_idx_sh_reg_offset,
 176             sizeof (uint32_t), DDI_DMA_SYNC_FORKERNEL);
 177 
 178         rtn = ddi_get32(qlge->host_copy_shadow_dma_attr.acc_handle,
 179             (uint32_t *)rx_ring->prod_idx_sh_reg);
 180 
 181         return (rtn);
 182 }
 183 
 184 /*
 185  * Read 32 bit atomically
 186  */
 187 uint32_t
 188 ql_atomic_read_32(volatile uint32_t *target)
 189 {
 190         /*
 191          * atomic_add_32_nv returns the new value after the add,
 192          * we are adding 0 so we should get the original value
 193          */
 194         return (atomic_add_32_nv(target, 0));
 195 }
 196 
 197 /*
 198  * Set 32 bit atomically
 199  */
 200 void
 201 ql_atomic_set_32(volatile uint32_t *target, uint32_t newval)
 202 {
 203         (void) atomic_swap_32(target, newval);
 204 }
 205 
 206 
 207 /*
 208  * Setup device PCI configuration registers.
 209  * Kernel context.
 210  */
 211 static void
 212 ql_pci_config(qlge_t *qlge)
 213 {
 214         uint16_t w;
 215 
 216         qlge->vendor_id = (uint16_t)pci_config_get16(qlge->pci_handle,
 217             PCI_CONF_VENID);
 218         qlge->device_id = (uint16_t)pci_config_get16(qlge->pci_handle,
 219             PCI_CONF_DEVID);
 220 
 221         /*
 222          * we want to respect framework's setting of PCI
 223          * configuration space command register and also
 224          * want to make sure that all bits of interest to us
 225          * are properly set in PCI Command register(0x04).
 226          * PCI_COMM_IO          0x1      I/O access enable
 227          * PCI_COMM_MAE         0x2      Memory access enable
 228          * PCI_COMM_ME          0x4      bus master enable
 229          * PCI_COMM_MEMWR_INVAL 0x10     memory write and invalidate enable.
 230          */
 231         w = (uint16_t)pci_config_get16(qlge->pci_handle, PCI_CONF_COMM);
 232         w = (uint16_t)(w & (~PCI_COMM_IO));
 233         w = (uint16_t)(w | PCI_COMM_MAE | PCI_COMM_ME |
 234             /* PCI_COMM_MEMWR_INVAL | */
 235             PCI_COMM_PARITY_DETECT | PCI_COMM_SERR_ENABLE);
 236 
 237         pci_config_put16(qlge->pci_handle, PCI_CONF_COMM, w);
 238 
 239         w = pci_config_get16(qlge->pci_handle, 0x54);
 240         w = (uint16_t)(w & (~0x7000));
 241         w = (uint16_t)(w | 0x5000);
 242         pci_config_put16(qlge->pci_handle, 0x54, w);
 243 
 244         ql_dump_pci_config(qlge);
 245 }
 246 
 247 /*
 248  * This routine parforms the neccessary steps to set GLD mac information
 249  * such as Function number, xgmac mask and shift bits
 250  */
 251 static int
 252 ql_set_mac_info(qlge_t *qlge)
 253 {
 254         uint32_t value;
 255         int rval = DDI_FAILURE;
 256         uint32_t fn0_net, fn1_net;
 257 
 258         /* set default value */
 259         qlge->fn0_net = FN0_NET;
 260         qlge->fn1_net = FN1_NET;
 261 
 262         if (ql_read_processor_data(qlge, MPI_REG, &value) != DDI_SUCCESS) {
 263                 cmn_err(CE_WARN, "%s(%d) read MPI register failed",
 264                     __func__, qlge->instance);
 265                 goto exit;
 266         } else {
 267                 fn0_net = (value >> 1) & 0x07;
 268                 fn1_net = (value >> 5) & 0x07;
 269                 if ((fn0_net > 4) || (fn1_net > 4) || (fn0_net == fn1_net)) {
 270                         cmn_err(CE_WARN, "%s(%d) bad mpi register value %x, \n"
 271                             "nic0 function number %d,"
 272                             "nic1 function number %d "
 273                             "use default\n",
 274                             __func__, qlge->instance, value, fn0_net, fn1_net);
 275                         goto exit;
 276                 } else {
 277                         qlge->fn0_net = fn0_net;
 278                         qlge->fn1_net = fn1_net;
 279                 }
 280         }
 281 
 282         /* Get the function number that the driver is associated with */
 283         value = ql_read_reg(qlge, REG_STATUS);
 284         qlge->func_number = (uint8_t)((value >> 6) & 0x03);
 285         QL_PRINT(DBG_INIT, ("status register is:%x, func_number: %d\n",
 286             value, qlge->func_number));
 287 
 288         /* The driver is loaded on a non-NIC function? */
 289         if ((qlge->func_number != qlge->fn0_net) &&
 290             (qlge->func_number != qlge->fn1_net)) {
 291                 cmn_err(CE_WARN,
 292                     "Invalid function number = 0x%x\n", qlge->func_number);
 293                 goto exit;
 294         }
 295         /* network port 0? */
 296         if (qlge->func_number == qlge->fn0_net) {
 297                 qlge->xgmac_sem_mask = QL_PORT0_XGMAC_SEM_MASK;
 298                 qlge->xgmac_sem_bits = QL_PORT0_XGMAC_SEM_BITS;
 299         } else {
 300                 qlge->xgmac_sem_mask = QL_PORT1_XGMAC_SEM_MASK;
 301                 qlge->xgmac_sem_bits = QL_PORT1_XGMAC_SEM_BITS;
 302         }
 303         rval = DDI_SUCCESS;
 304 exit:
 305         return (rval);
 306 
 307 }
 308 
 309 /*
 310  * write to doorbell register
 311  */
 312 void
 313 ql_write_doorbell_reg(qlge_t *qlge, uint32_t *addr, uint32_t data)
 314 {
 315         ddi_put32(qlge->dev_doorbell_reg_handle, addr, data);
 316 }
 317 
 318 /*
 319  * read from doorbell register
 320  */
 321 uint32_t
 322 ql_read_doorbell_reg(qlge_t *qlge, uint32_t *addr)
 323 {
 324         uint32_t ret;
 325 
 326         ret = ddi_get32(qlge->dev_doorbell_reg_handle, addr);
 327 
 328         return  (ret);
 329 }
 330 
 331 /*
 332  * This function waits for a specific bit to come ready
 333  * in a given register.  It is used mostly by the initialize
 334  * process, but is also used in kernel thread API such as
 335  * netdev->set_multi, netdev->set_mac_address, netdev->vlan_rx_add_vid.
 336  */
 337 static int
 338 ql_wait_reg_rdy(qlge_t *qlge, uint32_t reg, uint32_t bit, uint32_t err_bit)
 339 {
 340         uint32_t temp;
 341         int count = UDELAY_COUNT;
 342 
 343         while (count) {
 344                 temp = ql_read_reg(qlge, reg);
 345 
 346                 /* check for errors */
 347                 if ((temp & err_bit) != 0) {
 348                         break;
 349                 } else if ((temp & bit) != 0)
 350                         return (DDI_SUCCESS);
 351                 qlge_delay(UDELAY_DELAY);
 352                 count--;
 353         }
 354         cmn_err(CE_WARN,
 355             "Waiting for reg %x to come ready failed.", reg);
 356         if (qlge->fm_enable) {
 357                 ql_fm_ereport(qlge, DDI_FM_DEVICE_NO_RESPONSE);
 358                 atomic_or_32(&qlge->flags, ADAPTER_ERROR);
 359         }
 360         return (DDI_FAILURE);
 361 }
 362 
 363 /*
 364  * The CFG register is used to download TX and RX control blocks
 365  * to the chip. This function waits for an operation to complete.
 366  */
 367 static int
 368 ql_wait_cfg(qlge_t *qlge, uint32_t bit)
 369 {
 370         return (ql_wait_reg_bit(qlge, REG_CONFIGURATION, bit, BIT_RESET, 0));
 371 }
 372 
 373 
 374 /*
 375  * Used to issue init control blocks to hw. Maps control block,
 376  * sets address, triggers download, waits for completion.
 377  */
 378 static int
 379 ql_write_cfg(qlge_t *qlge, uint32_t bit, uint64_t phy_addr, uint16_t q_id)
 380 {
 381         int status = DDI_SUCCESS;
 382         uint32_t mask;
 383         uint32_t value;
 384 
 385         status = ql_sem_spinlock(qlge, SEM_ICB_MASK);
 386         if (status != DDI_SUCCESS) {
 387                 goto exit;
 388         }
 389         status = ql_wait_cfg(qlge, bit);
 390         if (status != DDI_SUCCESS) {
 391                 goto exit;
 392         }
 393 
 394         ql_write_reg(qlge, REG_ICB_ACCESS_ADDRESS_LOWER, LS_64BITS(phy_addr));
 395         ql_write_reg(qlge, REG_ICB_ACCESS_ADDRESS_UPPER, MS_64BITS(phy_addr));
 396 
 397         mask = CFG_Q_MASK | (bit << 16);
 398         value = bit | (q_id << CFG_Q_SHIFT);
 399         ql_write_reg(qlge, REG_CONFIGURATION, (mask | value));
 400 
 401         /*
 402          * Wait for the bit to clear after signaling hw.
 403          */
 404         status = ql_wait_cfg(qlge, bit);
 405         ql_sem_unlock(qlge, SEM_ICB_MASK); /* does flush too */
 406 
 407 exit:
 408         return (status);
 409 }
 410 
 411 /*
 412  * Initialize adapter instance
 413  */
 414 static int
 415 ql_init_instance(qlge_t *qlge)
 416 {
 417         int i;
 418 
 419         /* Default value */
 420         qlge->mac_flags = QL_MAC_INIT;
 421         qlge->mtu = ETHERMTU;                /* set normal size as default */
 422         qlge->page_size = VM_PAGE_SIZE;      /* default page size */
 423 
 424         for (i = 0; i < MAX_RX_RINGS; i++) {
 425                 qlge->rx_polls[i] = 0;
 426                 qlge->rx_interrupts[i] = 0;
 427         }
 428 
 429         /*
 430          * Set up the operating parameters.
 431          */
 432         qlge->multicast_list_count = 0;
 433 
 434         /*
 435          * Set up the max number of unicast list
 436          */
 437         qlge->unicst_total = MAX_UNICAST_LIST_SIZE;
 438         qlge->unicst_avail = MAX_UNICAST_LIST_SIZE;
 439 
 440         /*
 441          * read user defined properties in .conf file
 442          */
 443         ql_read_conf(qlge); /* mtu, pause, LSO etc */
 444         qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count;
 445 
 446         QL_PRINT(DBG_INIT, ("mtu is %d \n", qlge->mtu));
 447 
 448         /* choose Memory Space mapping and get Vendor Id, Device ID etc */
 449         ql_pci_config(qlge);
 450         qlge->ip_hdr_offset = 0;
 451 
 452         if (qlge->device_id == 0x8000) {
 453                 /* Schultz card */
 454                 qlge->cfg_flags |= CFG_CHIP_8100;
 455                 /* enable just ipv4 chksum offload for Schultz */
 456                 qlge->cfg_flags |= CFG_CKSUM_FULL_IPv4;
 457                 /*
 458                  * Schultz firmware does not do pseduo IP header checksum
 459                  * calculation, needed to be done by driver
 460                  */
 461                 qlge->cfg_flags |= CFG_HW_UNABLE_PSEUDO_HDR_CKSUM;
 462                 if (qlge->lso_enable)
 463                         qlge->cfg_flags |= CFG_LSO;
 464                 qlge->cfg_flags |= CFG_SUPPORT_SCATTER_GATHER;
 465                 /* Schultz must split packet header */
 466                 qlge->cfg_flags |= CFG_ENABLE_SPLIT_HEADER;
 467                 qlge->max_read_mbx = 5;
 468                 qlge->ip_hdr_offset = 2;
 469         }
 470 
 471         /* Set Function Number and some of the iocb mac information */
 472         if (ql_set_mac_info(qlge) != DDI_SUCCESS)
 473                 return (DDI_FAILURE);
 474 
 475         /* Read network settings from NVRAM */
 476         /* After nvram is read successfully, update dev_addr */
 477         if (ql_get_flash_params(qlge) == DDI_SUCCESS) {
 478                 QL_PRINT(DBG_INIT, ("mac%d address is \n", qlge->func_number));
 479                 for (i = 0; i < ETHERADDRL; i++) {
 480                         qlge->dev_addr.ether_addr_octet[i] =
 481                             qlge->nic_config.factory_MAC[i];
 482                 }
 483         } else {
 484                 cmn_err(CE_WARN, "%s(%d): Failed to read flash memory",
 485                     __func__, qlge->instance);
 486                 return (DDI_FAILURE);
 487         }
 488 
 489         bcopy(qlge->dev_addr.ether_addr_octet,
 490             qlge->unicst_addr[0].addr.ether_addr_octet,
 491             ETHERADDRL);
 492         QL_DUMP(DBG_INIT, "\t flash mac address dump:\n",
 493             &qlge->dev_addr.ether_addr_octet[0], 8, ETHERADDRL);
 494 
 495         qlge->port_link_state = LS_DOWN;
 496 
 497         return (DDI_SUCCESS);
 498 }
 499 
 500 
 501 /*
 502  * This hardware semaphore provides the mechanism for exclusive access to
 503  * resources shared between the NIC driver, MPI firmware,
 504  * FCOE firmware and the FC driver.
 505  */
 506 static int
 507 ql_sem_trylock(qlge_t *qlge, uint32_t sem_mask)
 508 {
 509         uint32_t sem_bits = 0;
 510 
 511         switch (sem_mask) {
 512         case SEM_XGMAC0_MASK:
 513                 sem_bits = SEM_SET << SEM_XGMAC0_SHIFT;
 514                 break;
 515         case SEM_XGMAC1_MASK:
 516                 sem_bits = SEM_SET << SEM_XGMAC1_SHIFT;
 517                 break;
 518         case SEM_ICB_MASK:
 519                 sem_bits = SEM_SET << SEM_ICB_SHIFT;
 520                 break;
 521         case SEM_MAC_ADDR_MASK:
 522                 sem_bits = SEM_SET << SEM_MAC_ADDR_SHIFT;
 523                 break;
 524         case SEM_FLASH_MASK:
 525                 sem_bits = SEM_SET << SEM_FLASH_SHIFT;
 526                 break;
 527         case SEM_PROBE_MASK:
 528                 sem_bits = SEM_SET << SEM_PROBE_SHIFT;
 529                 break;
 530         case SEM_RT_IDX_MASK:
 531                 sem_bits = SEM_SET << SEM_RT_IDX_SHIFT;
 532                 break;
 533         case SEM_PROC_REG_MASK:
 534                 sem_bits = SEM_SET << SEM_PROC_REG_SHIFT;
 535                 break;
 536         default:
 537                 cmn_err(CE_WARN, "Bad Semaphore mask!.");
 538                 return (DDI_FAILURE);
 539         }
 540 
 541         ql_write_reg(qlge, REG_SEMAPHORE, sem_bits | sem_mask);
 542         return (!(ql_read_reg(qlge, REG_SEMAPHORE) & sem_bits));
 543 }
 544 
 545 /*
 546  * Lock a specific bit of Semaphore register to gain
 547  * access to a particular shared register
 548  */
 549 int
 550 ql_sem_spinlock(qlge_t *qlge, uint32_t sem_mask)
 551 {
 552         unsigned int wait_count = 30;
 553 
 554         while (wait_count) {
 555                 if (!ql_sem_trylock(qlge, sem_mask))
 556                         return (DDI_SUCCESS);
 557                 qlge_delay(100);
 558                 wait_count--;
 559         }
 560         cmn_err(CE_WARN, "%s(%d) sem_mask 0x%x lock timeout ",
 561             __func__, qlge->instance, sem_mask);
 562         return (DDI_FAILURE);
 563 }
 564 
 565 /*
 566  * Unock a specific bit of Semaphore register to release
 567  * access to a particular shared register
 568  */
 569 void
 570 ql_sem_unlock(qlge_t *qlge, uint32_t sem_mask)
 571 {
 572         ql_write_reg(qlge, REG_SEMAPHORE, sem_mask);
 573         (void) ql_read_reg(qlge, REG_SEMAPHORE);        /* flush */
 574 }
 575 
 576 /*
 577  * Get property value from configuration file.
 578  *
 579  * string = property string pointer.
 580  *
 581  * Returns:
 582  * 0xFFFFFFFF = no property else property value.
 583  */
 584 static uint32_t
 585 ql_get_prop(qlge_t *qlge, char *string)
 586 {
 587         char buf[256];
 588         uint32_t data;
 589 
 590         /* Get adapter instance parameter. */
 591         (void) sprintf(buf, "hba%d-%s", qlge->instance, string);
 592         data = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, qlge->dip, 0, buf,
 593             (int)0xffffffff);
 594 
 595         /* Adapter instance parameter found? */
 596         if (data == 0xffffffff) {
 597                 /* No, get default parameter. */
 598                 data = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, qlge->dip, 0,
 599                     string, (int)0xffffffff);
 600         }
 601 
 602         return (data);
 603 }
 604 
 605 /*
 606  * Read user setting from configuration file.
 607  */
 608 static void
 609 ql_read_conf(qlge_t *qlge)
 610 {
 611         uint32_t data;
 612 
 613         /* clear configuration flags */
 614         qlge->cfg_flags = 0;
 615 
 616         /* Set up the default ring sizes. */
 617         qlge->tx_ring_size = NUM_TX_RING_ENTRIES;
 618         data = ql_get_prop(qlge, "tx_ring_size");
 619         /* if data is valid */
 620         if ((data != 0xffffffff) && data) {
 621                 if (qlge->tx_ring_size != data) {
 622                         qlge->tx_ring_size = (uint16_t)data;
 623                 }
 624         }
 625 
 626         qlge->rx_ring_size = NUM_RX_RING_ENTRIES;
 627         data = ql_get_prop(qlge, "rx_ring_size");
 628         /* if data is valid */
 629         if ((data != 0xffffffff) && data) {
 630                 if (qlge->rx_ring_size != data) {
 631                         qlge->rx_ring_size = (uint16_t)data;
 632                 }
 633         }
 634 
 635         qlge->tx_ring_count = 8;
 636         data = ql_get_prop(qlge, "tx_ring_count");
 637         /* if data is valid */
 638         if ((data != 0xffffffff) && data) {
 639                 if (qlge->tx_ring_count != data) {
 640                         qlge->tx_ring_count = (uint16_t)data;
 641                 }
 642         }
 643 
 644         qlge->rss_ring_count = 8;
 645         data = ql_get_prop(qlge, "rss_ring_count");
 646         /* if data is valid */
 647         if ((data != 0xffffffff) && data) {
 648                 if (qlge->rss_ring_count != data) {
 649                         qlge->rss_ring_count = (uint16_t)data;
 650                 }
 651         }
 652 
 653         /* Get default rx_copy enable/disable. */
 654         if ((data = ql_get_prop(qlge, "force-rx-copy")) == 0xffffffff ||
 655             data == 0) {
 656                 qlge->rx_copy = B_FALSE;
 657                 QL_PRINT(DBG_INIT, ("rx copy mode disabled\n"));
 658         } else if (data == 1) {
 659                 qlge->rx_copy = B_TRUE;
 660                 QL_PRINT(DBG_INIT, ("rx copy mode enabled\n"));
 661         }
 662 
 663         qlge->rx_copy_threshold = qlge->rx_ring_size / 4;
 664         data = ql_get_prop(qlge, "rx_copy_threshold");
 665         if ((data != 0xffffffff) && (data != 0)) {
 666                 qlge->rx_copy_threshold = data;
 667                 cmn_err(CE_NOTE, "!new rx_copy_threshold %d \n",
 668                     qlge->rx_copy_threshold);
 669         }
 670 
 671         /* Get mtu packet size. */
 672         data = ql_get_prop(qlge, "mtu");
 673         if ((data == ETHERMTU) || (data == JUMBO_MTU)) {
 674                 if (qlge->mtu != data) {
 675                         qlge->mtu = data;
 676                         cmn_err(CE_NOTE, "new mtu is %d\n", qlge->mtu);
 677                 }
 678         }
 679 
 680         if (qlge->mtu == JUMBO_MTU) {
 681                 qlge->rx_coalesce_usecs = DFLT_RX_COALESCE_WAIT_JUMBO;
 682                 qlge->tx_coalesce_usecs = DFLT_TX_COALESCE_WAIT_JUMBO;
 683                 qlge->rx_max_coalesced_frames = DFLT_RX_INTER_FRAME_WAIT_JUMBO;
 684                 qlge->tx_max_coalesced_frames = DFLT_TX_INTER_FRAME_WAIT_JUMBO;
 685         }
 686 
 687 
 688         /* Get pause mode, default is Per Priority mode. */
 689         qlge->pause = PAUSE_MODE_PER_PRIORITY;
 690         data = ql_get_prop(qlge, "pause");
 691         if (data <= PAUSE_MODE_PER_PRIORITY) {
 692                 if (qlge->pause != data) {
 693                         qlge->pause = data;
 694                         cmn_err(CE_NOTE, "new pause mode %d\n", qlge->pause);
 695                 }
 696         }
 697         /* Receive interrupt delay */
 698         qlge->rx_coalesce_usecs = DFLT_RX_COALESCE_WAIT;
 699         data = ql_get_prop(qlge, "rx_intr_delay");
 700         /* if data is valid */
 701         if ((data != 0xffffffff) && data) {
 702                 if (qlge->rx_coalesce_usecs != data) {
 703                         qlge->rx_coalesce_usecs = (uint16_t)data;
 704                 }
 705         }
 706         /* Rx inter-packet delay. */
 707         qlge->rx_max_coalesced_frames = DFLT_RX_INTER_FRAME_WAIT;
 708         data = ql_get_prop(qlge, "rx_ipkt_delay");
 709         /* if data is valid */
 710         if ((data != 0xffffffff) && data) {
 711                 if (qlge->rx_max_coalesced_frames != data) {
 712                         qlge->rx_max_coalesced_frames = (uint16_t)data;
 713                 }
 714         }
 715         /* Transmit interrupt delay */
 716         qlge->tx_coalesce_usecs = DFLT_TX_COALESCE_WAIT;
 717         data = ql_get_prop(qlge, "tx_intr_delay");
 718         /* if data is valid */
 719         if ((data != 0xffffffff) && data) {
 720                 if (qlge->tx_coalesce_usecs != data) {
 721                         qlge->tx_coalesce_usecs = (uint16_t)data;
 722                 }
 723         }
 724         /* Tx inter-packet delay. */
 725         qlge->tx_max_coalesced_frames = DFLT_TX_INTER_FRAME_WAIT;
 726         data = ql_get_prop(qlge, "tx_ipkt_delay");
 727         /* if data is valid */
 728         if ((data != 0xffffffff) && data) {
 729                 if (qlge->tx_max_coalesced_frames != data) {
 730                         qlge->tx_max_coalesced_frames = (uint16_t)data;
 731                 }
 732         }
 733 
 734         /* Get split header payload_copy_thresh. */
 735         qlge->payload_copy_thresh = DFLT_PAYLOAD_COPY_THRESH;
 736         data = ql_get_prop(qlge, "payload_copy_thresh");
 737         /* if data is valid */
 738         if ((data != 0xffffffff) && (data != 0)) {
 739                 if (qlge->payload_copy_thresh != data) {
 740                         qlge->payload_copy_thresh = data;
 741                 }
 742         }
 743 
 744         /* large send offload (LSO) capability. */
 745         qlge->lso_enable = 1;
 746         data = ql_get_prop(qlge, "lso_enable");
 747         /* if data is valid */
 748         if ((data == 0) || (data == 1)) {
 749                 if (qlge->lso_enable != data) {
 750                         qlge->lso_enable = (uint16_t)data;
 751                 }
 752         }
 753 
 754         /* dcbx capability. */
 755         qlge->dcbx_enable = 1;
 756         data = ql_get_prop(qlge, "dcbx_enable");
 757         /* if data is valid */
 758         if ((data == 0) || (data == 1)) {
 759                 if (qlge->dcbx_enable != data) {
 760                         qlge->dcbx_enable = (uint16_t)data;
 761                 }
 762         }
 763         /* fault management enable */
 764         qlge->fm_enable = B_TRUE;
 765         data = ql_get_prop(qlge, "fm-enable");
 766         if ((data == 0x1) || (data == 0)) {
 767                 qlge->fm_enable = (boolean_t)data;
 768         }
 769 
 770 }
 771 
 772 /*
 773  * Enable global interrupt
 774  */
 775 static void
 776 ql_enable_global_interrupt(qlge_t *qlge)
 777 {
 778         ql_write_reg(qlge, REG_INTERRUPT_ENABLE,
 779             (INTR_EN_EI << 16) | INTR_EN_EI);
 780         qlge->flags |= INTERRUPTS_ENABLED;
 781 }
 782 
 783 /*
 784  * Disable global interrupt
 785  */
 786 static void
 787 ql_disable_global_interrupt(qlge_t *qlge)
 788 {
 789         ql_write_reg(qlge, REG_INTERRUPT_ENABLE, (INTR_EN_EI << 16));
 790         qlge->flags &= ~INTERRUPTS_ENABLED;
 791 }
 792 
 793 /*
 794  * Enable one ring interrupt
 795  */
 796 void
 797 ql_enable_completion_interrupt(qlge_t *qlge, uint32_t intr)
 798 {
 799         struct intr_ctx *ctx = qlge->intr_ctx + intr;
 800 
 801         QL_PRINT(DBG_INTR, ("%s(%d): To enable intr %d, irq_cnt %d \n",
 802             __func__, qlge->instance, intr, ctx->irq_cnt));
 803 
 804         if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && intr) {
 805                 /*
 806                  * Always enable if we're MSIX multi interrupts and
 807                  * it's not the default (zeroeth) interrupt.
 808                  */
 809                 ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_en_mask);
 810                 return;
 811         }
 812 
 813         if (!atomic_dec_32_nv(&ctx->irq_cnt)) {
 814                 mutex_enter(&qlge->hw_mutex);
 815                 ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_en_mask);
 816                 mutex_exit(&qlge->hw_mutex);
 817                 QL_PRINT(DBG_INTR,
 818                     ("%s(%d): write %x to intr enable register \n",
 819                     __func__, qlge->instance, ctx->intr_en_mask));
 820         }
 821 }
 822 
 823 /*
 824  * ql_forced_disable_completion_interrupt
 825  * Used by call from OS, may be called without
 826  * a pending interrupt so force the disable
 827  */
 828 uint32_t
 829 ql_forced_disable_completion_interrupt(qlge_t *qlge, uint32_t intr)
 830 {
 831         uint32_t var = 0;
 832         struct intr_ctx *ctx = qlge->intr_ctx + intr;
 833 
 834         QL_PRINT(DBG_INTR, ("%s(%d): To disable intr %d, irq_cnt %d \n",
 835             __func__, qlge->instance, intr, ctx->irq_cnt));
 836 
 837         if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && intr) {
 838                 ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask);
 839                 var = ql_read_reg(qlge, REG_STATUS);
 840                 return (var);
 841         }
 842 
 843         mutex_enter(&qlge->hw_mutex);
 844         ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask);
 845         var = ql_read_reg(qlge, REG_STATUS);
 846         mutex_exit(&qlge->hw_mutex);
 847 
 848         return (var);
 849 }
 850 
 851 /*
 852  * Disable a completion interrupt
 853  */
 854 void
 855 ql_disable_completion_interrupt(qlge_t *qlge, uint32_t intr)
 856 {
 857         struct intr_ctx *ctx;
 858 
 859         ctx = qlge->intr_ctx + intr;
 860         QL_PRINT(DBG_INTR, ("%s(%d): To disable intr %d, irq_cnt %d \n",
 861             __func__, qlge->instance, intr, ctx->irq_cnt));
 862         /*
 863          * HW disables for us if we're MSIX multi interrupts and
 864          * it's not the default (zeroeth) interrupt.
 865          */
 866         if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && (intr != 0))
 867                 return;
 868 
 869         if (ql_atomic_read_32(&ctx->irq_cnt) == 0) {
 870                 mutex_enter(&qlge->hw_mutex);
 871                 ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask);
 872                 mutex_exit(&qlge->hw_mutex);
 873         }
 874         atomic_inc_32(&ctx->irq_cnt);
 875 }
 876 
 877 /*
 878  * Enable all completion interrupts
 879  */
 880 static void
 881 ql_enable_all_completion_interrupts(qlge_t *qlge)
 882 {
 883         int i;
 884         uint32_t value = 1;
 885 
 886         for (i = 0; i < qlge->intr_cnt; i++) {
 887                 /*
 888                  * Set the count to 1 for Legacy / MSI interrupts or for the
 889                  * default interrupt (0)
 890                  */
 891                 if ((qlge->intr_type != DDI_INTR_TYPE_MSIX) || i == 0) {
 892                         ql_atomic_set_32(&qlge->intr_ctx[i].irq_cnt, value);
 893                 }
 894                 ql_enable_completion_interrupt(qlge, i);
 895         }
 896 }
 897 
 898 /*
 899  * Disable all completion interrupts
 900  */
 901 static void
 902 ql_disable_all_completion_interrupts(qlge_t *qlge)
 903 {
 904         int i;
 905         uint32_t value = 0;
 906 
 907         for (i = 0; i < qlge->intr_cnt; i++) {
 908 
 909                 /*
 910                  * Set the count to 0 for Legacy / MSI interrupts or for the
 911                  * default interrupt (0)
 912                  */
 913                 if ((qlge->intr_type != DDI_INTR_TYPE_MSIX) || i == 0)
 914                         ql_atomic_set_32(&qlge->intr_ctx[i].irq_cnt, value);
 915 
 916                 ql_disable_completion_interrupt(qlge, i);
 917         }
 918 }
 919 
 920 /*
 921  * Update small buffer queue producer index
 922  */
 923 static void
 924 ql_update_sbq_prod_idx(qlge_t *qlge, struct rx_ring *rx_ring)
 925 {
 926         /* Update the buffer producer index */
 927         QL_PRINT(DBG_RX, ("sbq: updating prod idx = %d.\n",
 928             rx_ring->sbq_prod_idx));
 929         ql_write_doorbell_reg(qlge, rx_ring->sbq_prod_idx_db_reg,
 930             rx_ring->sbq_prod_idx);
 931 }
 932 
 933 /*
 934  * Update large buffer queue producer index
 935  */
 936 static void
 937 ql_update_lbq_prod_idx(qlge_t *qlge, struct rx_ring *rx_ring)
 938 {
 939         /* Update the buffer producer index */
 940         QL_PRINT(DBG_RX, ("lbq: updating prod idx = %d.\n",
 941             rx_ring->lbq_prod_idx));
 942         ql_write_doorbell_reg(qlge, rx_ring->lbq_prod_idx_db_reg,
 943             rx_ring->lbq_prod_idx);
 944 }
 945 
 946 /*
 947  * Adds a small buffer descriptor to end of its in use list,
 948  * assumes sbq_lock is already taken
 949  */
 950 static void
 951 ql_add_sbuf_to_in_use_list(struct rx_ring *rx_ring,
 952     struct bq_desc *sbq_desc)
 953 {
 954         uint32_t inuse_idx = rx_ring->sbq_use_tail;
 955 
 956         rx_ring->sbuf_in_use[inuse_idx] = sbq_desc;
 957         inuse_idx++;
 958         if (inuse_idx >= rx_ring->sbq_len)
 959                 inuse_idx = 0;
 960         rx_ring->sbq_use_tail = inuse_idx;
 961         atomic_inc_32(&rx_ring->sbuf_in_use_count);
 962         ASSERT(rx_ring->sbuf_in_use_count <= rx_ring->sbq_len);
 963 }
 964 
 965 /*
 966  * Get a small buffer descriptor from its in use list
 967  */
 968 static struct bq_desc *
 969 ql_get_sbuf_from_in_use_list(struct rx_ring *rx_ring)
 970 {
 971         struct bq_desc *sbq_desc = NULL;
 972         uint32_t inuse_idx;
 973 
 974         /* Pick from head of in use list */
 975         inuse_idx = rx_ring->sbq_use_head;
 976         sbq_desc = rx_ring->sbuf_in_use[inuse_idx];
 977         rx_ring->sbuf_in_use[inuse_idx] = NULL;
 978 
 979         if (sbq_desc != NULL) {
 980                 inuse_idx++;
 981                 if (inuse_idx >= rx_ring->sbq_len)
 982                         inuse_idx = 0;
 983                 rx_ring->sbq_use_head = inuse_idx;
 984                 atomic_dec_32(&rx_ring->sbuf_in_use_count);
 985                 atomic_inc_32(&rx_ring->rx_indicate);
 986                 sbq_desc->upl_inuse = 1;
 987                 /* if mp is NULL */
 988                 if (sbq_desc->mp == NULL) {
 989                         /* try to remap mp again */
 990                         sbq_desc->mp =
 991                             desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr),
 992                             rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle);
 993                 }
 994         }
 995 
 996         return (sbq_desc);
 997 }
 998 
 999 /*
1000  * Add a small buffer descriptor to its free list
1001  */
1002 static void
1003 ql_add_sbuf_to_free_list(struct rx_ring *rx_ring,
1004     struct bq_desc *sbq_desc)
1005 {
1006         uint32_t free_idx;
1007 
1008         /* Add to the end of free list */
1009         free_idx = rx_ring->sbq_free_tail;
1010         rx_ring->sbuf_free[free_idx] = sbq_desc;
1011         ASSERT(rx_ring->sbuf_free_count <= rx_ring->sbq_len);
1012         free_idx++;
1013         if (free_idx >= rx_ring->sbq_len)
1014                 free_idx = 0;
1015         rx_ring->sbq_free_tail = free_idx;
1016         atomic_inc_32(&rx_ring->sbuf_free_count);
1017 }
1018 
1019 /*
1020  * Get a small buffer descriptor from its free list
1021  */
1022 static struct bq_desc *
1023 ql_get_sbuf_from_free_list(struct rx_ring *rx_ring)
1024 {
1025         struct bq_desc *sbq_desc;
1026         uint32_t free_idx;
1027 
1028         free_idx = rx_ring->sbq_free_head;
1029         /* Pick from top of free list */
1030         sbq_desc = rx_ring->sbuf_free[free_idx];
1031         rx_ring->sbuf_free[free_idx] = NULL;
1032         if (sbq_desc != NULL) {
1033                 free_idx++;
1034                 if (free_idx >= rx_ring->sbq_len)
1035                         free_idx = 0;
1036                 rx_ring->sbq_free_head = free_idx;
1037                 atomic_dec_32(&rx_ring->sbuf_free_count);
1038         }
1039         return (sbq_desc);
1040 }
1041 
1042 /*
1043  * Add a large buffer descriptor to its in use list
1044  */
1045 static void
1046 ql_add_lbuf_to_in_use_list(struct rx_ring *rx_ring,
1047     struct bq_desc *lbq_desc)
1048 {
1049         uint32_t inuse_idx;
1050 
1051         inuse_idx = rx_ring->lbq_use_tail;
1052 
1053         rx_ring->lbuf_in_use[inuse_idx] = lbq_desc;
1054         inuse_idx++;
1055         if (inuse_idx >= rx_ring->lbq_len)
1056                 inuse_idx = 0;
1057         rx_ring->lbq_use_tail = inuse_idx;
1058         atomic_inc_32(&rx_ring->lbuf_in_use_count);
1059 }
1060 
1061 /*
1062  * Get a large buffer descriptor from in use list
1063  */
1064 static struct bq_desc *
1065 ql_get_lbuf_from_in_use_list(struct rx_ring *rx_ring)
1066 {
1067         struct bq_desc *lbq_desc;
1068         uint32_t inuse_idx;
1069 
1070         /* Pick from head of in use list */
1071         inuse_idx = rx_ring->lbq_use_head;
1072         lbq_desc = rx_ring->lbuf_in_use[inuse_idx];
1073         rx_ring->lbuf_in_use[inuse_idx] = NULL;
1074 
1075         if (lbq_desc != NULL) {
1076                 inuse_idx++;
1077                 if (inuse_idx >= rx_ring->lbq_len)
1078                         inuse_idx = 0;
1079                 rx_ring->lbq_use_head = inuse_idx;
1080                 atomic_dec_32(&rx_ring->lbuf_in_use_count);
1081                 atomic_inc_32(&rx_ring->rx_indicate);
1082                 lbq_desc->upl_inuse = 1;
1083 
1084                 /* if mp is NULL */
1085                 if (lbq_desc->mp == NULL) {
1086                         /* try to remap mp again */
1087                         lbq_desc->mp =
1088                             desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr),
1089                             rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle);
1090                 }
1091         }
1092         return (lbq_desc);
1093 }
1094 
1095 /*
1096  * Add a large buffer descriptor to free list
1097  */
1098 static void
1099 ql_add_lbuf_to_free_list(struct rx_ring *rx_ring,
1100     struct bq_desc *lbq_desc)
1101 {
1102         uint32_t free_idx;
1103 
1104         /* Add to the end of free list */
1105         free_idx = rx_ring->lbq_free_tail;
1106         rx_ring->lbuf_free[free_idx] = lbq_desc;
1107         free_idx++;
1108         if (free_idx >= rx_ring->lbq_len)
1109                 free_idx = 0;
1110         rx_ring->lbq_free_tail = free_idx;
1111         atomic_inc_32(&rx_ring->lbuf_free_count);
1112         ASSERT(rx_ring->lbuf_free_count <= rx_ring->lbq_len);
1113 }
1114 
1115 /*
1116  * Get a large buffer descriptor from its free list
1117  */
1118 static struct bq_desc *
1119 ql_get_lbuf_from_free_list(struct rx_ring *rx_ring)
1120 {
1121         struct bq_desc *lbq_desc;
1122         uint32_t free_idx;
1123 
1124         free_idx = rx_ring->lbq_free_head;
1125         /* Pick from head of free list */
1126         lbq_desc = rx_ring->lbuf_free[free_idx];
1127         rx_ring->lbuf_free[free_idx] = NULL;
1128 
1129         if (lbq_desc != NULL) {
1130                 free_idx++;
1131                 if (free_idx >= rx_ring->lbq_len)
1132                         free_idx = 0;
1133                 rx_ring->lbq_free_head = free_idx;
1134                 atomic_dec_32(&rx_ring->lbuf_free_count);
1135         }
1136         return (lbq_desc);
1137 }
1138 
1139 /*
1140  * Add a small buffer descriptor to free list
1141  */
1142 static void
1143 ql_refill_sbuf_free_list(struct bq_desc *sbq_desc, boolean_t alloc_memory)
1144 {
1145         struct rx_ring *rx_ring = sbq_desc->rx_ring;
1146         uint64_t *sbq_entry;
1147         qlge_t *qlge = (qlge_t *)rx_ring->qlge;
1148         /*
1149          * Sync access
1150          */
1151         mutex_enter(&rx_ring->sbq_lock);
1152 
1153         sbq_desc->upl_inuse = 0;
1154 
1155         /*
1156          * If we are freeing the buffers as a result of adapter unload, get out
1157          */
1158         if ((sbq_desc->free_buf != NULL) ||
1159             (qlge->mac_flags == QL_MAC_DETACH)) {
1160                 if (sbq_desc->free_buf == NULL)
1161                         atomic_dec_32(&rx_ring->rx_indicate);
1162                 mutex_exit(&rx_ring->sbq_lock);
1163                 return;
1164         }
1165 #ifdef QLGE_LOAD_UNLOAD
1166         if (rx_ring->rx_indicate == 0)
1167                 cmn_err(CE_WARN, "sbq: indicate wrong");
1168 #endif
1169 #ifdef QLGE_TRACK_BUFFER_USAGE
1170         uint32_t sb_consumer_idx;
1171         uint32_t sb_producer_idx;
1172         uint32_t num_free_buffers;
1173         uint32_t temp;
1174 
1175         temp = ql_read_doorbell_reg(qlge, rx_ring->sbq_prod_idx_db_reg);
1176         sb_producer_idx = temp & 0x0000ffff;
1177         sb_consumer_idx = (temp >> 16);
1178 
1179         if (sb_consumer_idx > sb_producer_idx)
1180                 num_free_buffers = NUM_SMALL_BUFFERS -
1181                     (sb_consumer_idx - sb_producer_idx);
1182         else
1183                 num_free_buffers = sb_producer_idx - sb_consumer_idx;
1184 
1185         if (num_free_buffers < qlge->rx_sb_low_count[rx_ring->cq_id])
1186                 qlge->rx_sb_low_count[rx_ring->cq_id] = num_free_buffers;
1187 
1188 #endif
1189 
1190 #ifdef QLGE_LOAD_UNLOAD
1191         if (rx_ring->rx_indicate > 0xFF000000)
1192                 cmn_err(CE_WARN, "sbq: indicate(%d) wrong: %d mac_flags %d,"
1193                     " sbq_desc index %d.",
1194                     rx_ring->cq_id, rx_ring->rx_indicate, rx_ring->mac_flags,
1195                     sbq_desc->index);
1196 #endif
1197         if (alloc_memory) {
1198                 sbq_desc->mp =
1199                     desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr),
1200                     rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle);
1201                 if (sbq_desc->mp == NULL) {
1202                         rx_ring->rx_failed_sbq_allocs++;
1203                 }
1204         }
1205 
1206         /* Got the packet from the stack decrement rx_indicate count */
1207         atomic_dec_32(&rx_ring->rx_indicate);
1208 
1209         ql_add_sbuf_to_free_list(rx_ring, sbq_desc);
1210 
1211         /* Rearm if possible */
1212         if ((rx_ring->sbuf_free_count >= MIN_BUFFERS_FREE_COUNT) &&
1213             (qlge->mac_flags == QL_MAC_STARTED)) {
1214                 sbq_entry = rx_ring->sbq_dma.vaddr;
1215                 sbq_entry += rx_ring->sbq_prod_idx;
1216 
1217                 while (rx_ring->sbuf_free_count > MIN_BUFFERS_ARM_COUNT) {
1218                         /* Get first one from free list */
1219                         sbq_desc = ql_get_sbuf_from_free_list(rx_ring);
1220 
1221                         *sbq_entry = cpu_to_le64(sbq_desc->bd_dma.dma_addr);
1222                         sbq_entry++;
1223                         rx_ring->sbq_prod_idx++;
1224                         if (rx_ring->sbq_prod_idx >= rx_ring->sbq_len) {
1225                                 rx_ring->sbq_prod_idx = 0;
1226                                 sbq_entry = rx_ring->sbq_dma.vaddr;
1227                         }
1228                         /* Add to end of in use list */
1229                         ql_add_sbuf_to_in_use_list(rx_ring, sbq_desc);
1230                 }
1231 
1232                 /* Update small buffer queue producer index */
1233                 ql_update_sbq_prod_idx(qlge, rx_ring);
1234         }
1235 
1236         mutex_exit(&rx_ring->sbq_lock);
1237         QL_PRINT(DBG_RX_RING, ("%s(%d) exited, sbuf_free_count %d\n",
1238             __func__, qlge->instance, rx_ring->sbuf_free_count));
1239 }
1240 
1241 /*
1242  * rx recycle call back function
1243  */
1244 static void
1245 ql_release_to_sbuf_free_list(caddr_t p)
1246 {
1247         struct bq_desc *sbq_desc = (struct bq_desc *)(void *)p;
1248 
1249         if (sbq_desc == NULL)
1250                 return;
1251         ql_refill_sbuf_free_list(sbq_desc, B_TRUE);
1252 }
1253 
1254 /*
1255  * Add a large buffer descriptor to free list
1256  */
1257 static void
1258 ql_refill_lbuf_free_list(struct bq_desc *lbq_desc, boolean_t alloc_memory)
1259 {
1260         struct rx_ring *rx_ring = lbq_desc->rx_ring;
1261         uint64_t *lbq_entry;
1262         qlge_t *qlge = rx_ring->qlge;
1263 
1264         /* Sync access */
1265         mutex_enter(&rx_ring->lbq_lock);
1266 
1267         lbq_desc->upl_inuse = 0;
1268         /*
1269          * If we are freeing the buffers as a result of adapter unload, get out
1270          */
1271         if ((lbq_desc->free_buf != NULL) ||
1272             (qlge->mac_flags == QL_MAC_DETACH)) {
1273                 if (lbq_desc->free_buf == NULL)
1274                         atomic_dec_32(&rx_ring->rx_indicate);
1275                 mutex_exit(&rx_ring->lbq_lock);
1276                 return;
1277         }
1278 #ifdef QLGE_LOAD_UNLOAD
1279         if (rx_ring->rx_indicate == 0)
1280                 cmn_err(CE_WARN, "lbq: indicate wrong");
1281 #endif
1282 #ifdef QLGE_TRACK_BUFFER_USAGE
1283         uint32_t lb_consumer_idx;
1284         uint32_t lb_producer_idx;
1285         uint32_t num_free_buffers;
1286         uint32_t temp;
1287 
1288         temp = ql_read_doorbell_reg(qlge, rx_ring->lbq_prod_idx_db_reg);
1289 
1290         lb_producer_idx = temp & 0x0000ffff;
1291         lb_consumer_idx = (temp >> 16);
1292 
1293         if (lb_consumer_idx > lb_producer_idx)
1294                 num_free_buffers = NUM_LARGE_BUFFERS -
1295                     (lb_consumer_idx - lb_producer_idx);
1296         else
1297                 num_free_buffers = lb_producer_idx - lb_consumer_idx;
1298 
1299         if (num_free_buffers < qlge->rx_lb_low_count[rx_ring->cq_id]) {
1300                 qlge->rx_lb_low_count[rx_ring->cq_id] = num_free_buffers;
1301         }
1302 #endif
1303 
1304 #ifdef QLGE_LOAD_UNLOAD
1305         if (rx_ring->rx_indicate > 0xFF000000)
1306                 cmn_err(CE_WARN, "lbq: indicate(%d) wrong: %d mac_flags %d,"
1307                     "lbq_desc index %d",
1308                     rx_ring->cq_id, rx_ring->rx_indicate, rx_ring->mac_flags,
1309                     lbq_desc->index);
1310 #endif
1311         if (alloc_memory) {
1312                 lbq_desc->mp =
1313                     desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr),
1314                     rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle);
1315                 if (lbq_desc->mp == NULL) {
1316                         rx_ring->rx_failed_lbq_allocs++;
1317                 }
1318         }
1319 
1320         /* Got the packet from the stack decrement rx_indicate count */
1321         atomic_dec_32(&rx_ring->rx_indicate);
1322 
1323         ql_add_lbuf_to_free_list(rx_ring, lbq_desc);
1324 
1325         /* Rearm if possible */
1326         if ((rx_ring->lbuf_free_count >= MIN_BUFFERS_FREE_COUNT) &&
1327             (qlge->mac_flags == QL_MAC_STARTED)) {
1328                 lbq_entry = rx_ring->lbq_dma.vaddr;
1329                 lbq_entry += rx_ring->lbq_prod_idx;
1330                 while (rx_ring->lbuf_free_count > MIN_BUFFERS_ARM_COUNT) {
1331                         /* Get first one from free list */
1332                         lbq_desc = ql_get_lbuf_from_free_list(rx_ring);
1333 
1334                         *lbq_entry = cpu_to_le64(lbq_desc->bd_dma.dma_addr);
1335                         lbq_entry++;
1336                         rx_ring->lbq_prod_idx++;
1337                         if (rx_ring->lbq_prod_idx >= rx_ring->lbq_len) {
1338                                 rx_ring->lbq_prod_idx = 0;
1339                                 lbq_entry = rx_ring->lbq_dma.vaddr;
1340                         }
1341 
1342                         /* Add to end of in use list */
1343                         ql_add_lbuf_to_in_use_list(rx_ring, lbq_desc);
1344                 }
1345 
1346                 /* Update large buffer queue producer index */
1347                 ql_update_lbq_prod_idx(rx_ring->qlge, rx_ring);
1348         }
1349 
1350         mutex_exit(&rx_ring->lbq_lock);
1351         QL_PRINT(DBG_RX_RING, ("%s exitd, lbuf_free_count %d\n",
1352             __func__, rx_ring->lbuf_free_count));
1353 }
1354 /*
1355  * rx recycle call back function
1356  */
1357 static void
1358 ql_release_to_lbuf_free_list(caddr_t p)
1359 {
1360         struct bq_desc *lbq_desc = (struct bq_desc *)(void *)p;
1361 
1362         if (lbq_desc == NULL)
1363                 return;
1364         ql_refill_lbuf_free_list(lbq_desc, B_TRUE);
1365 }
1366 
1367 /*
1368  * free small buffer queue buffers
1369  */
1370 static void
1371 ql_free_sbq_buffers(struct rx_ring *rx_ring)
1372 {
1373         struct bq_desc *sbq_desc;
1374         uint32_t i;
1375         uint32_t j = rx_ring->sbq_free_head;
1376         int  force_cnt = 0;
1377 
1378         for (i = 0; i < rx_ring->sbuf_free_count; i++) {
1379                 sbq_desc = rx_ring->sbuf_free[j];
1380                 sbq_desc->free_buf = 1;
1381                 j++;
1382                 if (j >= rx_ring->sbq_len) {
1383                         j = 0;
1384                 }
1385                 if (sbq_desc->mp != NULL) {
1386                         freemsg(sbq_desc->mp);
1387                         sbq_desc->mp = NULL;
1388                 }
1389         }
1390         rx_ring->sbuf_free_count = 0;
1391 
1392         j = rx_ring->sbq_use_head;
1393         for (i = 0; i < rx_ring->sbuf_in_use_count; i++) {
1394                 sbq_desc = rx_ring->sbuf_in_use[j];
1395                 sbq_desc->free_buf = 1;
1396                 j++;
1397                 if (j >= rx_ring->sbq_len) {
1398                         j = 0;
1399                 }
1400                 if (sbq_desc->mp != NULL) {
1401                         freemsg(sbq_desc->mp);
1402                         sbq_desc->mp = NULL;
1403                 }
1404         }
1405         rx_ring->sbuf_in_use_count = 0;
1406 
1407         sbq_desc = &rx_ring->sbq_desc[0];
1408         for (i = 0; i < rx_ring->sbq_len; i++, sbq_desc++) {
1409                 /*
1410                  * Set flag so that the callback does not allocate a new buffer
1411                  */
1412                 sbq_desc->free_buf = 1;
1413                 if (sbq_desc->upl_inuse != 0) {
1414                         force_cnt++;
1415                 }
1416                 if (sbq_desc->bd_dma.dma_handle != NULL) {
1417                         ql_free_phys(&sbq_desc->bd_dma.dma_handle,
1418                             &sbq_desc->bd_dma.acc_handle);
1419                         sbq_desc->bd_dma.dma_handle = NULL;
1420                         sbq_desc->bd_dma.acc_handle = NULL;
1421                 }
1422         }
1423 #ifdef QLGE_LOAD_UNLOAD
1424         cmn_err(CE_NOTE, "sbq: free %d inuse %d force %d\n",
1425             rx_ring->sbuf_free_count, rx_ring->sbuf_in_use_count, force_cnt);
1426 #endif
1427         if (rx_ring->sbuf_in_use != NULL) {
1428                 kmem_free(rx_ring->sbuf_in_use, (rx_ring->sbq_len *
1429                     sizeof (struct bq_desc *)));
1430                 rx_ring->sbuf_in_use = NULL;
1431         }
1432 
1433         if (rx_ring->sbuf_free != NULL) {
1434                 kmem_free(rx_ring->sbuf_free, (rx_ring->sbq_len *
1435                     sizeof (struct bq_desc *)));
1436                 rx_ring->sbuf_free = NULL;
1437         }
1438 }
1439 
1440 /* Allocate small buffers */
1441 static int
1442 ql_alloc_sbufs(qlge_t *qlge, struct rx_ring *rx_ring)
1443 {
1444         struct bq_desc *sbq_desc;
1445         int i;
1446         ddi_dma_cookie_t dma_cookie;
1447 
1448         rx_ring->sbq_use_head = 0;
1449         rx_ring->sbq_use_tail = 0;
1450         rx_ring->sbuf_in_use_count = 0;
1451         rx_ring->sbq_free_head = 0;
1452         rx_ring->sbq_free_tail = 0;
1453         rx_ring->sbuf_free_count = 0;
1454         rx_ring->sbuf_free = kmem_zalloc(rx_ring->sbq_len *
1455             sizeof (struct bq_desc *), KM_NOSLEEP);
1456         if (rx_ring->sbuf_free == NULL) {
1457                 cmn_err(CE_WARN,
1458                     "!%s: sbuf_free_list alloc: failed",
1459                     __func__);
1460                 goto alloc_sbuf_err;
1461         }
1462 
1463         rx_ring->sbuf_in_use = kmem_zalloc(rx_ring->sbq_len *
1464             sizeof (struct bq_desc *), KM_NOSLEEP);
1465         if (rx_ring->sbuf_in_use == NULL) {
1466                 cmn_err(CE_WARN,
1467                     "!%s: sbuf_inuse_list alloc: failed",
1468                     __func__);
1469                 goto alloc_sbuf_err;
1470         }
1471 
1472         sbq_desc = &rx_ring->sbq_desc[0];
1473 
1474         for (i = 0; i < rx_ring->sbq_len; i++, sbq_desc++) {
1475                 /* Allocate buffer */
1476                 if (ql_alloc_phys_rbuf(qlge->dip, &sbq_desc->bd_dma.dma_handle,
1477                     &ql_buf_acc_attr,
1478                     DDI_DMA_READ | DDI_DMA_STREAMING,
1479                     &sbq_desc->bd_dma.acc_handle,
1480                     (size_t)rx_ring->sbq_buf_size,   /* mem size */
1481                     (size_t)0,                          /* default alignment */
1482                     (caddr_t *)&sbq_desc->bd_dma.vaddr,
1483                     &dma_cookie) != 0) {
1484                         cmn_err(CE_WARN,
1485                             "!%s: ddi_dma_alloc_handle: failed",
1486                             __func__);
1487                         goto alloc_sbuf_err;
1488                 }
1489 
1490                 /* Set context for Return buffer callback */
1491                 sbq_desc->bd_dma.dma_addr = dma_cookie.dmac_laddress;
1492                 sbq_desc->rx_recycle.free_func = ql_release_to_sbuf_free_list;
1493                 sbq_desc->rx_recycle.free_arg  = (caddr_t)sbq_desc;
1494                 sbq_desc->rx_ring = rx_ring;
1495                 sbq_desc->upl_inuse = 0;
1496                 sbq_desc->free_buf = 0;
1497 
1498                 sbq_desc->mp =
1499                     desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr),
1500                     rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle);
1501                 if (sbq_desc->mp == NULL) {
1502                         cmn_err(CE_WARN, "%s: desballoc() failed", __func__);
1503                         goto alloc_sbuf_err;
1504                 }
1505                 ql_add_sbuf_to_free_list(rx_ring, sbq_desc);
1506         }
1507 
1508         return (DDI_SUCCESS);
1509 
1510 alloc_sbuf_err:
1511         ql_free_sbq_buffers(rx_ring);
1512         return (DDI_FAILURE);
1513 }
1514 
1515 static void
1516 ql_free_lbq_buffers(struct rx_ring *rx_ring)
1517 {
1518         struct bq_desc *lbq_desc;
1519         uint32_t i, j;
1520         int force_cnt = 0;
1521 
1522         j = rx_ring->lbq_free_head;
1523         for (i = 0; i < rx_ring->lbuf_free_count; i++) {
1524                 lbq_desc = rx_ring->lbuf_free[j];
1525                 lbq_desc->free_buf = 1;
1526                 j++;
1527                 if (j >= rx_ring->lbq_len)
1528                         j = 0;
1529                 if (lbq_desc->mp != NULL) {
1530                         freemsg(lbq_desc->mp);
1531                         lbq_desc->mp = NULL;
1532                 }
1533         }
1534         rx_ring->lbuf_free_count = 0;
1535 
1536         j = rx_ring->lbq_use_head;
1537         for (i = 0; i < rx_ring->lbuf_in_use_count; i++) {
1538                 lbq_desc = rx_ring->lbuf_in_use[j];
1539                 lbq_desc->free_buf = 1;
1540                 j++;
1541                 if (j >= rx_ring->lbq_len) {
1542                         j = 0;
1543                 }
1544                 if (lbq_desc->mp != NULL) {
1545                         freemsg(lbq_desc->mp);
1546                         lbq_desc->mp = NULL;
1547                 }
1548         }
1549         rx_ring->lbuf_in_use_count = 0;
1550 
1551         lbq_desc = &rx_ring->lbq_desc[0];
1552         for (i = 0; i < rx_ring->lbq_len; i++, lbq_desc++) {
1553                 /* Set flag so that callback will not allocate a new buffer */
1554                 lbq_desc->free_buf = 1;
1555                 if (lbq_desc->upl_inuse != 0) {
1556                         force_cnt++;
1557                 }
1558                 if (lbq_desc->bd_dma.dma_handle != NULL) {
1559                         ql_free_phys(&lbq_desc->bd_dma.dma_handle,
1560                             &lbq_desc->bd_dma.acc_handle);
1561                         lbq_desc->bd_dma.dma_handle = NULL;
1562                         lbq_desc->bd_dma.acc_handle = NULL;
1563                 }
1564         }
1565 #ifdef QLGE_LOAD_UNLOAD
1566         if (force_cnt) {
1567                 cmn_err(CE_WARN, "lbq: free %d inuse %d force %d",
1568                     rx_ring->lbuf_free_count, rx_ring->lbuf_in_use_count,
1569                     force_cnt);
1570         }
1571 #endif
1572         if (rx_ring->lbuf_in_use != NULL) {
1573                 kmem_free(rx_ring->lbuf_in_use, (rx_ring->lbq_len *
1574                     sizeof (struct bq_desc *)));
1575                 rx_ring->lbuf_in_use = NULL;
1576         }
1577 
1578         if (rx_ring->lbuf_free != NULL) {
1579                 kmem_free(rx_ring->lbuf_free, (rx_ring->lbq_len *
1580                     sizeof (struct bq_desc *)));
1581                 rx_ring->lbuf_free = NULL;
1582         }
1583 }
1584 
1585 /* Allocate large buffers */
1586 static int
1587 ql_alloc_lbufs(qlge_t *qlge, struct rx_ring *rx_ring)
1588 {
1589         struct bq_desc *lbq_desc;
1590         ddi_dma_cookie_t dma_cookie;
1591         int i;
1592         uint32_t lbq_buf_size;
1593 
1594         rx_ring->lbq_use_head = 0;
1595         rx_ring->lbq_use_tail = 0;
1596         rx_ring->lbuf_in_use_count = 0;
1597         rx_ring->lbq_free_head = 0;
1598         rx_ring->lbq_free_tail = 0;
1599         rx_ring->lbuf_free_count = 0;
1600         rx_ring->lbuf_free = kmem_zalloc(rx_ring->lbq_len *
1601             sizeof (struct bq_desc *), KM_NOSLEEP);
1602         if (rx_ring->lbuf_free == NULL) {
1603                 cmn_err(CE_WARN,
1604                     "!%s: lbuf_free_list alloc: failed",
1605                     __func__);
1606                 goto alloc_lbuf_err;
1607         }
1608 
1609         rx_ring->lbuf_in_use = kmem_zalloc(rx_ring->lbq_len *
1610             sizeof (struct bq_desc *), KM_NOSLEEP);
1611 
1612         if (rx_ring->lbuf_in_use == NULL) {
1613                 cmn_err(CE_WARN,
1614                     "!%s: lbuf_inuse_list alloc: failed",
1615                     __func__);
1616                 goto alloc_lbuf_err;
1617         }
1618 
1619         lbq_buf_size = (qlge->mtu == ETHERMTU) ?
1620             LRG_BUF_NORMAL_SIZE : LRG_BUF_JUMBO_SIZE;
1621 
1622         lbq_desc = &rx_ring->lbq_desc[0];
1623         for (i = 0; i < rx_ring->lbq_len; i++, lbq_desc++) {
1624                 rx_ring->lbq_buf_size = lbq_buf_size;
1625                 /* Allocate buffer */
1626                 if (ql_alloc_phys_rbuf(qlge->dip, &lbq_desc->bd_dma.dma_handle,
1627                     &ql_buf_acc_attr,
1628                     DDI_DMA_READ | DDI_DMA_STREAMING,
1629                     &lbq_desc->bd_dma.acc_handle,
1630                     (size_t)rx_ring->lbq_buf_size,  /* mem size */
1631                     (size_t)0, /* default alignment */
1632                     (caddr_t *)&lbq_desc->bd_dma.vaddr,
1633                     &dma_cookie) != 0) {
1634                         cmn_err(CE_WARN,
1635                             "!%s: ddi_dma_alloc_handle: failed",
1636                             __func__);
1637                         goto alloc_lbuf_err;
1638                 }
1639 
1640                 /* Set context for Return buffer callback */
1641                 lbq_desc->bd_dma.dma_addr = dma_cookie.dmac_laddress;
1642                 lbq_desc->rx_recycle.free_func = ql_release_to_lbuf_free_list;
1643                 lbq_desc->rx_recycle.free_arg  = (caddr_t)lbq_desc;
1644                 lbq_desc->rx_ring = rx_ring;
1645                 lbq_desc->upl_inuse = 0;
1646                 lbq_desc->free_buf = 0;
1647 
1648                 lbq_desc->mp =
1649                     desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr),
1650                     rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle);
1651                 if (lbq_desc->mp == NULL) {
1652                         cmn_err(CE_WARN, "%s: desballoc() failed", __func__);
1653                         goto alloc_lbuf_err;
1654                 }
1655                 ql_add_lbuf_to_free_list(rx_ring, lbq_desc);
1656         } /* For all large buffers */
1657 
1658         return (DDI_SUCCESS);
1659 
1660 alloc_lbuf_err:
1661         ql_free_lbq_buffers(rx_ring);
1662         return (DDI_FAILURE);
1663 }
1664 
1665 /*
1666  * Free rx buffers
1667  */
1668 static void
1669 ql_free_rx_buffers(qlge_t *qlge)
1670 {
1671         int i;
1672         struct rx_ring *rx_ring;
1673 
1674         for (i = 0; i < qlge->rx_ring_count; i++) {
1675                 rx_ring = &qlge->rx_ring[i];
1676                 if (rx_ring->type != TX_Q) {
1677                         ql_free_lbq_buffers(rx_ring);
1678                         ql_free_sbq_buffers(rx_ring);
1679                 }
1680         }
1681 }
1682 
1683 /*
1684  * Allocate rx buffers
1685  */
1686 static int
1687 ql_alloc_rx_buffers(qlge_t *qlge)
1688 {
1689         struct rx_ring *rx_ring;
1690         int i;
1691 
1692         for (i = 0; i < qlge->rx_ring_count; i++) {
1693                 rx_ring = &qlge->rx_ring[i];
1694                 if (rx_ring->type != TX_Q) {
1695                         if (ql_alloc_sbufs(qlge, rx_ring) != DDI_SUCCESS)
1696                                 goto alloc_err;
1697                         if (ql_alloc_lbufs(qlge, rx_ring) != DDI_SUCCESS)
1698                                 goto alloc_err;
1699                 }
1700         }
1701 #ifdef QLGE_TRACK_BUFFER_USAGE
1702         for (i = 0; i < qlge->rx_ring_count; i++) {
1703                 if (qlge->rx_ring[i].type == RX_Q) {
1704                         qlge->rx_sb_low_count[i] = NUM_SMALL_BUFFERS;
1705                         qlge->rx_lb_low_count[i] = NUM_LARGE_BUFFERS;
1706                 }
1707                 qlge->cq_low_count[i] = NUM_RX_RING_ENTRIES;
1708         }
1709 #endif
1710         return (DDI_SUCCESS);
1711 
1712 alloc_err:
1713         ql_free_rx_buffers(qlge);
1714         return (DDI_FAILURE);
1715 }
1716 
1717 /*
1718  * Initialize large buffer queue ring
1719  */
1720 static void
1721 ql_init_lbq_ring(struct rx_ring *rx_ring)
1722 {
1723         uint16_t i;
1724         struct bq_desc *lbq_desc;
1725 
1726         bzero(rx_ring->lbq_desc, rx_ring->lbq_len * sizeof (struct bq_desc));
1727         for (i = 0; i < rx_ring->lbq_len; i++) {
1728                 lbq_desc = &rx_ring->lbq_desc[i];
1729                 lbq_desc->index = i;
1730         }
1731 }
1732 
1733 /*
1734  * Initialize small buffer queue ring
1735  */
1736 static void
1737 ql_init_sbq_ring(struct rx_ring *rx_ring)
1738 {
1739         uint16_t i;
1740         struct bq_desc *sbq_desc;
1741 
1742         bzero(rx_ring->sbq_desc, rx_ring->sbq_len * sizeof (struct bq_desc));
1743         for (i = 0; i < rx_ring->sbq_len; i++) {
1744                 sbq_desc = &rx_ring->sbq_desc[i];
1745                 sbq_desc->index = i;
1746         }
1747 }
1748 
1749 /*
1750  * Calculate the pseudo-header checksum if hardware can not do
1751  */
1752 static void
1753 ql_pseudo_cksum(uint8_t *buf)
1754 {
1755         uint32_t cksum;
1756         uint16_t iphl;
1757         uint16_t proto;
1758 
1759         iphl = (uint16_t)(4 * (buf[0] & 0xF));
1760         cksum = (((uint16_t)buf[2])<<8) + buf[3] - iphl;
1761         cksum += proto = buf[9];
1762         cksum += (((uint16_t)buf[12])<<8) + buf[13];
1763         cksum += (((uint16_t)buf[14])<<8) + buf[15];
1764         cksum += (((uint16_t)buf[16])<<8) + buf[17];
1765         cksum += (((uint16_t)buf[18])<<8) + buf[19];
1766         cksum = (cksum>>16) + (cksum & 0xFFFF);
1767         cksum = (cksum>>16) + (cksum & 0xFFFF);
1768 
1769         /*
1770          * Point it to the TCP/UDP header, and
1771          * update the checksum field.
1772          */
1773         buf += iphl + ((proto == IPPROTO_TCP) ?
1774             TCP_CKSUM_OFFSET : UDP_CKSUM_OFFSET);
1775 
1776         *(uint16_t *)(void *)buf = (uint16_t)htons((uint16_t)cksum);
1777 
1778 }
1779 
1780 /*
1781  * Transmit an incoming packet.
1782  */
1783 mblk_t *
1784 ql_ring_tx(void *arg, mblk_t *mp)
1785 {
1786         struct tx_ring *tx_ring = (struct tx_ring *)arg;
1787         qlge_t *qlge = tx_ring->qlge;
1788         mblk_t *next;
1789         int rval;
1790         uint32_t tx_count = 0;
1791 
1792         if (qlge->port_link_state == LS_DOWN) {
1793                 /* can not send message while link is down */
1794                 mblk_t *tp;
1795 
1796                 while (mp != NULL) {
1797                         tp = mp->b_next;
1798                         mp->b_next = NULL;
1799                         freemsg(mp);
1800                         mp = tp;
1801                 }
1802                 goto exit;
1803         }
1804 
1805         mutex_enter(&tx_ring->tx_lock);
1806         /* if mac is not started, driver is not ready, can not send */
1807         if (tx_ring->mac_flags != QL_MAC_STARTED) {
1808                 cmn_err(CE_WARN, "%s(%d)ring not started, mode %d "
1809                     " return packets",
1810                     __func__, qlge->instance, tx_ring->mac_flags);
1811                 mutex_exit(&tx_ring->tx_lock);
1812                 goto exit;
1813         }
1814 
1815         /* we must try to send all */
1816         while (mp != NULL) {
1817                 /*
1818                  * if number of available slots is less than a threshold,
1819                  * then quit
1820                  */
1821                 if (tx_ring->tx_free_count <= TX_STOP_THRESHOLD) {
1822                         tx_ring->queue_stopped = 1;
1823                         rval = DDI_FAILURE;
1824 #ifdef QLGE_LOAD_UNLOAD
1825                         cmn_err(CE_WARN, "%s(%d) no resources",
1826                             __func__, qlge->instance);
1827 #endif
1828                         tx_ring->defer++;
1829                         /*
1830                          * If we return the buffer back we are expected to call
1831                          * mac_tx_ring_update() when resources are available
1832                          */
1833                         break;
1834                 }
1835 
1836                 next = mp->b_next;
1837                 mp->b_next = NULL;
1838 
1839                 rval = ql_send_common(tx_ring, mp);
1840 
1841                 if (rval != DDI_SUCCESS) {
1842                         mp->b_next = next;
1843                         break;
1844                 }
1845                 tx_count++;
1846                 mp = next;
1847         }
1848 
1849         /*
1850          * After all msg blocks are mapped or copied to tx buffer,
1851          * trigger the hardware to send!
1852          */
1853         if (tx_count > 0) {
1854                 ql_write_doorbell_reg(tx_ring->qlge, tx_ring->prod_idx_db_reg,
1855                     tx_ring->prod_idx);
1856         }
1857 
1858         mutex_exit(&tx_ring->tx_lock);
1859 exit:
1860         return (mp);
1861 }
1862 
1863 
1864 /*
1865  * This function builds an mblk list for the given inbound
1866  * completion.
1867  */
1868 
1869 static mblk_t *
1870 ql_build_rx_mp(qlge_t *qlge, struct rx_ring *rx_ring,
1871     struct ib_mac_iocb_rsp *ib_mac_rsp)
1872 {
1873         mblk_t *mp = NULL;
1874         mblk_t *mp1 = NULL;     /* packet header */
1875         mblk_t *mp2 = NULL;     /* packet content */
1876         struct bq_desc *lbq_desc;
1877         struct bq_desc *sbq_desc;
1878         uint32_t err_flag = (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK);
1879         uint32_t payload_len = le32_to_cpu(ib_mac_rsp->data_len);
1880         uint32_t header_len = le32_to_cpu(ib_mac_rsp->hdr_len);
1881         uint32_t pkt_len = payload_len + header_len;
1882         uint32_t done;
1883         uint64_t *curr_ial_ptr;
1884         uint32_t ial_data_addr_low;
1885         uint32_t actual_data_addr_low;
1886         mblk_t *mp_ial = NULL;  /* ial chained packets */
1887         uint32_t size;
1888         uint32_t cp_offset;
1889         boolean_t rx_copy = B_FALSE;
1890         mblk_t *tp = NULL;
1891 
1892         /*
1893          * Check if error flags are set
1894          */
1895         if (err_flag != 0) {
1896                 if ((err_flag & IB_MAC_IOCB_RSP_ERR_OVERSIZE) != 0)
1897                         rx_ring->frame_too_long++;
1898                 if ((err_flag & IB_MAC_IOCB_RSP_ERR_UNDERSIZE) != 0)
1899                         rx_ring->frame_too_short++;
1900                 if ((err_flag & IB_MAC_IOCB_RSP_ERR_CRC) != 0)
1901                         rx_ring->fcs_err++;
1902 #ifdef QLGE_LOAD_UNLOAD
1903                 cmn_err(CE_WARN, "bad packet, type 0x%x", err_flag);
1904 #endif
1905                 QL_DUMP(DBG_RX, "qlge_ring_rx: bad response iocb dump\n",
1906                     (uint8_t *)ib_mac_rsp, 8,
1907                     (size_t)sizeof (struct ib_mac_iocb_rsp));
1908         }
1909 
1910         /* header should not be in large buffer */
1911         if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HL) {
1912                 cmn_err(CE_WARN, "header in large buffer or invalid!");
1913                 err_flag |= 1;
1914         }
1915         /* if whole packet is too big than rx buffer size */
1916         if (pkt_len > qlge->max_frame_size) {
1917                 cmn_err(CE_WARN, "ql_build_rx_mpframe too long(%d)!", pkt_len);
1918                 err_flag |= 1;
1919         }
1920         if (qlge->rx_copy ||
1921             (rx_ring->sbuf_in_use_count <= qlge->rx_copy_threshold) ||
1922             (rx_ring->lbuf_in_use_count <= qlge->rx_copy_threshold)) {
1923                 rx_copy = B_TRUE;
1924         }
1925 
1926         /* if using rx copy mode, we need to allocate a big enough buffer */
1927         if (rx_copy) {
1928                 qlge->stats.norcvbuf++;
1929                 tp = allocb(payload_len + header_len + qlge->ip_hdr_offset,
1930                     BPRI_MED);
1931                 if (tp == NULL) {
1932                         cmn_err(CE_WARN, "rx copy failed to allocate memory");
1933                 } else {
1934                         tp->b_rptr += qlge->ip_hdr_offset;
1935                 }
1936         }
1937         /*
1938          * Handle the header buffer if present.
1939          * packet header must be valid and saved in one small buffer
1940          * broadcast/multicast packets' headers not splitted
1941          */
1942         if ((ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HV) &&
1943             (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS)) {
1944                 QL_PRINT(DBG_RX, ("Header of %d bytes in small buffer.\n",
1945                     header_len));
1946                 /* Sync access */
1947                 sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring);
1948 
1949                 ASSERT(sbq_desc != NULL);
1950 
1951                 /*
1952                  * Validate addresses from the ASIC with the
1953                  * expected sbuf address
1954                  */
1955                 if (cpu_to_le64(sbq_desc->bd_dma.dma_addr)
1956                     != ib_mac_rsp->hdr_addr) {
1957                         /* Small buffer address mismatch */
1958                         cmn_err(CE_WARN, "%s(%d) ring%d packet saved"
1959                             " in wrong small buffer",
1960                             __func__, qlge->instance, rx_ring->cq_id);
1961                         goto fatal_error;
1962                 }
1963                 /* get this packet */
1964                 mp1 = sbq_desc->mp;
1965                 /* Flush DMA'd data */
1966                 (void) ddi_dma_sync(sbq_desc->bd_dma.dma_handle,
1967                     0, header_len, DDI_DMA_SYNC_FORKERNEL);
1968 
1969                 if ((err_flag != 0)|| (mp1 == NULL)) {
1970                         /* failed on this packet, put it back for re-arming */
1971 #ifdef QLGE_LOAD_UNLOAD
1972                         cmn_err(CE_WARN, "get header from small buffer fail");
1973 #endif
1974                         ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
1975                         mp1 = NULL;
1976                 } else if (rx_copy) {
1977                         if (tp != NULL) {
1978                                 bcopy(sbq_desc->bd_dma.vaddr, tp->b_rptr,
1979                                     header_len);
1980                         }
1981                         ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
1982                         mp1 = NULL;
1983                 } else {
1984                         if ((qlge->ip_hdr_offset != 0)&&
1985                             (header_len < SMALL_BUFFER_SIZE)) {
1986                                 /*
1987                                  * copy entire header to a 2 bytes boundary
1988                                  * address for 8100 adapters so that the IP
1989                                  * header can be on a 4 byte boundary address
1990                                  */
1991                                 bcopy(mp1->b_rptr,
1992                                     (mp1->b_rptr + SMALL_BUFFER_SIZE +
1993                                     qlge->ip_hdr_offset),
1994                                     header_len);
1995                                 mp1->b_rptr += SMALL_BUFFER_SIZE +
1996                                     qlge->ip_hdr_offset;
1997                         }
1998 
1999                         /*
2000                          * Adjust the mp payload_len to match
2001                          * the packet header payload_len
2002                          */
2003                         mp1->b_wptr = mp1->b_rptr + header_len;
2004                         mp1->b_next = mp1->b_cont = NULL;
2005                         QL_DUMP(DBG_RX, "\t RX packet header dump:\n",
2006                             (uint8_t *)mp1->b_rptr, 8, header_len);
2007                 }
2008         }
2009 
2010         /*
2011          * packet data or whole packet can be in small or one or
2012          * several large buffer(s)
2013          */
2014         if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DS) {
2015                 /*
2016                  * The data is in a single small buffer.
2017                  */
2018                 sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring);
2019 
2020                 ASSERT(sbq_desc != NULL);
2021 
2022                 QL_PRINT(DBG_RX,
2023                     ("%d bytes in a single small buffer, sbq_desc = %p, "
2024                     "sbq_desc->bd_dma.dma_addr = %x,"
2025                     " ib_mac_rsp->data_addr = %x, mp = %p\n",
2026                     payload_len, sbq_desc, sbq_desc->bd_dma.dma_addr,
2027                     ib_mac_rsp->data_addr, sbq_desc->mp));
2028 
2029                 /*
2030                  * Validate  addresses from the ASIC with the
2031                  * expected sbuf address
2032                  */
2033                 if (cpu_to_le64(sbq_desc->bd_dma.dma_addr)
2034                     != ib_mac_rsp->data_addr) {
2035                         /* Small buffer address mismatch */
2036                         cmn_err(CE_WARN, "%s(%d) ring%d packet saved"
2037                             " in wrong small buffer",
2038                             __func__, qlge->instance, rx_ring->cq_id);
2039                         goto fatal_error;
2040                 }
2041                 /* get this packet */
2042                 mp2 = sbq_desc->mp;
2043                 (void) ddi_dma_sync(sbq_desc->bd_dma.dma_handle,
2044                     0, payload_len, DDI_DMA_SYNC_FORKERNEL);
2045                 if ((err_flag != 0) || (mp2 == NULL)) {
2046 #ifdef QLGE_LOAD_UNLOAD
2047                         /* failed on this packet, put it back for re-arming */
2048                         cmn_err(CE_WARN, "ignore bad data from small buffer");
2049 #endif
2050                         ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
2051                         mp2 = NULL;
2052                 } else if (rx_copy) {
2053                         if (tp != NULL) {
2054                                 bcopy(sbq_desc->bd_dma.vaddr,
2055                                     tp->b_rptr + header_len, payload_len);
2056                                 tp->b_wptr =
2057                                     tp->b_rptr + header_len + payload_len;
2058                         }
2059                         ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
2060                         mp2 = NULL;
2061                 } else {
2062                         /* Adjust the buffer length to match the payload_len */
2063                         mp2->b_wptr = mp2->b_rptr + payload_len;
2064                         mp2->b_next = mp2->b_cont = NULL;
2065                         /* Flush DMA'd data */
2066                         QL_DUMP(DBG_RX, "\t RX packet payload dump:\n",
2067                             (uint8_t *)mp2->b_rptr, 8, payload_len);
2068                         /*
2069                          * if payload is too small , copy to
2070                          * the end of packet header
2071                          */
2072                         if ((mp1 != NULL) &&
2073                             (payload_len <= qlge->payload_copy_thresh) &&
2074                             (pkt_len <
2075                             (SMALL_BUFFER_SIZE - qlge->ip_hdr_offset))) {
2076                                 bcopy(mp2->b_rptr, mp1->b_wptr, payload_len);
2077                                 mp1->b_wptr += payload_len;
2078                                 freemsg(mp2);
2079                                 mp2 = NULL;
2080                         }
2081                 }
2082         } else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) {
2083                 /*
2084                  * The data is in a single large buffer.
2085                  */
2086                 lbq_desc = ql_get_lbuf_from_in_use_list(rx_ring);
2087 
2088                 QL_PRINT(DBG_RX,
2089                     ("%d bytes in a single large buffer, lbq_desc = %p, "
2090                     "lbq_desc->bd_dma.dma_addr = %x,"
2091                     " ib_mac_rsp->data_addr = %x, mp = %p\n",
2092                     payload_len, lbq_desc, lbq_desc->bd_dma.dma_addr,
2093                     ib_mac_rsp->data_addr, lbq_desc->mp));
2094 
2095                 ASSERT(lbq_desc != NULL);
2096 
2097                 /*
2098                  * Validate  addresses from the ASIC with
2099                  * the expected lbuf address
2100                  */
2101                 if (cpu_to_le64(lbq_desc->bd_dma.dma_addr)
2102                     != ib_mac_rsp->data_addr) {
2103                         /* Large buffer address mismatch */
2104                         cmn_err(CE_WARN, "%s(%d) ring%d packet saved"
2105                             " in wrong large buffer",
2106                             __func__, qlge->instance, rx_ring->cq_id);
2107                         goto fatal_error;
2108                 }
2109                 mp2 = lbq_desc->mp;
2110                 /* Flush DMA'd data */
2111                 (void) ddi_dma_sync(lbq_desc->bd_dma.dma_handle,
2112                     0, payload_len, DDI_DMA_SYNC_FORKERNEL);
2113                 if ((err_flag != 0) || (mp2 == NULL)) {
2114 #ifdef QLGE_LOAD_UNLOAD
2115                         cmn_err(CE_WARN, "ignore bad data from large buffer");
2116 #endif
2117                         /* failed on this packet, put it back for re-arming */
2118                         ql_refill_lbuf_free_list(lbq_desc, B_FALSE);
2119                         mp2 = NULL;
2120                 } else if (rx_copy) {
2121                         if (tp != NULL) {
2122                                 bcopy(lbq_desc->bd_dma.vaddr,
2123                                     tp->b_rptr + header_len, payload_len);
2124                                 tp->b_wptr =
2125                                     tp->b_rptr + header_len + payload_len;
2126                         }
2127                         ql_refill_lbuf_free_list(lbq_desc, B_FALSE);
2128                         mp2 = NULL;
2129                 } else {
2130                         /*
2131                          * Adjust the buffer length to match
2132                          * the packet payload_len
2133                          */
2134                         mp2->b_wptr = mp2->b_rptr + payload_len;
2135                         mp2->b_next = mp2->b_cont = NULL;
2136                         QL_DUMP(DBG_RX, "\t RX packet payload dump:\n",
2137                             (uint8_t *)mp2->b_rptr, 8, payload_len);
2138                         /*
2139                          * if payload is too small , copy to
2140                          * the end of packet header
2141                          */
2142                         if ((mp1 != NULL) &&
2143                             (payload_len <= qlge->payload_copy_thresh) &&
2144                             (pkt_len<
2145                             (SMALL_BUFFER_SIZE - qlge->ip_hdr_offset))) {
2146                                 bcopy(mp2->b_rptr, mp1->b_wptr, payload_len);
2147                                 mp1->b_wptr += payload_len;
2148                                 freemsg(mp2);
2149                                 mp2 = NULL;
2150                         }
2151                 }
2152         } else if (payload_len) { /* ial case */
2153                 /*
2154                  * payload available but not in sml nor lrg buffer,
2155                  * so, it is saved in IAL
2156                  */
2157 #ifdef QLGE_LOAD_UNLOAD
2158                 cmn_err(CE_NOTE, "packet chained in IAL \n");
2159 #endif
2160                 /* lrg buf addresses are saved in one small buffer */
2161                 sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring);
2162                 curr_ial_ptr = (uint64_t *)sbq_desc->bd_dma.vaddr;
2163                 done = 0;
2164                 cp_offset = 0;
2165 
2166                 while (!done) {
2167                         ial_data_addr_low =
2168                             (uint32_t)(le64_to_cpu(*curr_ial_ptr) &
2169                             0xFFFFFFFE);
2170                         /* check if this is the last packet fragment */
2171                         done = (uint32_t)(le64_to_cpu(*curr_ial_ptr) & 1);
2172                         curr_ial_ptr++;
2173                         /*
2174                          * The data is in one or several large buffer(s).
2175                          */
2176                         lbq_desc = ql_get_lbuf_from_in_use_list(rx_ring);
2177                         actual_data_addr_low =
2178                             (uint32_t)(lbq_desc->bd_dma.dma_addr &
2179                             0xFFFFFFFE);
2180                         if (ial_data_addr_low != actual_data_addr_low) {
2181                                 cmn_err(CE_WARN,
2182                                     "packet saved in wrong ial lrg buffer"
2183                                     " expected %x, actual %lx",
2184                                     ial_data_addr_low,
2185                                     (uintptr_t)lbq_desc->bd_dma.dma_addr);
2186                                 goto fatal_error;
2187                         }
2188 
2189                         size = (payload_len < rx_ring->lbq_buf_size)?
2190                             payload_len : rx_ring->lbq_buf_size;
2191                         payload_len -= size;
2192                         mp2 = lbq_desc->mp;
2193                         if ((err_flag != 0) || (mp2 == NULL)) {
2194 #ifdef QLGE_LOAD_UNLOAD
2195                                 cmn_err(CE_WARN,
2196                                     "ignore bad data from large buffer");
2197 #endif
2198                                 ql_refill_lbuf_free_list(lbq_desc, B_FALSE);
2199                                 mp2 = NULL;
2200                         } else if (rx_copy) {
2201                                 if (tp != NULL) {
2202                                         (void) ddi_dma_sync(
2203                                             lbq_desc->bd_dma.dma_handle,
2204                                             0, size, DDI_DMA_SYNC_FORKERNEL);
2205                                         bcopy(lbq_desc->bd_dma.vaddr,
2206                                             tp->b_rptr + header_len + cp_offset,
2207                                             size);
2208                                         tp->b_wptr =
2209                                             tp->b_rptr + size + cp_offset +
2210                                             header_len;
2211                                         cp_offset += size;
2212                                 }
2213                                 ql_refill_lbuf_free_list(lbq_desc, B_FALSE);
2214                                 mp2 = NULL;
2215                         } else {
2216                                 if (mp_ial == NULL) {
2217                                         mp_ial = mp2;
2218                                 } else {
2219                                         linkb(mp_ial, mp2);
2220                                 }
2221 
2222                                 mp2->b_next = NULL;
2223                                 mp2->b_cont = NULL;
2224                                 mp2->b_wptr = mp2->b_rptr + size;
2225                                 /* Flush DMA'd data */
2226                                 (void) ddi_dma_sync(lbq_desc->bd_dma.dma_handle,
2227                                     0, size, DDI_DMA_SYNC_FORKERNEL);
2228                                 QL_PRINT(DBG_RX, ("ial %d payload received \n",
2229                                     size));
2230                                 QL_DUMP(DBG_RX, "\t Mac data dump:\n",
2231                                     (uint8_t *)mp2->b_rptr, 8, size);
2232                         }
2233                 }
2234                 if (err_flag != 0) {
2235 #ifdef QLGE_LOAD_UNLOAD
2236                         /* failed on this packet, put it back for re-arming */
2237                         cmn_err(CE_WARN, "ignore bad data from small buffer");
2238 #endif
2239                         ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
2240                 } else {
2241                         mp2 = mp_ial;
2242                         freemsg(sbq_desc->mp);
2243                 }
2244         }
2245         /*
2246          * some packets' hdr not split, then send mp2 upstream, otherwise,
2247          * concatenate message block mp2 to the tail of message header, mp1
2248          */
2249         if (!err_flag) {
2250                 if (rx_copy) {
2251                         if (tp != NULL) {
2252                                 tp->b_next = NULL;
2253                                 tp->b_cont = NULL;
2254                                 tp->b_wptr = tp->b_rptr +
2255                                     header_len + payload_len;
2256                         }
2257                         mp = tp;
2258                 } else {
2259                         if (mp1) {
2260                                 if (mp2) {
2261                                         QL_PRINT(DBG_RX,
2262                                             ("packet in mp1 and mp2\n"));
2263                                         /* mp1->b_cont = mp2; */
2264                                         linkb(mp1, mp2);
2265                                         mp = mp1;
2266                                 } else {
2267                                         QL_PRINT(DBG_RX,
2268                                             ("packet in mp1 only\n"));
2269                                         mp = mp1;
2270                                 }
2271                         } else if (mp2) {
2272                                 QL_PRINT(DBG_RX, ("packet in mp2 only\n"));
2273                                 mp = mp2;
2274                         }
2275                 }
2276         }
2277         return (mp);
2278 
2279 fatal_error:
2280         /* fatal Error! */
2281         if (qlge->fm_enable) {
2282                 ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED);
2283                 ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
2284                 atomic_or_32(&qlge->flags, ADAPTER_ERROR);
2285         }
2286         if (tp) {
2287                 freemsg(tp);
2288         }
2289 
2290         /* *mp->b_wptr = 0; */
2291         ql_wake_asic_reset_soft_intr(qlge);
2292         return (NULL);
2293 
2294 }
2295 
2296 /*
2297  * Bump completion queue consumer index.
2298  */
2299 static void
2300 ql_update_cq(struct rx_ring *rx_ring)
2301 {
2302         rx_ring->cnsmr_idx++;
2303         rx_ring->curr_entry++;
2304         if (rx_ring->cnsmr_idx >= rx_ring->cq_len) {
2305                 rx_ring->cnsmr_idx = 0;
2306                 rx_ring->curr_entry = rx_ring->cq_dma.vaddr;
2307         }
2308 }
2309 
2310 /*
2311  * Update completion queue consumer index.
2312  */
2313 static void
2314 ql_write_cq_idx(struct rx_ring *rx_ring)
2315 {
2316         qlge_t *qlge = rx_ring->qlge;
2317 
2318         ql_write_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg,
2319             rx_ring->cnsmr_idx);
2320 }
2321 
2322 /*
2323  * Processes a SYS-Chip Event Notification Completion Event.
2324  * The incoming notification event that describes a link up/down
2325  * or some sorts of error happens.
2326  */
2327 static void
2328 ql_process_chip_ae_intr(qlge_t *qlge,
2329     struct ib_sys_event_iocb_rsp *ib_sys_event_rsp_ptr)
2330 {
2331         uint8_t eventType = ib_sys_event_rsp_ptr->event_type;
2332         uint32_t soft_req = 0;
2333 
2334         switch (eventType) {
2335                 case SYS_EVENT_PORT_LINK_UP:    /* 0x0h */
2336                         QL_PRINT(DBG_MBX, ("Port Link Up\n"));
2337                         break;
2338 
2339                 case SYS_EVENT_PORT_LINK_DOWN:  /* 0x1h */
2340                         QL_PRINT(DBG_MBX, ("Port Link Down\n"));
2341                         break;
2342 
2343                 case SYS_EVENT_MULTIPLE_CAM_HITS : /* 0x6h */
2344                         cmn_err(CE_WARN, "A multiple CAM hits look up error "
2345                             "occurred");
2346                         soft_req |= NEED_HW_RESET;
2347                         break;
2348 
2349                 case SYS_EVENT_SOFT_ECC_ERR:    /* 0x7h */
2350                         cmn_err(CE_WARN, "Soft ECC error detected");
2351                         soft_req |= NEED_HW_RESET;
2352                         break;
2353 
2354                 case SYS_EVENT_MGMT_FATAL_ERR:  /* 0x8h */
2355                         cmn_err(CE_WARN, "Management (MPI) Processor fatal"
2356                             " error occured");
2357                         soft_req |= NEED_MPI_RESET;
2358                         break;
2359 
2360                 case SYS_EVENT_MAC_INTERRUPT:   /* 0x9h */
2361                         QL_PRINT(DBG_MBX, ("MAC Interrupt"));
2362                         break;
2363 
2364                 case SYS_EVENT_PCI_ERR_READING_SML_LRG_BUF:     /* 0x40h */
2365                         cmn_err(CE_WARN, "PCI Error reading small/large "
2366                             "buffers occured");
2367                         soft_req |= NEED_HW_RESET;
2368                         break;
2369 
2370                 default:
2371                         QL_PRINT(DBG_RX, ("%s(%d) unknown Sys Event: "
2372                             "type 0x%x occured",
2373                             __func__, qlge->instance, eventType));
2374                         break;
2375         }
2376 
2377         if ((soft_req & NEED_MPI_RESET) != 0) {
2378                 ql_wake_mpi_reset_soft_intr(qlge);
2379                 if (qlge->fm_enable) {
2380                         ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
2381                         ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED);
2382                 }
2383         } else if ((soft_req & NEED_HW_RESET) != 0) {
2384                 ql_wake_asic_reset_soft_intr(qlge);
2385                 if (qlge->fm_enable) {
2386                         ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
2387                         ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED);
2388                 }
2389         }
2390 }
2391 
2392 /*
2393  * set received packet checksum flag
2394  */
2395 void
2396 ql_set_rx_cksum(mblk_t *mp, struct ib_mac_iocb_rsp *net_rsp)
2397 {
2398         uint32_t flags;
2399 
2400         /* Not TCP or UDP packet? nothing more to do */
2401         if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_T) == 0) &&
2402             ((net_rsp->flags2 & IB_MAC_IOCB_RSP_U) == 0))
2403         return;
2404 
2405         /* No CKO support for IPv6 */
2406         if ((net_rsp->flags3 & IB_MAC_IOCB_RSP_V6) != 0)
2407                 return;
2408 
2409         /*
2410          * If checksum error, don't set flags; stack will calculate
2411          * checksum, detect the error and update statistics
2412          */
2413         if (((net_rsp->flags1 & IB_MAC_IOCB_RSP_TE) != 0) ||
2414             ((net_rsp->flags1 & IB_MAC_IOCB_RSP_IE) != 0))
2415                 return;
2416 
2417         /* TCP or UDP packet and checksum valid */
2418         if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_T) != 0) &&
2419             ((net_rsp->flags1 & IB_MAC_IOCB_RSP_NU) == 0)) {
2420                 flags = HCK_FULLCKSUM_OK;
2421                 mac_hcksum_set(mp, 0, 0, 0, 0, flags);
2422         }
2423         if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_U) != 0) &&
2424             ((net_rsp->flags1 & IB_MAC_IOCB_RSP_NU) == 0)) {
2425                 flags = HCK_FULLCKSUM_OK;
2426                 mac_hcksum_set(mp, 0, 0, 0, 0, flags);
2427         }
2428 }
2429 
2430 /*
2431  * This function goes through h/w descriptor in one specified rx ring,
2432  * receives the data if the descriptor status shows the data is ready.
2433  * It returns a chain of mblks containing the received data, to be
2434  * passed up to mac_rx_ring().
2435  */
2436 mblk_t *
2437 ql_ring_rx(struct rx_ring *rx_ring, int poll_bytes)
2438 {
2439         qlge_t *qlge = rx_ring->qlge;
2440         uint32_t prod = ql_read_sh_reg(qlge, rx_ring);
2441         struct ib_mac_iocb_rsp *net_rsp;
2442         mblk_t *mp;
2443         mblk_t *mblk_head;
2444         mblk_t **mblk_tail;
2445         uint32_t received_bytes = 0;
2446         uint32_t length;
2447 #ifdef QLGE_PERFORMANCE
2448         uint32_t pkt_ct = 0;
2449 #endif
2450 
2451 #ifdef QLGE_TRACK_BUFFER_USAGE
2452         uint32_t consumer_idx;
2453         uint32_t producer_idx;
2454         uint32_t num_free_entries;
2455         uint32_t temp;
2456 
2457         temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg);
2458         consumer_idx = temp & 0x0000ffff;
2459         producer_idx = (temp >> 16);
2460 
2461         if (consumer_idx > producer_idx)
2462                 num_free_entries = (consumer_idx - producer_idx);
2463         else
2464                 num_free_entries = NUM_RX_RING_ENTRIES - (
2465                     producer_idx - consumer_idx);
2466 
2467         if (num_free_entries < qlge->cq_low_count[rx_ring->cq_id])
2468                 qlge->cq_low_count[rx_ring->cq_id] = num_free_entries;
2469 
2470 #endif
2471         mblk_head = NULL;
2472         mblk_tail = &mblk_head;
2473 
2474         while ((prod != rx_ring->cnsmr_idx)) {
2475                 QL_PRINT(DBG_RX,
2476                     ("%s cq_id = %d, prod = %d, cnsmr = %d.\n",
2477                     __func__, rx_ring->cq_id, prod, rx_ring->cnsmr_idx));
2478 
2479                 net_rsp = (struct ib_mac_iocb_rsp *)rx_ring->curr_entry;
2480                 (void) ddi_dma_sync(rx_ring->cq_dma.dma_handle,
2481                     (off_t)((uintptr_t)net_rsp -
2482                     (uintptr_t)rx_ring->cq_dma.vaddr),
2483                     (size_t)sizeof (*net_rsp), DDI_DMA_SYNC_FORKERNEL);
2484                 QL_DUMP(DBG_RX, "qlge_ring_rx: rx completion iocb\n",
2485                     rx_ring->curr_entry, 8, (size_t)sizeof (*net_rsp));
2486 
2487                 switch (net_rsp->opcode) {
2488 
2489                 case OPCODE_IB_MAC_IOCB:
2490                         /* Adding length of pkt header and payload */
2491                         length = le32_to_cpu(net_rsp->data_len) +
2492                             le32_to_cpu(net_rsp->hdr_len);
2493                         if ((poll_bytes != QLGE_POLL_ALL) &&
2494                             ((received_bytes + length) > poll_bytes)) {
2495                                 continue;
2496                         }
2497                         received_bytes += length;
2498 
2499 #ifdef QLGE_PERFORMANCE
2500                         pkt_ct++;
2501 #endif
2502                         mp = ql_build_rx_mp(qlge, rx_ring, net_rsp);
2503                         if (mp != NULL) {
2504                                 if (rx_ring->mac_flags != QL_MAC_STARTED) {
2505                                         /*
2506                                          * Increment number of packets we have
2507                                          * indicated to the stack, should be
2508                                          * decremented when we get it back
2509                                          * or when freemsg is called
2510                                          */
2511                                         ASSERT(rx_ring->rx_indicate
2512                                             <= rx_ring->cq_len);
2513 #ifdef QLGE_LOAD_UNLOAD
2514                                         cmn_err(CE_WARN, "%s do not send to OS,"
2515                                             " mac_flags %d, indicate %d",
2516                                             __func__, rx_ring->mac_flags,
2517                                             rx_ring->rx_indicate);
2518 #endif
2519                                         QL_PRINT(DBG_RX,
2520                                             ("cq_id = %d, packet "
2521                                             "dropped, mac not "
2522                                             "enabled.\n",
2523                                             rx_ring->cq_id));
2524                                         rx_ring->rx_pkt_dropped_mac_unenabled++;
2525 
2526                                         /* rx_lock is expected to be held */
2527                                         mutex_exit(&rx_ring->rx_lock);
2528                                         freemsg(mp);
2529                                         mutex_enter(&rx_ring->rx_lock);
2530                                         mp = NULL;
2531                                 }
2532 
2533                                 if (mp != NULL) {
2534                                         /*
2535                                          * IP full packet has been
2536                                          * successfully verified by
2537                                          * H/W and is correct
2538                                          */
2539                                         ql_set_rx_cksum(mp, net_rsp);
2540 
2541                                         rx_ring->rx_packets++;
2542                                         rx_ring->rx_bytes += length;
2543                                         *mblk_tail = mp;
2544                                         mblk_tail = &mp->b_next;
2545                                 }
2546                         } else {
2547                                 QL_PRINT(DBG_RX,
2548                                     ("cq_id = %d, packet dropped\n",
2549                                     rx_ring->cq_id));
2550                                 rx_ring->rx_packets_dropped_no_buffer++;
2551                         }
2552                         break;
2553 
2554                 case OPCODE_IB_SYS_EVENT_IOCB:
2555                         ql_process_chip_ae_intr(qlge,
2556                             (struct ib_sys_event_iocb_rsp *)
2557                             net_rsp);
2558                         break;
2559 
2560                 default:
2561                         cmn_err(CE_WARN,
2562                             "%s Ring(%d)Hit default case, not handled!"
2563                             " dropping the packet, "
2564                             "opcode = %x.", __func__, rx_ring->cq_id,
2565                             net_rsp->opcode);
2566                         break;
2567                 }
2568                 /* increment cnsmr_idx and curr_entry */
2569                 ql_update_cq(rx_ring);
2570                 prod = ql_read_sh_reg(qlge, rx_ring);
2571 
2572         }
2573 
2574 #ifdef QLGE_PERFORMANCE
2575         if (pkt_ct >= 7)
2576                 rx_ring->hist[7]++;
2577         else if (pkt_ct == 6)
2578                 rx_ring->hist[6]++;
2579         else if (pkt_ct == 5)
2580                 rx_ring->hist[5]++;
2581         else if (pkt_ct == 4)
2582                 rx_ring->hist[4]++;
2583         else if (pkt_ct == 3)
2584                 rx_ring->hist[3]++;
2585         else if (pkt_ct == 2)
2586                 rx_ring->hist[2]++;
2587         else if (pkt_ct == 1)
2588                 rx_ring->hist[1]++;
2589         else if (pkt_ct == 0)
2590                 rx_ring->hist[0]++;
2591 #endif
2592 
2593         /* update cnsmr_idx */
2594         ql_write_cq_idx(rx_ring);
2595         /* do not enable interrupt for polling mode */
2596         if (poll_bytes == QLGE_POLL_ALL)
2597                 ql_enable_completion_interrupt(rx_ring->qlge, rx_ring->irq);
2598         return (mblk_head);
2599 }
2600 
2601 /* Process an outbound completion from an rx ring. */
2602 static void
2603 ql_process_mac_tx_intr(qlge_t *qlge, struct ob_mac_iocb_rsp *mac_rsp)
2604 {
2605         struct tx_ring *tx_ring;
2606         struct tx_ring_desc *tx_ring_desc;
2607         int j;
2608 
2609         tx_ring = &qlge->tx_ring[mac_rsp->txq_idx];
2610         tx_ring_desc = tx_ring->wq_desc;
2611         tx_ring_desc += mac_rsp->tid;
2612 
2613         if (tx_ring_desc->tx_type == USE_DMA) {
2614                 QL_PRINT(DBG_TX, ("%s(%d): tx type USE_DMA\n",
2615                     __func__, qlge->instance));
2616 
2617                 /*
2618                  * Release the DMA resource that is used for
2619                  * DMA binding.
2620                  */
2621                 for (j = 0; j < tx_ring_desc->tx_dma_handle_used; j++) {
2622                         (void) ddi_dma_unbind_handle(
2623                             tx_ring_desc->tx_dma_handle[j]);
2624                 }
2625 
2626                 tx_ring_desc->tx_dma_handle_used = 0;
2627                 /*
2628                  * Free the mblk after sending completed
2629                  */
2630                 if (tx_ring_desc->mp != NULL) {
2631                         freemsg(tx_ring_desc->mp);
2632                         tx_ring_desc->mp = NULL;
2633                 }
2634         }
2635 
2636         tx_ring->obytes += tx_ring_desc->tx_bytes;
2637         tx_ring->opackets++;
2638 
2639         if (mac_rsp->flags1 & (OB_MAC_IOCB_RSP_E | OB_MAC_IOCB_RSP_S |
2640             OB_MAC_IOCB_RSP_L | OB_MAC_IOCB_RSP_B)) {
2641                 tx_ring->errxmt++;
2642                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_E) {
2643                         /* EMPTY */
2644                         QL_PRINT(DBG_TX,
2645                             ("Total descriptor length did not match "
2646                             "transfer length.\n"));
2647                 }
2648                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_S) {
2649                         /* EMPTY */
2650                         QL_PRINT(DBG_TX,
2651                             ("Frame too short to be legal, not sent.\n"));
2652                 }
2653                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_L) {
2654                         /* EMPTY */
2655                         QL_PRINT(DBG_TX,
2656                             ("Frame too long, but sent anyway.\n"));
2657                 }
2658                 if (mac_rsp->flags3 & OB_MAC_IOCB_RSP_B) {
2659                         /* EMPTY */
2660                         QL_PRINT(DBG_TX,
2661                             ("PCI backplane error. Frame not sent.\n"));
2662                 }
2663         }
2664         atomic_inc_32(&tx_ring->tx_free_count);
2665 }
2666 
2667 /*
2668  * clean up tx completion iocbs
2669  */
2670 int
2671 ql_clean_outbound_rx_ring(struct rx_ring *rx_ring)
2672 {
2673         qlge_t *qlge = rx_ring->qlge;
2674         uint32_t prod = ql_read_sh_reg(qlge, rx_ring);
2675         struct ob_mac_iocb_rsp *net_rsp = NULL;
2676         int count = 0;
2677         struct tx_ring *tx_ring;
2678         boolean_t resume_tx = B_FALSE;
2679 
2680         mutex_enter(&rx_ring->rx_lock);
2681 #ifdef QLGE_TRACK_BUFFER_USAGE
2682         {
2683         uint32_t consumer_idx;
2684         uint32_t producer_idx;
2685         uint32_t num_free_entries;
2686         uint32_t temp;
2687 
2688         temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg);
2689         consumer_idx = temp & 0x0000ffff;
2690         producer_idx = (temp >> 16);
2691 
2692         if (consumer_idx > producer_idx)
2693                 num_free_entries = (consumer_idx - producer_idx);
2694         else
2695                 num_free_entries = NUM_RX_RING_ENTRIES -
2696                     (producer_idx - consumer_idx);
2697 
2698         if (num_free_entries < qlge->cq_low_count[rx_ring->cq_id])
2699                 qlge->cq_low_count[rx_ring->cq_id] = num_free_entries;
2700 
2701         }
2702 #endif
2703         /* While there are entries in the completion queue. */
2704         while (prod != rx_ring->cnsmr_idx) {
2705 
2706                 QL_PRINT(DBG_RX,
2707                     ("%s cq_id = %d, prod = %d, cnsmr = %d.\n", __func__,
2708                     rx_ring->cq_id, prod, rx_ring->cnsmr_idx));
2709 
2710                 net_rsp = (struct ob_mac_iocb_rsp *)rx_ring->curr_entry;
2711                 (void) ddi_dma_sync(rx_ring->cq_dma.dma_handle,
2712                     (off_t)((uintptr_t)net_rsp -
2713                     (uintptr_t)rx_ring->cq_dma.vaddr),
2714                     (size_t)sizeof (*net_rsp), DDI_DMA_SYNC_FORKERNEL);
2715 
2716                 QL_DUMP(DBG_RX, "ql_clean_outbound_rx_ring: "
2717                     "response packet data\n",
2718                     rx_ring->curr_entry, 8,
2719                     (size_t)sizeof (*net_rsp));
2720 
2721                 switch (net_rsp->opcode) {
2722 
2723                 case OPCODE_OB_MAC_OFFLOAD_IOCB:
2724                 case OPCODE_OB_MAC_IOCB:
2725                         ql_process_mac_tx_intr(qlge, net_rsp);
2726                         break;
2727 
2728                 default:
2729                         cmn_err(CE_WARN,
2730                             "%s Hit default case, not handled! "
2731                             "dropping the packet,"
2732                             " opcode = %x.",
2733                             __func__, net_rsp->opcode);
2734                         break;
2735                 }
2736                 count++;
2737                 ql_update_cq(rx_ring);
2738                 prod = ql_read_sh_reg(qlge, rx_ring);
2739         }
2740         ql_write_cq_idx(rx_ring);
2741 
2742         mutex_exit(&rx_ring->rx_lock);
2743 
2744         net_rsp = (struct ob_mac_iocb_rsp *)rx_ring->curr_entry;
2745         tx_ring = &qlge->tx_ring[net_rsp->txq_idx];
2746 
2747         mutex_enter(&tx_ring->tx_lock);
2748 
2749         if (tx_ring->queue_stopped &&
2750             (tx_ring->tx_free_count > TX_RESUME_THRESHOLD)) {
2751                 /*
2752                  * The queue got stopped because the tx_ring was full.
2753                  * Wake it up, because it's now at least 25% empty.
2754                  */
2755                 tx_ring->queue_stopped = 0;
2756                 resume_tx = B_TRUE;
2757         }
2758 
2759         mutex_exit(&tx_ring->tx_lock);
2760         /* Don't hold the lock during OS callback */
2761         if (resume_tx)
2762                 RESUME_TX(tx_ring);
2763         return (count);
2764 }
2765 
2766 /*
2767  * reset asic when error happens
2768  */
2769 /* ARGSUSED */
2770 static uint_t
2771 ql_asic_reset_work(caddr_t arg1, caddr_t arg2)
2772 {
2773         qlge_t *qlge = (qlge_t *)((void *)arg1);
2774         int status;
2775 
2776         mutex_enter(&qlge->gen_mutex);
2777         (void) ql_do_stop(qlge);
2778         /*
2779          * Write default ethernet address to chip register Mac
2780          * Address slot 0 and Enable Primary Mac Function.
2781          */
2782         mutex_enter(&qlge->hw_mutex);
2783         (void) ql_unicst_set(qlge,
2784             (uint8_t *)qlge->unicst_addr[0].addr.ether_addr_octet, 0);
2785         mutex_exit(&qlge->hw_mutex);
2786         qlge->mac_flags = QL_MAC_INIT;
2787         status = ql_do_start(qlge);
2788         if (status != DDI_SUCCESS)
2789                 goto error;
2790         qlge->mac_flags = QL_MAC_STARTED;
2791         mutex_exit(&qlge->gen_mutex);
2792         ddi_fm_service_impact(qlge->dip, DDI_SERVICE_RESTORED);
2793 
2794         return (DDI_INTR_CLAIMED);
2795 
2796 error:
2797         mutex_exit(&qlge->gen_mutex);
2798         cmn_err(CE_WARN,
2799             "qlge up/down cycle failed, closing device");
2800         if (qlge->fm_enable) {
2801                 ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
2802                 ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST);
2803                 atomic_or_32(&qlge->flags, ADAPTER_ERROR);
2804         }
2805         return (DDI_INTR_CLAIMED);
2806 }
2807 
2808 /*
2809  * Reset MPI
2810  */
2811 /* ARGSUSED */
2812 static uint_t
2813 ql_mpi_reset_work(caddr_t arg1, caddr_t arg2)
2814 {
2815         qlge_t *qlge = (qlge_t *)((void *)arg1);
2816 
2817         (void) ql_reset_mpi_risc(qlge);
2818         return (DDI_INTR_CLAIMED);
2819 }
2820 
2821 /*
2822  * Process MPI mailbox messages
2823  */
2824 /* ARGSUSED */
2825 static uint_t
2826 ql_mpi_event_work(caddr_t arg1, caddr_t arg2)
2827 {
2828         qlge_t *qlge = (qlge_t *)((void *)arg1);
2829 
2830         ql_do_mpi_intr(qlge);
2831         return (DDI_INTR_CLAIMED);
2832 }
2833 
2834 /* Fire up a handler to reset the MPI processor. */
2835 void
2836 ql_wake_asic_reset_soft_intr(qlge_t *qlge)
2837 {
2838         (void) ddi_intr_trigger_softint(qlge->asic_reset_intr_hdl, NULL);
2839 }
2840 
2841 static void
2842 ql_wake_mpi_reset_soft_intr(qlge_t *qlge)
2843 {
2844         (void) ddi_intr_trigger_softint(qlge->mpi_reset_intr_hdl, NULL);
2845 }
2846 
2847 static void
2848 ql_wake_mpi_event_soft_intr(qlge_t *qlge)
2849 {
2850         (void) ddi_intr_trigger_softint(qlge->mpi_event_intr_hdl, NULL);
2851 }
2852 
2853 /*
2854  * This handles a fatal error, MPI activity, and the default
2855  * rx_ring in an MSI-X multiple interrupt vector environment.
2856  * In MSI/Legacy environment it also process the rest of
2857  * the rx_rings.
2858  */
2859 /* ARGSUSED */
2860 static uint_t
2861 ql_isr(caddr_t arg1, caddr_t arg2)
2862 {
2863         struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
2864         struct rx_ring *ob_ring;
2865         qlge_t *qlge = rx_ring->qlge;
2866         struct intr_ctx *intr_ctx = &qlge->intr_ctx[0];
2867         uint32_t var, prod;
2868         int i;
2869         int work_done = 0;
2870 
2871         mblk_t *mp;
2872 
2873         _NOTE(ARGUNUSED(arg2));
2874 
2875         ++qlge->rx_interrupts[rx_ring->cq_id];
2876 
2877         if (ql_atomic_read_32(&qlge->intr_ctx[0].irq_cnt)) {
2878                 ql_write_reg(qlge, REG_RSVD7, 0xfeed0002);
2879                 var = ql_read_reg(qlge, REG_ERROR_STATUS);
2880                 var = ql_read_reg(qlge, REG_STATUS);
2881                 var = ql_read_reg(qlge, REG_INTERRUPT_STATUS_1);
2882                 return (DDI_INTR_CLAIMED);
2883         }
2884 
2885         ql_disable_completion_interrupt(qlge, intr_ctx->intr);
2886 
2887         /*
2888          * process send completes on first stride tx ring if available
2889          */
2890         if (qlge->isr_stride) {
2891                 ob_ring = &qlge->rx_ring[qlge->isr_stride];
2892                 if (ql_read_sh_reg(qlge, ob_ring) !=
2893                     ob_ring->cnsmr_idx) {
2894                         (void) ql_clean_outbound_rx_ring(ob_ring);
2895                 }
2896         }
2897         /*
2898          * Check the default queue and wake handler if active.
2899          */
2900         rx_ring = &qlge->rx_ring[0];
2901         prod = ql_read_sh_reg(qlge, rx_ring);
2902         QL_PRINT(DBG_INTR, ("rx-ring[0] prod index 0x%x, consumer 0x%x ",
2903             prod, rx_ring->cnsmr_idx));
2904         /* check if interrupt is due to incoming packet */
2905         if (prod != rx_ring->cnsmr_idx) {
2906                 QL_PRINT(DBG_INTR, ("Waking handler for rx_ring[0].\n"));
2907                 ql_disable_completion_interrupt(qlge, intr_ctx->intr);
2908                 mutex_enter(&rx_ring->rx_lock);
2909                 mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
2910                 mutex_exit(&rx_ring->rx_lock);
2911 
2912                 if (mp != NULL)
2913                         RX_UPSTREAM(rx_ring, mp);
2914                 work_done++;
2915         } else {
2916                 /*
2917                  * If interrupt is not due to incoming packet, read status
2918                  * register to see if error happens or mailbox interrupt.
2919                  */
2920                 var = ql_read_reg(qlge, REG_STATUS);
2921                 if ((var & STATUS_FE) != 0) {
2922                         ql_write_reg(qlge, REG_RSVD7, 0xfeed0003);
2923                         if (qlge->fm_enable) {
2924                                 atomic_or_32(&qlge->flags, ADAPTER_ERROR);
2925                                 ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
2926                                 ddi_fm_service_impact(qlge->dip,
2927                                     DDI_SERVICE_LOST);
2928                         }
2929                         cmn_err(CE_WARN, "Got fatal error, STS = %x.", var);
2930                         var = ql_read_reg(qlge, REG_ERROR_STATUS);
2931                         cmn_err(CE_WARN,
2932                             "Resetting chip. Error Status Register = 0x%x",
2933                             var);
2934                         ql_wake_asic_reset_soft_intr(qlge);
2935                         return (DDI_INTR_CLAIMED);
2936                 }
2937 
2938                 /*
2939                  * Check MPI processor activity.
2940                  */
2941                 if ((var & STATUS_PI) != 0) {
2942                         /*
2943                          * We've got an async event or mailbox completion.
2944                          * Handle it and clear the source of the interrupt.
2945                          */
2946                         ql_write_reg(qlge, REG_RSVD7, 0xfeed0004);
2947 
2948                         QL_PRINT(DBG_INTR, ("Got MPI processor interrupt.\n"));
2949                         ql_disable_completion_interrupt(qlge, intr_ctx->intr);
2950                         ql_wake_mpi_event_soft_intr(qlge);
2951                         work_done++;
2952                 }
2953         }
2954 
2955 
2956         if (qlge->intr_type != DDI_INTR_TYPE_MSIX) {
2957                 /*
2958                  * Start the DPC for each active queue.
2959                  */
2960                 for (i = 1; i < qlge->rx_ring_count; i++) {
2961                         rx_ring = &qlge->rx_ring[i];
2962 
2963                         if (ql_read_sh_reg(qlge, rx_ring) !=
2964                             rx_ring->cnsmr_idx) {
2965                                 QL_PRINT(DBG_INTR,
2966                                     ("Waking handler for rx_ring[%d].\n", i));
2967 
2968                                 ql_disable_completion_interrupt(qlge,
2969                                     rx_ring->irq);
2970                                 if (rx_ring->type == TX_Q) {
2971                                         (void) ql_clean_outbound_rx_ring(
2972                                             rx_ring);
2973                                         ql_enable_completion_interrupt(
2974                                             rx_ring->qlge, rx_ring->irq);
2975                                 } else {
2976                                         mutex_enter(&rx_ring->rx_lock);
2977                                         mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
2978                                         mutex_exit(&rx_ring->rx_lock);
2979                                         if (mp != NULL)
2980                                                 RX_UPSTREAM(rx_ring, mp);
2981 #ifdef QLGE_LOAD_UNLOAD
2982                                         if (rx_ring->mac_flags ==
2983                                             QL_MAC_STOPPED)
2984                                                 cmn_err(CE_NOTE,
2985                                                     "%s rx_indicate(%d) %d\n",
2986                                                     __func__, i,
2987                                                     rx_ring->rx_indicate);
2988 #endif
2989                                 }
2990                                 work_done++;
2991                         }
2992                 }
2993         }
2994 
2995         ql_enable_completion_interrupt(qlge, intr_ctx->intr);
2996 
2997         return (work_done ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
2998 }
2999 
3000 /*
3001  * MSI-X Multiple Vector Interrupt Handler for outbound (TX) completions.
3002  */
3003 /* ARGSUSED */
3004 static uint_t
3005 ql_msix_tx_isr(caddr_t arg1, caddr_t arg2)
3006 {
3007         struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
3008         qlge_t *qlge = rx_ring->qlge;
3009         _NOTE(ARGUNUSED(arg2));
3010 
3011         ++qlge->rx_interrupts[rx_ring->cq_id];
3012         (void) ql_clean_outbound_rx_ring(rx_ring);
3013         ql_enable_completion_interrupt(rx_ring->qlge, rx_ring->irq);
3014 
3015         return (DDI_INTR_CLAIMED);
3016 }
3017 
3018 /*
3019  * MSI-X Multiple Vector Interrupt Handler
3020  */
3021 /* ARGSUSED */
3022 static uint_t
3023 ql_msix_isr(caddr_t arg1, caddr_t arg2)
3024 {
3025         struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
3026         struct rx_ring *ob_ring;
3027         qlge_t *qlge = rx_ring->qlge;
3028         mblk_t *mp;
3029         _NOTE(ARGUNUSED(arg2));
3030 
3031         QL_PRINT(DBG_INTR, ("%s for ring %d\n", __func__, rx_ring->cq_id));
3032 
3033         ql_disable_completion_interrupt(qlge, rx_ring->irq);
3034 
3035         /*
3036          * process send completes on stride tx ring if available
3037          */
3038         if (qlge->isr_stride) {
3039                 ob_ring = rx_ring + qlge->isr_stride;
3040                 if (ql_read_sh_reg(qlge, ob_ring) !=
3041                     ob_ring->cnsmr_idx) {
3042                         ++qlge->rx_interrupts[ob_ring->cq_id];
3043                         (void) ql_clean_outbound_rx_ring(ob_ring);
3044                 }
3045         }
3046 
3047         ++qlge->rx_interrupts[rx_ring->cq_id];
3048 
3049         mutex_enter(&rx_ring->rx_lock);
3050         mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
3051         mutex_exit(&rx_ring->rx_lock);
3052 
3053         if (mp != NULL)
3054                 RX_UPSTREAM(rx_ring, mp);
3055 
3056         return (DDI_INTR_CLAIMED);
3057 }
3058 
3059 /*
3060  * Poll n_bytes of chained incoming packets
3061  */
3062 mblk_t *
3063 ql_ring_rx_poll(void *arg, int n_bytes)
3064 {
3065         struct rx_ring *rx_ring = (struct rx_ring *)arg;
3066         qlge_t *qlge = rx_ring->qlge;
3067         mblk_t *mp = NULL;
3068         uint32_t var;
3069 
3070         ASSERT(n_bytes >= 0);
3071         QL_PRINT(DBG_GLD, ("%s for ring(%d) to read max %d bytes\n",
3072             __func__, rx_ring->cq_id, n_bytes));
3073 
3074         ++qlge->rx_polls[rx_ring->cq_id];
3075 
3076         if (n_bytes == 0)
3077                 return (mp);
3078         mutex_enter(&rx_ring->rx_lock);
3079         mp = ql_ring_rx(rx_ring, n_bytes);
3080         mutex_exit(&rx_ring->rx_lock);
3081 
3082         if ((rx_ring->cq_id == 0) && (mp == NULL)) {
3083                 var = ql_read_reg(qlge, REG_STATUS);
3084                 /*
3085                  * Check for fatal error.
3086                  */
3087                 if ((var & STATUS_FE) != 0) {
3088                         ql_write_reg(qlge, REG_RSVD7, 0xfeed0003);
3089                         var = ql_read_reg(qlge, REG_ERROR_STATUS);
3090                         cmn_err(CE_WARN, "Got fatal error %x.", var);
3091                         ql_wake_asic_reset_soft_intr(qlge);
3092                         if (qlge->fm_enable) {
3093                                 atomic_or_32(&qlge->flags, ADAPTER_ERROR);
3094                                 ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
3095                                 ddi_fm_service_impact(qlge->dip,
3096                                     DDI_SERVICE_LOST);
3097                         }
3098                 }
3099                 /*
3100                  * Check MPI processor activity.
3101                  */
3102                 if ((var & STATUS_PI) != 0) {
3103                         /*
3104                          * We've got an async event or mailbox completion.
3105                          * Handle it and clear the source of the interrupt.
3106                          */
3107                         ql_write_reg(qlge, REG_RSVD7, 0xfeed0004);
3108                         ql_do_mpi_intr(qlge);
3109                 }
3110         }
3111 
3112         return (mp);
3113 }
3114 
3115 /*
3116  * MSI-X Multiple Vector Interrupt Handler for inbound (RX) completions.
3117  */
3118 /* ARGSUSED */
3119 static uint_t
3120 ql_msix_rx_isr(caddr_t arg1, caddr_t arg2)
3121 {
3122         struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
3123         qlge_t *qlge = rx_ring->qlge;
3124         mblk_t *mp;
3125         _NOTE(ARGUNUSED(arg2));
3126 
3127         QL_PRINT(DBG_INTR, ("%s for ring %d\n", __func__, rx_ring->cq_id));
3128 
3129         ++qlge->rx_interrupts[rx_ring->cq_id];
3130 
3131         mutex_enter(&rx_ring->rx_lock);
3132         mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
3133         mutex_exit(&rx_ring->rx_lock);
3134 
3135         if (mp != NULL)
3136                 RX_UPSTREAM(rx_ring, mp);
3137 
3138         return (DDI_INTR_CLAIMED);
3139 }
3140 
3141 
3142 /*
3143  *
3144  * Allocate DMA Buffer for ioctl service
3145  *
3146  */
3147 static int
3148 ql_alloc_ioctl_dma_buf(qlge_t *qlge)
3149 {
3150         uint64_t phy_addr;
3151         uint64_t alloc_size;
3152         ddi_dma_cookie_t dma_cookie;
3153 
3154         alloc_size = qlge->ioctl_buf_dma_attr.mem_len =
3155             max(WCS_MPI_CODE_RAM_LENGTH, MEMC_MPI_RAM_LENGTH);
3156         if (ql_alloc_phys(qlge->dip, &qlge->ioctl_buf_dma_attr.dma_handle,
3157             &ql_buf_acc_attr,
3158             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3159             &qlge->ioctl_buf_dma_attr.acc_handle,
3160             (size_t)alloc_size,  /* mem size */
3161             (size_t)0,  /* alignment */
3162             (caddr_t *)&qlge->ioctl_buf_dma_attr.vaddr,
3163             &dma_cookie) != 0) {
3164                 cmn_err(CE_WARN, "%s(%d): ioctl DMA allocation failed.",
3165                     __func__, qlge->instance);
3166                 return (DDI_FAILURE);
3167         }
3168 
3169         phy_addr = dma_cookie.dmac_laddress;
3170 
3171         if (qlge->ioctl_buf_dma_attr.vaddr == NULL) {
3172                 cmn_err(CE_WARN, "%s(%d): failed.", __func__, qlge->instance);
3173                 return (DDI_FAILURE);
3174         }
3175 
3176         qlge->ioctl_buf_dma_attr.dma_addr = phy_addr;
3177 
3178         QL_PRINT(DBG_MBX, ("%s: ioctl_dma_buf_virt_addr = 0x%lx, "
3179             "phy_addr = 0x%lx\n",
3180             __func__, qlge->ioctl_buf_dma_attr.vaddr, phy_addr));
3181 
3182         return (DDI_SUCCESS);
3183 }
3184 
3185 
3186 /*
3187  * Function to free physical memory.
3188  */
3189 static void
3190 ql_free_phys(ddi_dma_handle_t *dma_handle, ddi_acc_handle_t *acc_handle)
3191 {
3192         if (*dma_handle != NULL) {
3193                 (void) ddi_dma_unbind_handle(*dma_handle);
3194                 if (*acc_handle != NULL)
3195                         ddi_dma_mem_free(acc_handle);
3196                 ddi_dma_free_handle(dma_handle);
3197                 *acc_handle = NULL;
3198                 *dma_handle = NULL;
3199         }
3200 }
3201 
3202 /*
3203  * Function to free ioctl dma buffer.
3204  */
3205 static void
3206 ql_free_ioctl_dma_buf(qlge_t *qlge)
3207 {
3208         if (qlge->ioctl_buf_dma_attr.dma_handle != NULL) {
3209                 ql_free_phys(&qlge->ioctl_buf_dma_attr.dma_handle,
3210                     &qlge->ioctl_buf_dma_attr.acc_handle);
3211 
3212                 qlge->ioctl_buf_dma_attr.vaddr = NULL;
3213                 qlge->ioctl_buf_dma_attr.dma_handle = NULL;
3214         }
3215 }
3216 
3217 /*
3218  * Free shadow register space used for request and completion queues
3219  */
3220 static void
3221 ql_free_shadow_space(qlge_t *qlge)
3222 {
3223         if (qlge->host_copy_shadow_dma_attr.dma_handle != NULL) {
3224                 ql_free_phys(&qlge->host_copy_shadow_dma_attr.dma_handle,
3225                     &qlge->host_copy_shadow_dma_attr.acc_handle);
3226                 bzero(&qlge->host_copy_shadow_dma_attr,
3227                     sizeof (qlge->host_copy_shadow_dma_attr));
3228         }
3229 
3230         if (qlge->buf_q_ptr_base_addr_dma_attr.dma_handle != NULL) {
3231                 ql_free_phys(&qlge->buf_q_ptr_base_addr_dma_attr.dma_handle,
3232                     &qlge->buf_q_ptr_base_addr_dma_attr.acc_handle);
3233                 bzero(&qlge->buf_q_ptr_base_addr_dma_attr,
3234                     sizeof (qlge->buf_q_ptr_base_addr_dma_attr));
3235         }
3236 }
3237 
3238 /*
3239  * Allocate shadow register space for request and completion queues
3240  */
3241 static int
3242 ql_alloc_shadow_space(qlge_t *qlge)
3243 {
3244         ddi_dma_cookie_t dma_cookie;
3245 
3246         if (ql_alloc_phys(qlge->dip,
3247             &qlge->host_copy_shadow_dma_attr.dma_handle,
3248             &ql_dev_acc_attr,
3249             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3250             &qlge->host_copy_shadow_dma_attr.acc_handle,
3251             (size_t)VM_PAGE_SIZE,  /* mem size */
3252             (size_t)4, /* 4 bytes alignment */
3253             (caddr_t *)&qlge->host_copy_shadow_dma_attr.vaddr,
3254             &dma_cookie) != 0) {
3255                 bzero(&qlge->host_copy_shadow_dma_attr,
3256                     sizeof (qlge->host_copy_shadow_dma_attr));
3257 
3258                 cmn_err(CE_WARN, "%s(%d): Unable to allocate DMA memory for "
3259                     "response shadow registers", __func__, qlge->instance);
3260                 return (DDI_FAILURE);
3261         }
3262 
3263         qlge->host_copy_shadow_dma_attr.dma_addr = dma_cookie.dmac_laddress;
3264 
3265         if (ql_alloc_phys(qlge->dip,
3266             &qlge->buf_q_ptr_base_addr_dma_attr.dma_handle,
3267             &ql_desc_acc_attr,
3268             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3269             &qlge->buf_q_ptr_base_addr_dma_attr.acc_handle,
3270             (size_t)VM_PAGE_SIZE,  /* mem size */
3271             (size_t)4, /* 4 bytes alignment */
3272             (caddr_t *)&qlge->buf_q_ptr_base_addr_dma_attr.vaddr,
3273             &dma_cookie) != 0) {
3274                 bzero(&qlge->buf_q_ptr_base_addr_dma_attr,
3275                     sizeof (qlge->buf_q_ptr_base_addr_dma_attr));
3276 
3277                 cmn_err(CE_WARN, "%s(%d): Unable to allocate DMA memory "
3278                     "for request shadow registers",
3279                     __func__, qlge->instance);
3280                 goto err_wqp_sh_area;
3281         }
3282         qlge->buf_q_ptr_base_addr_dma_attr.dma_addr = dma_cookie.dmac_laddress;
3283 
3284         return (DDI_SUCCESS);
3285 
3286 err_wqp_sh_area:
3287         ql_free_phys(&qlge->host_copy_shadow_dma_attr.dma_handle,
3288             &qlge->host_copy_shadow_dma_attr.acc_handle);
3289         bzero(&qlge->host_copy_shadow_dma_attr,
3290             sizeof (qlge->host_copy_shadow_dma_attr));
3291 
3292         return (DDI_FAILURE);
3293 }
3294 
3295 /*
3296  * Initialize a tx ring
3297  */
3298 static void
3299 ql_init_tx_ring(struct tx_ring *tx_ring)
3300 {
3301         int i;
3302         struct ob_mac_iocb_req *mac_iocb_ptr = tx_ring->wq_dma.vaddr;
3303         struct tx_ring_desc *tx_ring_desc = tx_ring->wq_desc;
3304 
3305         for (i = 0; i < tx_ring->wq_len; i++) {
3306                 tx_ring_desc->index = i;
3307                 tx_ring_desc->queue_entry = mac_iocb_ptr;
3308                 mac_iocb_ptr++;
3309                 tx_ring_desc++;
3310         }
3311         tx_ring->tx_free_count = tx_ring->wq_len;
3312         tx_ring->queue_stopped = 0;
3313 }
3314 
3315 /*
3316  * Free one tx ring resources
3317  */
3318 static void
3319 ql_free_tx_resources(struct tx_ring *tx_ring)
3320 {
3321         struct tx_ring_desc *tx_ring_desc;
3322         int i, j;
3323 
3324         if (tx_ring->wq_dma.dma_handle != NULL) {
3325                 ql_free_phys(&tx_ring->wq_dma.dma_handle,
3326                     &tx_ring->wq_dma.acc_handle);
3327                 bzero(&tx_ring->wq_dma, sizeof (tx_ring->wq_dma));
3328         }
3329         if (tx_ring->wq_desc != NULL) {
3330                 tx_ring_desc = tx_ring->wq_desc;
3331                 for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) {
3332                         for (j = 0; j < QL_MAX_TX_DMA_HANDLES; j++) {
3333                                 if (tx_ring_desc->tx_dma_handle[j]) {
3334                                         /*
3335                                          * The unbinding will happen in tx
3336                                          * completion, here we just free the
3337                                          * handles
3338                                          */
3339                                         ddi_dma_free_handle(
3340                                             &(tx_ring_desc->tx_dma_handle[j]));
3341                                         tx_ring_desc->tx_dma_handle[j] = NULL;
3342                                 }
3343                         }
3344                         if (tx_ring_desc->oal != NULL) {
3345                                 tx_ring_desc->oal_dma_addr = 0;
3346                                 tx_ring_desc->oal = NULL;
3347                                 tx_ring_desc->copy_buffer = NULL;
3348                                 tx_ring_desc->copy_buffer_dma_addr = 0;
3349 
3350                                 ql_free_phys(&tx_ring_desc->oal_dma.dma_handle,
3351                                     &tx_ring_desc->oal_dma.acc_handle);
3352                         }
3353                 }
3354                 kmem_free(tx_ring->wq_desc,
3355                     tx_ring->wq_len * sizeof (struct tx_ring_desc));
3356                 tx_ring->wq_desc = NULL;
3357         }
3358         /* free the wqicb struct */
3359         if (tx_ring->wqicb_dma.dma_handle) {
3360                 ql_free_phys(&tx_ring->wqicb_dma.dma_handle,
3361                     &tx_ring->wqicb_dma.acc_handle);
3362                 bzero(&tx_ring->wqicb_dma, sizeof (tx_ring->wqicb_dma));
3363         }
3364 }
3365 
3366 /*
3367  * Allocate work (request) queue memory and transmit
3368  * descriptors for this transmit ring
3369  */
3370 static int
3371 ql_alloc_tx_resources(qlge_t *qlge, struct tx_ring *tx_ring)
3372 {
3373         ddi_dma_cookie_t dma_cookie;
3374         struct tx_ring_desc *tx_ring_desc;
3375         int i, j;
3376         uint32_t length;
3377 
3378         /* allocate dma buffers for obiocbs */
3379         if (ql_alloc_phys(qlge->dip, &tx_ring->wq_dma.dma_handle,
3380             &ql_desc_acc_attr,
3381             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3382             &tx_ring->wq_dma.acc_handle,
3383             (size_t)tx_ring->wq_size,        /* mem size */
3384             (size_t)128, /* alignment:128 bytes boundary */
3385             (caddr_t *)&tx_ring->wq_dma.vaddr,
3386             &dma_cookie) != 0) {
3387                 bzero(&tx_ring->wq_dma, sizeof (&tx_ring->wq_dma));
3388                 cmn_err(CE_WARN, "%s(%d): reqQ allocation failed.",
3389                     __func__, qlge->instance);
3390                 return (DDI_FAILURE);
3391         }
3392         tx_ring->wq_dma.dma_addr = dma_cookie.dmac_laddress;
3393 
3394         tx_ring->wq_desc =
3395             kmem_zalloc(tx_ring->wq_len * sizeof (struct tx_ring_desc),
3396             KM_NOSLEEP);
3397         if (tx_ring->wq_desc == NULL) {
3398                 goto err;
3399         } else {
3400                 tx_ring_desc = tx_ring->wq_desc;
3401                 /*
3402                  * Allocate a large enough structure to hold the following
3403                  * 1. oal buffer MAX_SGELEMENTS * sizeof (oal_entry) bytes
3404                  * 2. copy buffer of QL_MAX_COPY_LENGTH bytes
3405                  */
3406                 length = (sizeof (struct oal_entry) * MAX_SG_ELEMENTS)
3407                     + QL_MAX_COPY_LENGTH;
3408                 for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) {
3409 
3410                         if (ql_alloc_phys(qlge->dip,
3411                             &tx_ring_desc->oal_dma.dma_handle,
3412                             &ql_desc_acc_attr,
3413                             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3414                             &tx_ring_desc->oal_dma.acc_handle,
3415                             (size_t)length,     /* mem size */
3416                             (size_t)0, /* default alignment:8 bytes boundary */
3417                             (caddr_t *)&tx_ring_desc->oal_dma.vaddr,
3418                             &dma_cookie) != 0) {
3419                                 bzero(&tx_ring_desc->oal_dma,
3420                                     sizeof (tx_ring_desc->oal_dma));
3421                                 cmn_err(CE_WARN, "%s(%d): reqQ tx buf &"
3422                                     "oal alloc failed.",
3423                                     __func__, qlge->instance);
3424                                 goto err;
3425                         }
3426 
3427                         tx_ring_desc->oal = tx_ring_desc->oal_dma.vaddr;
3428                         tx_ring_desc->oal_dma_addr = dma_cookie.dmac_laddress;
3429                         tx_ring_desc->copy_buffer =
3430                             (caddr_t)((uint8_t *)tx_ring_desc->oal
3431                             + (sizeof (struct oal_entry) * MAX_SG_ELEMENTS));
3432                         tx_ring_desc->copy_buffer_dma_addr =
3433                             (tx_ring_desc->oal_dma_addr
3434                             + (sizeof (struct oal_entry) * MAX_SG_ELEMENTS));
3435 
3436                         /* Allocate dma handles for transmit buffers */
3437                         for (j = 0; j < QL_MAX_TX_DMA_HANDLES; j++) {
3438                                 if (ddi_dma_alloc_handle(qlge->dip,
3439                                     &tx_mapping_dma_attr,
3440                                     DDI_DMA_DONTWAIT,
3441                                     0, &tx_ring_desc->tx_dma_handle[j])
3442                                     != DDI_SUCCESS) {
3443                                         tx_ring_desc->tx_dma_handle[j] = NULL;
3444                                         cmn_err(CE_WARN,
3445                                             "!%s: ddi_dma_alloc_handle: "
3446                                             "tx_dma_handle "
3447                                             "alloc failed", __func__);
3448                                         ql_free_phys(
3449                                             &tx_ring_desc->oal_dma.dma_handle,
3450                                             &tx_ring_desc->oal_dma.acc_handle);
3451                                         goto err;
3452                                 }
3453                         }
3454                 }
3455         }
3456         /* alloc a wqicb control block to load this tx ring to hw */
3457         if (ql_alloc_phys(qlge->dip, &tx_ring->wqicb_dma.dma_handle,
3458             &ql_desc_acc_attr,
3459             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3460             &tx_ring->wqicb_dma.acc_handle,
3461             (size_t)sizeof (struct wqicb_t),    /* mem size */
3462             (size_t)0, /* alignment:128 bytes boundary */
3463             (caddr_t *)&tx_ring->wqicb_dma.vaddr,
3464             &dma_cookie) != 0) {
3465                 bzero(&tx_ring->wqicb_dma, sizeof (tx_ring->wqicb_dma));
3466                 cmn_err(CE_WARN, "%s(%d): wqicb allocation failed.",
3467                     __func__, qlge->instance);
3468                 goto err;
3469         }
3470         tx_ring->wqicb_dma.dma_addr = dma_cookie.dmac_laddress;
3471 
3472         return (DDI_SUCCESS);
3473 
3474 err:
3475         ql_free_tx_resources(tx_ring);
3476         return (DDI_FAILURE);
3477 }
3478 
3479 /*
3480  * Free one rx ring resources
3481  */
3482 static void
3483 ql_free_rx_resources(struct rx_ring *rx_ring)
3484 {
3485         /* Free the small buffer queue. */
3486         if (rx_ring->sbq_dma.dma_handle) {
3487                 ql_free_phys(&rx_ring->sbq_dma.dma_handle,
3488                     &rx_ring->sbq_dma.acc_handle);
3489                 bzero(&rx_ring->sbq_dma, sizeof (rx_ring->sbq_dma));
3490         }
3491 
3492         /* Free the small buffer queue control blocks. */
3493         if (rx_ring->sbq_desc != NULL) {
3494                 kmem_free(rx_ring->sbq_desc, rx_ring->sbq_len *
3495                     sizeof (struct bq_desc));
3496                 rx_ring->sbq_desc = NULL;
3497         }
3498 
3499         /* Free the large buffer queue. */
3500         if (rx_ring->lbq_dma.dma_handle) {
3501                 ql_free_phys(&rx_ring->lbq_dma.dma_handle,
3502                     &rx_ring->lbq_dma.acc_handle);
3503                 bzero(&rx_ring->lbq_dma, sizeof (rx_ring->lbq_dma));
3504         }
3505 
3506         /* Free the large buffer queue control blocks. */
3507         if (rx_ring->lbq_desc != NULL) {
3508                 kmem_free(rx_ring->lbq_desc, rx_ring->lbq_len *
3509                     sizeof (struct bq_desc));
3510                 rx_ring->lbq_desc = NULL;
3511         }
3512 
3513         /* Free cqicb struct */
3514         if (rx_ring->cqicb_dma.dma_handle) {
3515                 ql_free_phys(&rx_ring->cqicb_dma.dma_handle,
3516                     &rx_ring->cqicb_dma.acc_handle);
3517                 bzero(&rx_ring->cqicb_dma, sizeof (rx_ring->cqicb_dma));
3518         }
3519         /* Free the rx queue. */
3520         if (rx_ring->cq_dma.dma_handle) {
3521                 ql_free_phys(&rx_ring->cq_dma.dma_handle,
3522                     &rx_ring->cq_dma.acc_handle);
3523                 bzero(&rx_ring->cq_dma, sizeof (rx_ring->cq_dma));
3524         }
3525 }
3526 
3527 /*
3528  * Allocate queues and buffers for this completions queue based
3529  * on the values in the parameter structure.
3530  */
3531 static int
3532 ql_alloc_rx_resources(qlge_t *qlge, struct rx_ring *rx_ring)
3533 {
3534         ddi_dma_cookie_t dma_cookie;
3535 
3536         if (ql_alloc_phys(qlge->dip, &rx_ring->cq_dma.dma_handle,
3537             &ql_desc_acc_attr,
3538             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3539             &rx_ring->cq_dma.acc_handle,
3540             (size_t)rx_ring->cq_size,  /* mem size */
3541             (size_t)128, /* alignment:128 bytes boundary */
3542             (caddr_t *)&rx_ring->cq_dma.vaddr,
3543             &dma_cookie) != 0)      {
3544                 bzero(&rx_ring->cq_dma, sizeof (rx_ring->cq_dma));
3545                 cmn_err(CE_WARN, "%s(%d): rspQ allocation failed.",
3546                     __func__, qlge->instance);
3547                 return (DDI_FAILURE);
3548         }
3549         rx_ring->cq_dma.dma_addr = dma_cookie.dmac_laddress;
3550 
3551         if (rx_ring->sbq_len != 0) {
3552                 /*
3553                  * Allocate small buffer queue.
3554                  */
3555                 if (ql_alloc_phys(qlge->dip, &rx_ring->sbq_dma.dma_handle,
3556                     &ql_desc_acc_attr,
3557                     DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3558                     &rx_ring->sbq_dma.acc_handle,
3559                     (size_t)rx_ring->sbq_size,  /* mem size */
3560                     (size_t)128, /* alignment:128 bytes boundary */
3561                     (caddr_t *)&rx_ring->sbq_dma.vaddr,
3562                     &dma_cookie) != 0) {
3563                         bzero(&rx_ring->sbq_dma, sizeof (rx_ring->sbq_dma));
3564                         cmn_err(CE_WARN,
3565                             "%s(%d): small buffer queue allocation failed.",
3566                             __func__, qlge->instance);
3567                         goto err_mem;
3568                 }
3569                 rx_ring->sbq_dma.dma_addr = dma_cookie.dmac_laddress;
3570 
3571                 /*
3572                  * Allocate small buffer queue control blocks.
3573                  */
3574                 rx_ring->sbq_desc =
3575                     kmem_zalloc(rx_ring->sbq_len * sizeof (struct bq_desc),
3576                     KM_NOSLEEP);
3577                 if (rx_ring->sbq_desc == NULL) {
3578                         cmn_err(CE_WARN,
3579                             "sbq control block allocation failed.");
3580                         goto err_mem;
3581                 }
3582 
3583                 ql_init_sbq_ring(rx_ring);
3584         }
3585 
3586         if (rx_ring->lbq_len != 0) {
3587                 /*
3588                  * Allocate large buffer queue.
3589                  */
3590                 if (ql_alloc_phys(qlge->dip, &rx_ring->lbq_dma.dma_handle,
3591                     &ql_desc_acc_attr,
3592                     DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3593                     &rx_ring->lbq_dma.acc_handle,
3594                     (size_t)rx_ring->lbq_size,  /* mem size */
3595                     (size_t)128, /* alignment:128 bytes boundary */
3596                     (caddr_t *)&rx_ring->lbq_dma.vaddr,
3597                     &dma_cookie) != 0) {
3598                         bzero(&rx_ring->lbq_dma, sizeof (rx_ring->lbq_dma));
3599                         cmn_err(CE_WARN, "%s(%d): lbq allocation failed.",
3600                             __func__, qlge->instance);
3601                         goto err_mem;
3602                 }
3603                 rx_ring->lbq_dma.dma_addr = dma_cookie.dmac_laddress;
3604 
3605                 /*
3606                  * Allocate large buffer queue control blocks.
3607                  */
3608                 rx_ring->lbq_desc =
3609                     kmem_zalloc(rx_ring->lbq_len * sizeof (struct bq_desc),
3610                     KM_NOSLEEP);
3611                 if (rx_ring->lbq_desc == NULL) {
3612                         cmn_err(CE_WARN,
3613                             "Large buffer queue control block allocation "
3614                             "failed.");
3615                         goto err_mem;
3616                 }
3617                 ql_init_lbq_ring(rx_ring);
3618         }
3619 
3620         if (ql_alloc_phys(qlge->dip, &rx_ring->cqicb_dma.dma_handle,
3621             &ql_desc_acc_attr,
3622             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3623             &rx_ring->cqicb_dma.acc_handle,
3624             (size_t)sizeof (struct cqicb_t),  /* mem size */
3625             (size_t)0, /* alignment:128 bytes boundary */
3626             (caddr_t *)&rx_ring->cqicb_dma.vaddr,
3627             &dma_cookie) != 0) {
3628                 bzero(&rx_ring->cqicb_dma, sizeof (rx_ring->cqicb_dma));
3629                 cmn_err(CE_WARN, "%s(%d): cqicb allocation failed.",
3630                     __func__, qlge->instance);
3631                 goto err_mem;
3632         }
3633         rx_ring->cqicb_dma.dma_addr = dma_cookie.dmac_laddress;
3634 
3635         return (DDI_SUCCESS);
3636 
3637 err_mem:
3638         ql_free_rx_resources(rx_ring);
3639         return (DDI_FAILURE);
3640 }
3641 
3642 /*
3643  * Frees tx/rx queues memory resources
3644  */
3645 static void
3646 ql_free_mem_resources(qlge_t *qlge)
3647 {
3648         int i;
3649 
3650         if (qlge->ricb_dma.dma_handle) {
3651                 /* free the ricb struct */
3652                 ql_free_phys(&qlge->ricb_dma.dma_handle,
3653                     &qlge->ricb_dma.acc_handle);
3654                 bzero(&qlge->ricb_dma, sizeof (qlge->ricb_dma));
3655         }
3656 
3657         ql_free_rx_buffers(qlge);
3658 
3659         ql_free_ioctl_dma_buf(qlge);
3660 
3661         for (i = 0; i < qlge->tx_ring_count; i++)
3662                 ql_free_tx_resources(&qlge->tx_ring[i]);
3663 
3664         for (i = 0; i < qlge->rx_ring_count; i++)
3665                 ql_free_rx_resources(&qlge->rx_ring[i]);
3666 
3667         ql_free_shadow_space(qlge);
3668 }
3669 
3670 /*
3671  * Allocate buffer queues, large buffers and small buffers etc
3672  *
3673  * This API is called in the gld_attach member function. It is called
3674  * only once.  Later reset,reboot should not re-allocate all rings and
3675  * buffers.
3676  */
3677 static int
3678 ql_alloc_mem_resources(qlge_t *qlge)
3679 {
3680         int i;
3681         ddi_dma_cookie_t dma_cookie;
3682 
3683         /* Allocate space for our shadow registers */
3684         if (ql_alloc_shadow_space(qlge))
3685                 return (DDI_FAILURE);
3686 
3687         for (i = 0; i < qlge->rx_ring_count; i++) {
3688                 if (ql_alloc_rx_resources(qlge, &qlge->rx_ring[i]) != 0) {
3689                         cmn_err(CE_WARN, "RX resource allocation failed.");
3690                         goto err_mem;
3691                 }
3692         }
3693         /* Allocate tx queue resources */
3694         for (i = 0; i < qlge->tx_ring_count; i++) {
3695                 if (ql_alloc_tx_resources(qlge, &qlge->tx_ring[i]) != 0) {
3696                         cmn_err(CE_WARN, "Tx resource allocation failed.");
3697                         goto err_mem;
3698                 }
3699         }
3700 
3701         if (ql_alloc_ioctl_dma_buf(qlge) != DDI_SUCCESS) {
3702                 goto err_mem;
3703         }
3704 
3705         if (ql_alloc_rx_buffers(qlge) != DDI_SUCCESS) {
3706                 cmn_err(CE_WARN, "?%s(%d): ql_alloc_rx_buffers failed",
3707                     __func__, qlge->instance);
3708                 goto err_mem;
3709         }
3710 
3711         qlge->sequence |= INIT_ALLOC_RX_BUF;
3712 
3713         if (ql_alloc_phys(qlge->dip, &qlge->ricb_dma.dma_handle,
3714             &ql_desc_acc_attr,
3715             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3716             &qlge->ricb_dma.acc_handle,
3717             (size_t)sizeof (struct ricb),  /* mem size */
3718             (size_t)0, /* alignment:128 bytes boundary */
3719             (caddr_t *)&qlge->ricb_dma.vaddr,
3720             &dma_cookie) != 0) {
3721                 bzero(&qlge->ricb_dma, sizeof (qlge->ricb_dma));
3722                 cmn_err(CE_WARN, "%s(%d): ricb allocation failed.",
3723                     __func__, qlge->instance);
3724                 goto err_mem;
3725         }
3726         qlge->ricb_dma.dma_addr = dma_cookie.dmac_laddress;
3727 
3728         return (DDI_SUCCESS);
3729 
3730 err_mem:
3731         ql_free_mem_resources(qlge);
3732         return (DDI_FAILURE);
3733 }
3734 
3735 
3736 /*
3737  * Function used to allocate physical memory and zero it.
3738  */
3739 
3740 static int
3741 ql_alloc_phys_rbuf(dev_info_t *dip, ddi_dma_handle_t *dma_handle,
3742     ddi_device_acc_attr_t *device_acc_attr,
3743     uint_t dma_flags,
3744     ddi_acc_handle_t *acc_handle,
3745     size_t size,
3746     size_t alignment,
3747     caddr_t *vaddr,
3748     ddi_dma_cookie_t *dma_cookie)
3749 {
3750         size_t rlen;
3751         uint_t cnt;
3752 
3753         /*
3754          * Workaround for SUN XMITS buffer must end and start on 8 byte
3755          * boundary. Else, hardware will overrun the buffer. Simple fix is
3756          * to make sure buffer has enough room for overrun.
3757          */
3758         if (size & 7) {
3759                 size += 8 - (size & 7);
3760         }
3761 
3762         /* Adjust the alignment if requested */
3763         if (alignment) {
3764                 dma_attr.dma_attr_align = alignment;
3765         }
3766 
3767         /*
3768          * Allocate DMA handle
3769          */
3770         if (ddi_dma_alloc_handle(dip, &dma_attr_rbuf, DDI_DMA_DONTWAIT, NULL,
3771             dma_handle) != DDI_SUCCESS) {
3772                 cmn_err(CE_WARN, QL_BANG "%s:  ddi_dma_alloc_handle FAILED",
3773                     __func__);
3774                 *dma_handle = NULL;
3775                 return (QL_ERROR);
3776         }
3777         /*
3778          * Allocate DMA memory
3779          */
3780         if (ddi_dma_mem_alloc(*dma_handle, size, device_acc_attr,
3781             dma_flags & (DDI_DMA_CONSISTENT|DDI_DMA_STREAMING),
3782             DDI_DMA_DONTWAIT,
3783             NULL, vaddr, &rlen, acc_handle) != DDI_SUCCESS) {
3784                 cmn_err(CE_WARN, "alloc_phys: DMA Memory alloc Failed");
3785                 ddi_dma_free_handle(dma_handle);
3786                 *acc_handle = NULL;
3787                 *dma_handle = NULL;
3788                 return (QL_ERROR);
3789         }
3790 
3791         if (ddi_dma_addr_bind_handle(*dma_handle, NULL, *vaddr, rlen,
3792             dma_flags, DDI_DMA_DONTWAIT, NULL,
3793             dma_cookie, &cnt) != DDI_DMA_MAPPED) {
3794                 ddi_dma_mem_free(acc_handle);
3795 
3796                 ddi_dma_free_handle(dma_handle);
3797                 cmn_err(CE_WARN, "%s ddi_dma_addr_bind_handle FAILED",
3798                     __func__);
3799                 *acc_handle = NULL;
3800                 *dma_handle = NULL;
3801                 return (QL_ERROR);
3802         }
3803 
3804         if (cnt != 1) {
3805 
3806                 ql_free_phys(dma_handle, acc_handle);
3807 
3808                 cmn_err(CE_WARN, "%s: cnt != 1; Failed segment count",
3809                     __func__);
3810                 return (QL_ERROR);
3811         }
3812 
3813         bzero((caddr_t)*vaddr, rlen);
3814 
3815         return (0);
3816 }
3817 
3818 /*
3819  * Function used to allocate physical memory and zero it.
3820  */
3821 static int
3822 ql_alloc_phys(dev_info_t *dip, ddi_dma_handle_t *dma_handle,
3823     ddi_device_acc_attr_t *device_acc_attr,
3824     uint_t dma_flags,
3825     ddi_acc_handle_t *acc_handle,
3826     size_t size,
3827     size_t alignment,
3828     caddr_t *vaddr,
3829     ddi_dma_cookie_t *dma_cookie)
3830 {
3831         size_t rlen;
3832         uint_t cnt;
3833 
3834         /*
3835          * Workaround for SUN XMITS buffer must end and start on 8 byte
3836          * boundary. Else, hardware will overrun the buffer. Simple fix is
3837          * to make sure buffer has enough room for overrun.
3838          */
3839         if (size & 7) {
3840                 size += 8 - (size & 7);
3841         }
3842 
3843         /* Adjust the alignment if requested */
3844         if (alignment) {
3845                 dma_attr.dma_attr_align = alignment;
3846         }
3847 
3848         /*
3849          * Allocate DMA handle
3850          */
3851         if (ddi_dma_alloc_handle(dip, &dma_attr, DDI_DMA_DONTWAIT, NULL,
3852             dma_handle) != DDI_SUCCESS) {
3853                 cmn_err(CE_WARN, QL_BANG "%s:  ddi_dma_alloc_handle FAILED",
3854                     __func__);
3855                 *dma_handle = NULL;
3856                 return (QL_ERROR);
3857         }
3858         /*
3859          * Allocate DMA memory
3860          */
3861         if (ddi_dma_mem_alloc(*dma_handle, size, device_acc_attr,
3862             dma_flags & (DDI_DMA_CONSISTENT|DDI_DMA_STREAMING),
3863             DDI_DMA_DONTWAIT,
3864             NULL, vaddr, &rlen, acc_handle) != DDI_SUCCESS) {
3865                 cmn_err(CE_WARN, "alloc_phys: DMA Memory alloc Failed");
3866                 ddi_dma_free_handle(dma_handle);
3867                 *acc_handle = NULL;
3868                 *dma_handle = NULL;
3869                 return (QL_ERROR);
3870         }
3871 
3872         if (ddi_dma_addr_bind_handle(*dma_handle, NULL, *vaddr, rlen,
3873             dma_flags, DDI_DMA_DONTWAIT, NULL,
3874             dma_cookie, &cnt) != DDI_DMA_MAPPED) {
3875                 ddi_dma_mem_free(acc_handle);
3876                 ddi_dma_free_handle(dma_handle);
3877                 cmn_err(CE_WARN, "%s ddi_dma_addr_bind_handle FAILED",
3878                     __func__);
3879                 *acc_handle = NULL;
3880                 *dma_handle = NULL;
3881                 return (QL_ERROR);
3882         }
3883 
3884         if (cnt != 1) {
3885 
3886                 ql_free_phys(dma_handle, acc_handle);
3887 
3888                 cmn_err(CE_WARN, "%s: cnt != 1; Failed segment count",
3889                     __func__);
3890                 return (QL_ERROR);
3891         }
3892 
3893         bzero((caddr_t)*vaddr, rlen);
3894 
3895         return (0);
3896 }
3897 
3898 /*
3899  * Add interrupt handlers based on the interrupt type.
3900  * Before adding the interrupt handlers, the interrupt vectors should
3901  * have been allocated, and the rx/tx rings have also been allocated.
3902  */
3903 static int
3904 ql_add_intr_handlers(qlge_t *qlge)
3905 {
3906         int vector = 0;
3907         int rc, i;
3908         uint32_t value;
3909         struct intr_ctx *intr_ctx = &qlge->intr_ctx[0];
3910 
3911         switch (qlge->intr_type) {
3912         case DDI_INTR_TYPE_MSIX:
3913                 /*
3914                  * Add interrupt handler for rx and tx rings: vector[0 -
3915                  * (qlge->intr_cnt -1)].
3916                  */
3917                 value = 0;
3918                 for (vector = 0; vector < qlge->intr_cnt; vector++) {
3919                         ql_atomic_set_32(&intr_ctx->irq_cnt, value);
3920 
3921                         /*
3922                          * associate interrupt vector with interrupt handler
3923                          */
3924                         rc = ddi_intr_add_handler(qlge->htable[vector],
3925                             (ddi_intr_handler_t *)intr_ctx->handler,
3926                             (void *)&qlge->rx_ring[vector], NULL);
3927 
3928                         QL_PRINT(DBG_INIT, ("rx_ring[%d] 0x%p\n",
3929                             vector, &qlge->rx_ring[vector]));
3930                         if (rc != DDI_SUCCESS) {
3931                                 QL_PRINT(DBG_INIT,
3932                                     ("Add rx interrupt handler failed. "
3933                                     "return: %d, vector: %d", rc, vector));
3934                                 for (vector--; vector >= 0; vector--) {
3935                                         (void) ddi_intr_remove_handler(
3936                                             qlge->htable[vector]);
3937                                 }
3938                                 return (DDI_FAILURE);
3939                         }
3940                         intr_ctx++;
3941                 }
3942                 break;
3943 
3944         case DDI_INTR_TYPE_MSI:
3945                 /*
3946                  * Add interrupt handlers for the only vector
3947                  */
3948                 ql_atomic_set_32(&intr_ctx->irq_cnt, value);
3949 
3950                 rc = ddi_intr_add_handler(qlge->htable[vector],
3951                     ql_isr,
3952                     (caddr_t)&qlge->rx_ring[0], NULL);
3953 
3954                 if (rc != DDI_SUCCESS) {
3955                         QL_PRINT(DBG_INIT,
3956                             ("Add MSI interrupt handler failed: %d\n", rc));
3957                         return (DDI_FAILURE);
3958                 }
3959                 break;
3960 
3961         case DDI_INTR_TYPE_FIXED:
3962                 /*
3963                  * Add interrupt handlers for the only vector
3964                  */
3965                 ql_atomic_set_32(&intr_ctx->irq_cnt, value);
3966 
3967                 rc = ddi_intr_add_handler(qlge->htable[vector],
3968                     ql_isr,
3969                     (caddr_t)&qlge->rx_ring[0], NULL);
3970 
3971                 if (rc != DDI_SUCCESS) {
3972                         QL_PRINT(DBG_INIT,
3973                             ("Add legacy interrupt handler failed: %d\n", rc));
3974                         return (DDI_FAILURE);
3975                 }
3976                 break;
3977 
3978         default:
3979                 return (DDI_FAILURE);
3980         }
3981 
3982         /* Enable interrupts */
3983         /* Block enable */
3984         if (qlge->intr_cap & DDI_INTR_FLAG_BLOCK) {
3985                 QL_PRINT(DBG_INIT, ("Block enabling %d interrupt(s)\n",
3986                     qlge->intr_cnt));
3987                 (void) ddi_intr_block_enable(qlge->htable, qlge->intr_cnt);
3988         } else { /* Non block enable */
3989                 for (i = 0; i < qlge->intr_cnt; i++) {
3990                         QL_PRINT(DBG_INIT, ("Non Block Enabling interrupt %d "
3991                             "handle 0x%x\n", i, qlge->htable[i]));
3992                         (void) ddi_intr_enable(qlge->htable[i]);
3993                 }
3994         }
3995         qlge->sequence |= INIT_INTR_ENABLED;
3996 
3997         return (DDI_SUCCESS);
3998 }
3999 
4000 /*
4001  * Here we build the intr_ctx structures based on
4002  * our rx_ring count and intr vector count.
4003  * The intr_ctx structure is used to hook each vector
4004  * to possibly different handlers.
4005  */
4006 static void
4007 ql_resolve_queues_to_irqs(qlge_t *qlge)
4008 {
4009         int i = 0;
4010         struct intr_ctx *intr_ctx = &qlge->intr_ctx[0];
4011 
4012         if (qlge->intr_type == DDI_INTR_TYPE_MSIX) {
4013                 /*
4014                  * Each rx_ring has its own intr_ctx since we
4015                  * have separate vectors for each queue.
4016                  * This only true when MSI-X is enabled.
4017                  */
4018                 for (i = 0; i < qlge->intr_cnt; i++, intr_ctx++) {
4019                         qlge->rx_ring[i].irq = i;
4020                         intr_ctx->intr = i;
4021                         intr_ctx->qlge = qlge;
4022 
4023                         /*
4024                          * We set up each vectors enable/disable/read bits so
4025                          * there's no bit/mask calculations in critical path.
4026                          */
4027                         intr_ctx->intr_en_mask =
4028                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4029                             INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK |
4030                             INTR_EN_IHD | i;
4031                         intr_ctx->intr_dis_mask =
4032                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4033                             INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK |
4034                             INTR_EN_IHD | i;
4035                         intr_ctx->intr_read_mask =
4036                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4037                             INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD
4038                             | i;
4039 
4040                         if (i == 0) {
4041                                 /*
4042                                  * Default queue handles bcast/mcast plus
4043                                  * async events.
4044                                  */
4045                                 intr_ctx->handler = ql_isr;
4046                         } else if (qlge->rx_ring[i].type == TX_Q) {
4047                                 /*
4048                                  * Outbound queue is for outbound completions
4049                                  * only.
4050                                  */
4051                                 if (qlge->isr_stride)
4052                                         intr_ctx->handler = ql_msix_isr;
4053                                 else
4054                                         intr_ctx->handler = ql_msix_tx_isr;
4055                         } else {
4056                                 /*
4057                                  * Inbound queues handle unicast frames only.
4058                                  */
4059                                 if (qlge->isr_stride)
4060                                         intr_ctx->handler = ql_msix_isr;
4061                                 else
4062                                         intr_ctx->handler = ql_msix_rx_isr;
4063                         }
4064                 }
4065                 i = qlge->intr_cnt;
4066                 for (; i < qlge->rx_ring_count; i++, intr_ctx++) {
4067                         int iv = i - qlge->isr_stride;
4068                         qlge->rx_ring[i].irq = iv;
4069                         intr_ctx->intr = iv;
4070                         intr_ctx->qlge = qlge;
4071 
4072                         /*
4073                          * We set up each vectors enable/disable/read bits so
4074                          * there's no bit/mask calculations in critical path.
4075                          */
4076                         intr_ctx->intr_en_mask =
4077                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4078                             INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK |
4079                             INTR_EN_IHD | iv;
4080                         intr_ctx->intr_dis_mask =
4081                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4082                             INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK |
4083                             INTR_EN_IHD | iv;
4084                         intr_ctx->intr_read_mask =
4085                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4086                             INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD
4087                             | iv;
4088 
4089                         if (qlge->rx_ring[i].type == TX_Q) {
4090                                 /*
4091                                  * Outbound queue is for outbound completions
4092                                  * only.
4093                                  */
4094                                 intr_ctx->handler = ql_msix_isr;
4095                         } else {
4096                                 /*
4097                                  * Inbound queues handle unicast frames only.
4098                                  */
4099                                 intr_ctx->handler = ql_msix_rx_isr;
4100                         }
4101                 }
4102         } else {
4103                 /*
4104                  * All rx_rings use the same intr_ctx since
4105                  * there is only one vector.
4106                  */
4107                 intr_ctx->intr = 0;
4108                 intr_ctx->qlge = qlge;
4109                 /*
4110                  * We set up each vectors enable/disable/read bits so
4111                  * there's no bit/mask calculations in the critical path.
4112                  */
4113                 intr_ctx->intr_en_mask =
4114                     INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4115                     INTR_EN_TYPE_ENABLE;
4116                 intr_ctx->intr_dis_mask =
4117                     INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4118                     INTR_EN_TYPE_DISABLE;
4119                 intr_ctx->intr_read_mask =
4120                     INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
4121                     INTR_EN_TYPE_READ;
4122                 /*
4123                  * Single interrupt means one handler for all rings.
4124                  */
4125                 intr_ctx->handler = ql_isr;
4126                 for (i = 0; i < qlge->rx_ring_count; i++)
4127                         qlge->rx_ring[i].irq = 0;
4128         }
4129 }
4130 
4131 
4132 /*
4133  * Free allocated interrupts.
4134  */
4135 static void
4136 ql_free_irq_vectors(qlge_t *qlge)
4137 {
4138         int i;
4139         int rc;
4140 
4141         if (qlge->sequence & INIT_INTR_ENABLED) {
4142                 /* Disable all interrupts */
4143                 if (qlge->intr_cap & DDI_INTR_FLAG_BLOCK) {
4144                         /* Call ddi_intr_block_disable() */
4145                         (void) ddi_intr_block_disable(qlge->htable,
4146                             qlge->intr_cnt);
4147                 } else {
4148                         for (i = 0; i < qlge->intr_cnt; i++) {
4149                                 (void) ddi_intr_disable(qlge->htable[i]);
4150                         }
4151                 }
4152 
4153                 qlge->sequence &= ~INIT_INTR_ENABLED;
4154         }
4155 
4156         for (i = 0; i < qlge->intr_cnt; i++) {
4157 
4158                 if (qlge->sequence & INIT_ADD_INTERRUPT)
4159                         (void) ddi_intr_remove_handler(qlge->htable[i]);
4160 
4161                 if (qlge->sequence & INIT_INTR_ALLOC) {
4162                         rc = ddi_intr_free(qlge->htable[i]);
4163                         if (rc != DDI_SUCCESS) {
4164                                 /* EMPTY */
4165                                 QL_PRINT(DBG_INIT, ("Free intr failed: %d",
4166                                     rc));
4167                         }
4168                 }
4169         }
4170         if (qlge->sequence & INIT_INTR_ALLOC)
4171                 qlge->sequence &= ~INIT_INTR_ALLOC;
4172 
4173         if (qlge->sequence & INIT_ADD_INTERRUPT)
4174                 qlge->sequence &= ~INIT_ADD_INTERRUPT;
4175 
4176         if (qlge->htable) {
4177                 kmem_free(qlge->htable, qlge->intr_size);
4178                 qlge->htable = NULL;
4179         }
4180 }
4181 
4182 /*
4183  * Allocate interrupt vectors
4184  * For legacy and MSI, only 1 handle is needed.
4185  * For MSI-X, if fewer than 2 vectors are available, return failure.
4186  * Upon success, this maps the vectors to rx and tx rings for
4187  * interrupts.
4188  */
4189 static int
4190 ql_request_irq_vectors(qlge_t *qlge, int intr_type)
4191 {
4192         dev_info_t *devinfo;
4193         uint32_t request, orig;
4194         int count, avail, actual;
4195         int minimum;
4196         int rc;
4197 
4198         devinfo = qlge->dip;
4199 
4200         switch (intr_type) {
4201         case DDI_INTR_TYPE_FIXED:
4202                 request = 1;    /* Request 1 legacy interrupt handle */
4203                 minimum = 1;
4204                 QL_PRINT(DBG_INIT, ("interrupt type: legacy\n"));
4205                 break;
4206 
4207         case DDI_INTR_TYPE_MSI:
4208                 request = 1;    /* Request 1 MSI interrupt handle */
4209                 minimum = 1;
4210                 QL_PRINT(DBG_INIT, ("interrupt type: MSI\n"));
4211                 break;
4212 
4213         case DDI_INTR_TYPE_MSIX:
4214                 /*
4215                  * Ideal number of vectors for the adapter is
4216                  * # rss rings + tx completion rings for default completion
4217                  * queue.
4218                  */
4219                 request = qlge->rx_ring_count;
4220 
4221                 orig = request;
4222                 if (request > (MAX_RX_RINGS))
4223                         request = MAX_RX_RINGS;
4224                 minimum = 2;
4225                 QL_PRINT(DBG_INIT, ("interrupt type: MSI-X\n"));
4226                 break;
4227 
4228         default:
4229                 QL_PRINT(DBG_INIT, ("Invalid parameter\n"));
4230                 return (DDI_FAILURE);
4231         }
4232 
4233         QL_PRINT(DBG_INIT, ("interrupt handles requested: %d  minimum: %d\n",
4234             request, minimum));
4235 
4236         /*
4237          * Get number of supported interrupts
4238          */
4239         rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4240         if ((rc != DDI_SUCCESS) || (count < minimum)) {
4241                 QL_PRINT(DBG_INIT, ("Get interrupt number failed. Return: %d, "
4242                     "count: %d\n", rc, count));
4243                 return (DDI_FAILURE);
4244         }
4245         QL_PRINT(DBG_INIT, ("interrupts supported: %d\n", count));
4246 
4247         /*
4248          * Get number of available interrupts
4249          */
4250         rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
4251         if ((rc != DDI_SUCCESS) || (avail < minimum)) {
4252                 QL_PRINT(DBG_INIT,
4253                     ("Get interrupt available number failed. Return:"
4254                     " %d, available: %d\n", rc, avail));
4255                 return (DDI_FAILURE);
4256         }
4257         QL_PRINT(DBG_INIT, ("interrupts available: %d\n", avail));
4258 
4259         if (avail < request) {
4260                 QL_PRINT(DBG_INIT, ("Request %d handles, %d available\n",
4261                     request, avail));
4262                 request = avail;
4263         }
4264 
4265         actual = 0;
4266         qlge->intr_cnt = 0;
4267 
4268         /*
4269          * Allocate an array of interrupt handles
4270          */
4271         qlge->intr_size = (size_t)(request * sizeof (ddi_intr_handle_t));
4272         qlge->htable = kmem_alloc(qlge->intr_size, KM_SLEEP);
4273 
4274         rc = ddi_intr_alloc(devinfo, qlge->htable, intr_type, 0,
4275             (int)request, &actual, DDI_INTR_ALLOC_NORMAL);
4276         if (rc != DDI_SUCCESS) {
4277                 cmn_err(CE_WARN, "%s(%d) Allocate interrupts failed. return:"
4278                     " %d, request: %d, actual: %d",
4279                     __func__, qlge->instance, rc, request, actual);
4280                 goto ql_intr_alloc_fail;
4281         }
4282         qlge->intr_cnt = actual;
4283 
4284         qlge->sequence |= INIT_INTR_ALLOC;
4285 
4286         /*
4287          * If the actual number of vectors is less than the minumum
4288          * then fail.
4289          */
4290         if (actual < minimum) {
4291                 cmn_err(CE_WARN,
4292                     "Insufficient interrupt handles available: %d", actual);
4293                 goto ql_intr_alloc_fail;
4294         }
4295 
4296         /*
4297          * For MSI-X, actual might force us to reduce number of tx & rx rings
4298          */
4299         if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
4300                 if (actual >= (orig / 2)) {
4301                         count = orig / 2;
4302                         qlge->rss_ring_count = count;
4303                         qlge->tx_ring_count = count;
4304                         qlge->isr_stride = count;
4305                 } else if (actual >= (orig / 4)) {
4306                         count = orig / 4;
4307                         qlge->rss_ring_count = count;
4308                         qlge->tx_ring_count = count;
4309                         qlge->isr_stride = count;
4310                 } else if (actual >= (orig / 8)) {
4311                         count = orig / 8;
4312                         qlge->rss_ring_count = count;
4313                         qlge->tx_ring_count = count;
4314                         qlge->isr_stride = count;
4315                 } else if (actual < MAX_RX_RINGS) {
4316                         qlge->tx_ring_count = 1;
4317                         qlge->rss_ring_count = actual - 1;
4318                 }
4319                 qlge->intr_cnt = count;
4320                 qlge->rx_ring_count = qlge->tx_ring_count +
4321                     qlge->rss_ring_count;
4322         }
4323         cmn_err(CE_NOTE, "!qlge(%d) tx %d, rss %d, stride %d\n", qlge->instance,
4324             qlge->tx_ring_count, qlge->rss_ring_count, qlge->isr_stride);
4325 
4326         /*
4327          * Get priority for first vector, assume remaining are all the same
4328          */
4329         rc = ddi_intr_get_pri(qlge->htable[0], &qlge->intr_pri);
4330         if (rc != DDI_SUCCESS) {
4331                 QL_PRINT(DBG_INIT, ("Get interrupt priority failed: %d\n", rc));
4332                 goto ql_intr_alloc_fail;
4333         }
4334 
4335         rc = ddi_intr_get_cap(qlge->htable[0], &qlge->intr_cap);
4336         if (rc != DDI_SUCCESS) {
4337                 QL_PRINT(DBG_INIT, ("Get interrupt cap failed: %d\n", rc));
4338                 goto ql_intr_alloc_fail;
4339         }
4340 
4341         qlge->intr_type = intr_type;
4342 
4343         return (DDI_SUCCESS);
4344 
4345 ql_intr_alloc_fail:
4346         ql_free_irq_vectors(qlge);
4347 
4348         return (DDI_FAILURE);
4349 }
4350 
4351 /*
4352  * Allocate interrupt vector(s) for one of the following interrupt types, MSI-X,
4353  * MSI or Legacy. In MSI and Legacy modes we only support a single receive and
4354  * transmit queue.
4355  */
4356 int
4357 ql_alloc_irqs(qlge_t *qlge)
4358 {
4359         int intr_types;
4360         int rval;
4361 
4362         /*
4363          * Get supported interrupt types
4364          */
4365         if (ddi_intr_get_supported_types(qlge->dip, &intr_types)
4366             != DDI_SUCCESS) {
4367                 cmn_err(CE_WARN, "%s(%d):ddi_intr_get_supported_types failed",
4368                     __func__, qlge->instance);
4369 
4370                 return (DDI_FAILURE);
4371         }
4372 
4373         QL_PRINT(DBG_INIT, ("%s(%d) Interrupt types supported %d\n",
4374             __func__, qlge->instance, intr_types));
4375 
4376         /* Install MSI-X interrupts */
4377         if ((intr_types & DDI_INTR_TYPE_MSIX) != 0) {
4378                 QL_PRINT(DBG_INIT, ("%s(%d) MSI-X interrupt supported %d\n",
4379                     __func__, qlge->instance, intr_types));
4380                 rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_MSIX);
4381                 if (rval == DDI_SUCCESS) {
4382                         return (rval);
4383                 }
4384                 QL_PRINT(DBG_INIT, ("%s(%d) MSI-X interrupt allocation failed,"
4385                     " trying MSI interrupts ...\n", __func__, qlge->instance));
4386         }
4387 
4388         /*
4389          * We will have 2 completion queues in MSI / Legacy mode,
4390          * Queue 0 for default completions
4391          * Queue 1 for transmit completions
4392          */
4393         qlge->rss_ring_count = 1; /* Default completion queue (0) for all */
4394         qlge->tx_ring_count = 1; /* Single tx completion queue */
4395         qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count;
4396 
4397         QL_PRINT(DBG_INIT, ("%s(%d) Falling back to single completion queue \n",
4398             __func__, qlge->instance));
4399         /*
4400          * Add the h/w interrupt handler and initialise mutexes
4401          */
4402         rval = DDI_FAILURE;
4403 
4404         /*
4405          * If OS supports MSIX interrupt but fails to allocate, then try
4406          * MSI interrupt. If MSI interrupt allocation fails also, then roll
4407          * back to fixed interrupt.
4408          */
4409         if (intr_types & DDI_INTR_TYPE_MSI) {
4410                 rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_MSI);
4411                 if (rval == DDI_SUCCESS) {
4412                         qlge->intr_type = DDI_INTR_TYPE_MSI;
4413                         QL_PRINT(DBG_INIT, ("%s(%d) use MSI Interrupt \n",
4414                             __func__, qlge->instance));
4415                 }
4416         }
4417 
4418         /* Try Fixed interrupt Legacy mode */
4419         if (rval != DDI_SUCCESS) {
4420                 rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_FIXED);
4421                 if (rval != DDI_SUCCESS) {
4422                         cmn_err(CE_WARN, "%s(%d):Legacy mode interrupt "
4423                             "allocation failed",
4424                             __func__, qlge->instance);
4425                 } else {
4426                         qlge->intr_type = DDI_INTR_TYPE_FIXED;
4427                         QL_PRINT(DBG_INIT, ("%s(%d) use Fixed Interrupt \n",
4428                             __func__, qlge->instance));
4429                 }
4430         }
4431 
4432         return (rval);
4433 }
4434 
4435 static void
4436 ql_free_rx_tx_locks(qlge_t *qlge)
4437 {
4438         int i;
4439         struct rx_ring *rx_ring;
4440         struct tx_ring *tx_ring;
4441 
4442         for (i = 0; i < qlge->tx_ring_count; i++) {
4443                 tx_ring = &qlge->tx_ring[i];
4444                 mutex_destroy(&tx_ring->tx_lock);
4445         }
4446 
4447         for (i = 0; i < qlge->rx_ring_count; i++) {
4448                 rx_ring = &qlge->rx_ring[i];
4449                 mutex_destroy(&rx_ring->rx_lock);
4450                 mutex_destroy(&rx_ring->sbq_lock);
4451                 mutex_destroy(&rx_ring->lbq_lock);
4452         }
4453 }
4454 
4455 /*
4456  * Frees all resources allocated during attach.
4457  *
4458  * Input:
4459  * dip = pointer to device information structure.
4460  * sequence = bits indicating resources to free.
4461  *
4462  * Context:
4463  * Kernel context.
4464  */
4465 static void
4466 ql_free_resources(qlge_t *qlge)
4467 {
4468 
4469         /* Disable driver timer */
4470         ql_stop_timer(qlge);
4471 
4472         if (qlge->sequence & INIT_MAC_REGISTERED) {
4473                 (void) mac_unregister(qlge->mh);
4474                 qlge->sequence &= ~INIT_MAC_REGISTERED;
4475         }
4476 
4477         if (qlge->sequence & INIT_MAC_ALLOC) {
4478                 /* Nothing to do, macp is already freed */
4479                 qlge->sequence &= ~INIT_MAC_ALLOC;
4480         }
4481 
4482         if (qlge->sequence & INIT_PCI_CONFIG_SETUP) {
4483                 pci_config_teardown(&qlge->pci_handle);
4484                 qlge->sequence &= ~INIT_PCI_CONFIG_SETUP;
4485         }
4486 
4487         if (qlge->sequence & INIT_INTR_ALLOC) {
4488                 ql_free_irq_vectors(qlge);
4489                 qlge->sequence &= ~INIT_ADD_INTERRUPT;
4490         }
4491 
4492         if (qlge->sequence & INIT_ADD_SOFT_INTERRUPT) {
4493                 (void) ddi_intr_remove_softint(qlge->mpi_event_intr_hdl);
4494                 (void) ddi_intr_remove_softint(qlge->mpi_reset_intr_hdl);
4495                 (void) ddi_intr_remove_softint(qlge->asic_reset_intr_hdl);
4496                 qlge->sequence &= ~INIT_ADD_SOFT_INTERRUPT;
4497         }
4498 
4499         if (qlge->sequence & INIT_KSTATS) {
4500                 ql_fini_kstats(qlge);
4501                 qlge->sequence &= ~INIT_KSTATS;
4502         }
4503 
4504         if (qlge->sequence & INIT_MUTEX) {
4505                 mutex_destroy(&qlge->gen_mutex);
4506                 mutex_destroy(&qlge->hw_mutex);
4507                 mutex_destroy(&qlge->mbx_mutex);
4508                 cv_destroy(&qlge->cv_mbx_intr);
4509                 qlge->sequence &= ~INIT_MUTEX;
4510         }
4511 
4512         if (qlge->sequence & INIT_LOCKS_CREATED) {
4513                 ql_free_rx_tx_locks(qlge);
4514                 qlge->sequence &= ~INIT_LOCKS_CREATED;
4515         }
4516 
4517         if (qlge->sequence & INIT_MEMORY_ALLOC) {
4518                 ql_free_mem_resources(qlge);
4519                 qlge->sequence &= ~INIT_MEMORY_ALLOC;
4520         }
4521 
4522         if (qlge->sequence & INIT_REGS_SETUP) {
4523                 ddi_regs_map_free(&qlge->dev_handle);
4524                 qlge->sequence &= ~INIT_REGS_SETUP;
4525         }
4526 
4527         if (qlge->sequence & INIT_DOORBELL_REGS_SETUP) {
4528                 ddi_regs_map_free(&qlge->dev_doorbell_reg_handle);
4529                 qlge->sequence &= ~INIT_DOORBELL_REGS_SETUP;
4530         }
4531 
4532         /*
4533          * free flash flt table that allocated in attach stage
4534          */
4535         if ((qlge->flt.ql_flt_entry_ptr != NULL)&&
4536             (qlge->flt.header.length != 0)) {
4537                 kmem_free(qlge->flt.ql_flt_entry_ptr, qlge->flt.header.length);
4538                 qlge->flt.ql_flt_entry_ptr = NULL;
4539         }
4540 
4541         if (qlge->sequence & INIT_FM) {
4542                 ql_fm_fini(qlge);
4543                 qlge->sequence &= ~INIT_FM;
4544         }
4545 
4546         ddi_prop_remove_all(qlge->dip);
4547         ddi_set_driver_private(qlge->dip, NULL);
4548 
4549         /* finally, free qlge structure */
4550         if (qlge->sequence & INIT_SOFTSTATE_ALLOC) {
4551                 kmem_free(qlge, sizeof (qlge_t));
4552         }
4553 }
4554 
4555 /*
4556  * Set promiscuous mode of the driver
4557  * Caller must catch HW_LOCK
4558  */
4559 void
4560 ql_set_promiscuous(qlge_t *qlge, int mode)
4561 {
4562         if (mode) {
4563                 (void) ql_set_routing_reg(qlge, RT_IDX_PROMISCUOUS_SLOT,
4564                     RT_IDX_VALID, 1);
4565         } else {
4566                 (void) ql_set_routing_reg(qlge, RT_IDX_PROMISCUOUS_SLOT,
4567                     RT_IDX_VALID, 0);
4568         }
4569 }
4570 /*
4571  * Write 'data1' to Mac Protocol Address Index Register and
4572  * 'data2' to Mac Protocol Address Data Register
4573  *  Assuming that the Mac Protocol semaphore lock has been acquired.
4574  */
4575 static int
4576 ql_write_mac_proto_regs(qlge_t *qlge, uint32_t data1, uint32_t data2)
4577 {
4578         int return_value = DDI_SUCCESS;
4579 
4580         if (ql_wait_reg_bit(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
4581             MAC_PROTOCOL_ADDRESS_INDEX_MW, BIT_SET, 5) != DDI_SUCCESS) {
4582                 cmn_err(CE_WARN, "Wait for MAC_PROTOCOL Address Register "
4583                     "timeout.");
4584                 return_value = DDI_FAILURE;
4585                 goto out;
4586         }
4587         ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX /* A8 */, data1);
4588         ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA /* 0xAC */, data2);
4589 out:
4590         return (return_value);
4591 }
4592 /*
4593  * Enable the 'index'ed multicast address in the host memory's multicast_list
4594  */
4595 int
4596 ql_add_multicast_address(qlge_t *qlge, int index)
4597 {
4598         int rtn_val = DDI_FAILURE;
4599         uint32_t offset;
4600         uint32_t value1, value2;
4601 
4602         /* Acquire the required semaphore */
4603         if (ql_sem_spinlock(qlge, QL_MAC_PROTOCOL_SEM_MASK) != DDI_SUCCESS) {
4604                 return (rtn_val);
4605         }
4606 
4607         /* Program Offset0 - lower 32 bits of the MAC address */
4608         offset = 0;
4609         value1 = MAC_PROTOCOL_ADDRESS_ENABLE | MAC_PROTOCOL_TYPE_MULTICAST |
4610             (index << 4) | offset;
4611         value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[2] << 24)
4612             |(qlge->multicast_list[index].addr.ether_addr_octet[3] << 16)
4613             |(qlge->multicast_list[index].addr.ether_addr_octet[4] << 8)
4614             |(qlge->multicast_list[index].addr.ether_addr_octet[5]));
4615         if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS)
4616                 goto out;
4617 
4618         /* Program offset1: upper 16 bits of the MAC address */
4619         offset = 1;
4620         value1 = MAC_PROTOCOL_ADDRESS_ENABLE | MAC_PROTOCOL_TYPE_MULTICAST |
4621             (index<<4) | offset;
4622         value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[0] << 8)
4623             |qlge->multicast_list[index].addr.ether_addr_octet[1]);
4624         if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) {
4625                 goto out;
4626         }
4627         rtn_val = DDI_SUCCESS;
4628 out:
4629         ql_sem_unlock(qlge, QL_MAC_PROTOCOL_SEM_MASK);
4630         return (rtn_val);
4631 }
4632 
4633 /*
4634  * Disable the 'index'ed multicast address in the host memory's multicast_list
4635  */
4636 int
4637 ql_remove_multicast_address(qlge_t *qlge, int index)
4638 {
4639         int rtn_val = DDI_FAILURE;
4640         uint32_t offset;
4641         uint32_t value1, value2;
4642 
4643         /* Acquire the required semaphore */
4644         if (ql_sem_spinlock(qlge, QL_MAC_PROTOCOL_SEM_MASK) != DDI_SUCCESS) {
4645                 return (rtn_val);
4646         }
4647         /* Program Offset0 - lower 32 bits of the MAC address */
4648         offset = 0;
4649         value1 = (MAC_PROTOCOL_TYPE_MULTICAST | offset)|(index<<4);
4650         value2 =
4651             ((qlge->multicast_list[index].addr.ether_addr_octet[2] << 24)
4652             |(qlge->multicast_list[index].addr.ether_addr_octet[3] << 16)
4653             |(qlge->multicast_list[index].addr.ether_addr_octet[4] << 8)
4654             |(qlge->multicast_list[index].addr.ether_addr_octet[5]));
4655         if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) {
4656                 goto out;
4657         }
4658         /* Program offset1: upper 16 bits of the MAC address */
4659         offset = 1;
4660         value1 = (MAC_PROTOCOL_TYPE_MULTICAST | offset)|(index<<4);
4661         value2 = 0;
4662         if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) {
4663                 goto out;
4664         }
4665         rtn_val = DDI_SUCCESS;
4666 out:
4667         ql_sem_unlock(qlge, QL_MAC_PROTOCOL_SEM_MASK);
4668         return (rtn_val);
4669 }
4670 
4671 /*
4672  * Add a new multicast address to the list of supported list
4673  * This API is called after OS called gld_set_multicast (GLDv2)
4674  * or m_multicst (GLDv3)
4675  *
4676  * Restriction:
4677  * The number of maximum multicast address is limited by hardware.
4678  */
4679 int
4680 ql_add_to_multicast_list(qlge_t *qlge, uint8_t *ep)
4681 {
4682         uint32_t index = qlge->multicast_list_count;
4683         int rval = DDI_SUCCESS;
4684         int status;
4685 
4686         if ((ep[0] & 01) == 0) {
4687                 rval = EINVAL;
4688                 goto exit;
4689         }
4690 
4691         /* if there is an availabe space in multicast_list, then add it */
4692         if (index < MAX_MULTICAST_LIST_SIZE) {
4693                 bcopy(ep, qlge->multicast_list[index].addr.ether_addr_octet,
4694                     ETHERADDRL);
4695                 /* increment the total number of addresses in multicast list */
4696                 (void) ql_add_multicast_address(qlge, index);
4697                 qlge->multicast_list_count++;
4698                 QL_PRINT(DBG_GLD,
4699                     ("%s(%d): added to index of multicast list= 0x%x, "
4700                     "total %d\n", __func__, qlge->instance, index,
4701                     qlge->multicast_list_count));
4702 
4703                 if (index > MAX_MULTICAST_HW_SIZE) {
4704                         if (!qlge->multicast_promisc) {
4705                                 status = ql_set_routing_reg(qlge,
4706                                     RT_IDX_ALLMULTI_SLOT,
4707                                     RT_IDX_MCAST, 1);
4708                                 if (status) {
4709                                         cmn_err(CE_WARN,
4710                                             "Failed to init routing reg "
4711                                             "for mcast promisc mode.");
4712                                         rval = ENOENT;
4713                                         goto exit;
4714                                 }
4715                                 qlge->multicast_promisc = B_TRUE;
4716                         }
4717                 }
4718         } else {
4719                 rval = ENOENT;
4720         }
4721 exit:
4722         return (rval);
4723 }
4724 
4725 /*
4726  * Remove an old multicast address from the list of supported multicast
4727  * addresses. This API is called after OS called gld_set_multicast (GLDv2)
4728  * or m_multicst (GLDv3)
4729  * The number of maximum multicast address is limited by hardware.
4730  */
4731 int
4732 ql_remove_from_multicast_list(qlge_t *qlge, uint8_t *ep)
4733 {
4734         uint32_t total = qlge->multicast_list_count;
4735         int i = 0;
4736         int rmv_index = 0;
4737         size_t length = sizeof (ql_multicast_addr);
4738         int status;
4739 
4740         for (i = 0; i < total; i++) {
4741                 if (bcmp(ep, &qlge->multicast_list[i].addr, ETHERADDRL) != 0) {
4742                         continue;
4743                 }
4744 
4745                 rmv_index = i;
4746                 /* block move the reset of other multicast address forward */
4747                 length = ((total -1) -i) * sizeof (ql_multicast_addr);
4748                 if (length > 0) {
4749                         bcopy(&qlge->multicast_list[i+1],
4750                             &qlge->multicast_list[i], length);
4751                 }
4752                 qlge->multicast_list_count--;
4753                 if (qlge->multicast_list_count <= MAX_MULTICAST_HW_SIZE) {
4754                         /*
4755                          * there is a deletion in multicast list table,
4756                          * re-enable them
4757                          */
4758                         for (i = rmv_index; i < qlge->multicast_list_count;
4759                             i++) {
4760                                 (void) ql_add_multicast_address(qlge, i);
4761                         }
4762                         /* and disable the last one */
4763                         (void) ql_remove_multicast_address(qlge, i);
4764 
4765                         /* disable multicast promiscuous mode */
4766                         if (qlge->multicast_promisc) {
4767                                 status = ql_set_routing_reg(qlge,
4768                                     RT_IDX_ALLMULTI_SLOT,
4769                                     RT_IDX_MCAST, 0);
4770                                 if (status) {
4771                                         cmn_err(CE_WARN,
4772                                             "Failed to init routing reg for "
4773                                             "mcast promisc mode.");
4774                                         goto exit;
4775                                 }
4776                                 /* write to config register */
4777                                 qlge->multicast_promisc = B_FALSE;
4778                         }
4779                 }
4780                 break;
4781         }
4782 exit:
4783         return (DDI_SUCCESS);
4784 }
4785 
4786 /*
4787  * Read a XGMAC register
4788  */
4789 int
4790 ql_read_xgmac_reg(qlge_t *qlge, uint32_t addr, uint32_t *val)
4791 {
4792         int rtn_val = DDI_FAILURE;
4793 
4794         /* wait for XGMAC Address register RDY bit set */
4795         if (ql_wait_reg_bit(qlge, REG_XGMAC_ADDRESS, XGMAC_ADDRESS_RDY,
4796             BIT_SET, 10) != DDI_SUCCESS) {
4797                 goto out;
4798         }
4799         /* start rx transaction */
4800         ql_write_reg(qlge, REG_XGMAC_ADDRESS, addr|XGMAC_ADDRESS_READ_TRANSACT);
4801 
4802         /*
4803          * wait for XGMAC Address register RDY bit set,
4804          * which indicates data is ready
4805          */
4806         if (ql_wait_reg_bit(qlge, REG_XGMAC_ADDRESS, XGMAC_ADDRESS_RDY,
4807             BIT_SET, 10) != DDI_SUCCESS) {
4808                 goto out;
4809         }
4810         /* read data from XGAMC_DATA register */
4811         *val = ql_read_reg(qlge, REG_XGMAC_DATA);
4812         rtn_val = DDI_SUCCESS;
4813 out:
4814         return (rtn_val);
4815 }
4816 
4817 /*
4818  * Implement checksum offload for IPv4 IP packets
4819  */
4820 static void
4821 ql_hw_csum_setup(qlge_t *qlge, uint32_t pflags, caddr_t bp,
4822     struct ob_mac_iocb_req *mac_iocb_ptr)
4823 {
4824         struct ip *iphdr = NULL;
4825         struct ether_header *ethhdr;
4826         struct ether_vlan_header *ethvhdr;
4827         struct tcphdr *tcp_hdr;
4828         uint32_t etherType;
4829         int mac_hdr_len, ip_hdr_len, tcp_udp_hdr_len;
4830         int ip_hdr_off, tcp_udp_hdr_off, hdr_off;
4831 
4832         ethhdr  = (struct ether_header *)((void *)bp);
4833         ethvhdr = (struct ether_vlan_header *)((void *)bp);
4834         /* Is this vlan packet? */
4835         if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) {
4836                 mac_hdr_len = sizeof (struct ether_vlan_header);
4837                 etherType = ntohs(ethvhdr->ether_type);
4838         } else {
4839                 mac_hdr_len = sizeof (struct ether_header);
4840                 etherType = ntohs(ethhdr->ether_type);
4841         }
4842         /* Is this IPv4 or IPv6 packet? */
4843         if (IPH_HDR_VERSION((ipha_t *)(void *)(bp+mac_hdr_len)) ==
4844             IPV4_VERSION) {
4845                 if (etherType == ETHERTYPE_IP /* 0800 */) {
4846                         iphdr = (struct ip *)(void *)(bp+mac_hdr_len);
4847                 } else {
4848                         /* EMPTY */
4849                         QL_PRINT(DBG_TX,
4850                             ("%s(%d) : IPv4 None IP packet type 0x%x\n",
4851                             __func__, qlge->instance, etherType));
4852                 }
4853         }
4854         /* ipV4 packets */
4855         if (iphdr != NULL) {
4856 
4857                 ip_hdr_len = IPH_HDR_LENGTH(iphdr);
4858                 QL_PRINT(DBG_TX,
4859                     ("%s(%d) : IPv4 header length using IPH_HDR_LENGTH:"
4860                     " %d bytes \n", __func__, qlge->instance, ip_hdr_len));
4861 
4862                 ip_hdr_off = mac_hdr_len;
4863                 QL_PRINT(DBG_TX, ("%s(%d) : ip_hdr_len=%d\n",
4864                     __func__, qlge->instance, ip_hdr_len));
4865 
4866                 mac_iocb_ptr->flag0 = (uint8_t)(mac_iocb_ptr->flag0 |
4867                     OB_MAC_IOCB_REQ_IPv4);
4868 
4869                 if (pflags & HCK_IPV4_HDRCKSUM) {
4870                         QL_PRINT(DBG_TX, ("%s(%d) : Do IPv4 header checksum\n",
4871                             __func__, qlge->instance));
4872                         mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB;
4873                         mac_iocb_ptr->flag2 = (uint8_t)(mac_iocb_ptr->flag2 |
4874                             OB_MAC_IOCB_REQ_IC);
4875                         iphdr->ip_sum = 0;
4876                         mac_iocb_ptr->hdr_off = (uint16_t)
4877                             cpu_to_le16(ip_hdr_off);
4878                 }
4879                 if (pflags & HCK_FULLCKSUM) {
4880                         if (iphdr->ip_p == IPPROTO_TCP) {
4881                                 tcp_hdr =
4882                                     (struct tcphdr *)(void *)
4883                                     ((uint8_t *)(void *)iphdr + ip_hdr_len);
4884                                 QL_PRINT(DBG_TX, ("%s(%d) : Do TCP checksum\n",
4885                                     __func__, qlge->instance));
4886                                 mac_iocb_ptr->opcode =
4887                                     OPCODE_OB_MAC_OFFLOAD_IOCB;
4888                                 mac_iocb_ptr->flag1 =
4889                                     (uint8_t)(mac_iocb_ptr->flag1 |
4890                                     OB_MAC_IOCB_REQ_TC);
4891                                 mac_iocb_ptr->flag2 =
4892                                     (uint8_t)(mac_iocb_ptr->flag2 |
4893                                     OB_MAC_IOCB_REQ_IC);
4894                                 iphdr->ip_sum = 0;
4895                                 tcp_udp_hdr_off = mac_hdr_len+ip_hdr_len;
4896                                 tcp_udp_hdr_len = tcp_hdr->th_off*4;
4897                                 QL_PRINT(DBG_TX, ("%s(%d): tcp header len:%d\n",
4898                                     __func__, qlge->instance, tcp_udp_hdr_len));
4899                                 hdr_off = ip_hdr_off;
4900                                 tcp_udp_hdr_off <<= 6;
4901                                 hdr_off |= tcp_udp_hdr_off;
4902                                 mac_iocb_ptr->hdr_off =
4903                                     (uint16_t)cpu_to_le16(hdr_off);
4904                                 mac_iocb_ptr->protocol_hdr_len = (uint16_t)
4905                                     cpu_to_le16(mac_hdr_len + ip_hdr_len +
4906                                     tcp_udp_hdr_len);
4907 
4908                                 /*
4909                                  * if the chip is unable to do pseudo header
4910                                  * cksum calculation, do it in then put the
4911                                  * result to the data passed to the chip
4912                                  */
4913                                 if (qlge->cfg_flags &
4914                                     CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) {
4915                                         ql_pseudo_cksum((uint8_t *)iphdr);
4916                                 }
4917                         } else if (iphdr->ip_p == IPPROTO_UDP) {
4918                                 QL_PRINT(DBG_TX, ("%s(%d) : Do UDP checksum\n",
4919                                     __func__, qlge->instance));
4920                                 mac_iocb_ptr->opcode =
4921                                     OPCODE_OB_MAC_OFFLOAD_IOCB;
4922                                 mac_iocb_ptr->flag1 =
4923                                     (uint8_t)(mac_iocb_ptr->flag1 |
4924                                     OB_MAC_IOCB_REQ_UC);
4925                                 mac_iocb_ptr->flag2 =
4926                                     (uint8_t)(mac_iocb_ptr->flag2 |
4927                                     OB_MAC_IOCB_REQ_IC);
4928                                 iphdr->ip_sum = 0;
4929                                 tcp_udp_hdr_off = mac_hdr_len + ip_hdr_len;
4930                                 tcp_udp_hdr_len = sizeof (struct udphdr);
4931                                 QL_PRINT(DBG_TX, ("%s(%d):udp header len:%d\n",
4932                                     __func__, qlge->instance, tcp_udp_hdr_len));
4933                                 hdr_off = ip_hdr_off;
4934                                 tcp_udp_hdr_off <<= 6;
4935                                 hdr_off |= tcp_udp_hdr_off;
4936                                 mac_iocb_ptr->hdr_off =
4937                                     (uint16_t)cpu_to_le16(hdr_off);
4938                                 mac_iocb_ptr->protocol_hdr_len = (uint16_t)
4939                                     cpu_to_le16(mac_hdr_len + ip_hdr_len
4940                                     + tcp_udp_hdr_len);
4941 
4942                                 /*
4943                                  * if the chip is unable to calculate pseudo
4944                                  * hdr cksum,do it in then put the result to
4945                                  * the data passed to the chip
4946                                  */
4947                                 if (qlge->cfg_flags &
4948                                     CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) {
4949                                         ql_pseudo_cksum((uint8_t *)iphdr);
4950                                 }
4951                         }
4952                 }
4953         }
4954 }
4955 
4956 /*
4957  * For TSO/LSO:
4958  * MAC frame transmission with TCP large segment offload is performed in the
4959  * same way as the MAC frame transmission with checksum offload with the
4960  * exception that the maximum TCP segment size (MSS) must be specified to
4961  * allow the chip to segment the data into legal sized frames.
4962  * The host also needs to calculate a pseudo-header checksum over the
4963  * following fields:
4964  * Source IP Address, Destination IP Address, and the Protocol.
4965  * The TCP length is not included in the pseudo-header calculation.
4966  * The pseudo-header checksum is place in the TCP checksum field of the
4967  * prototype header.
4968  */
4969 static void
4970 ql_lso_pseudo_cksum(uint8_t *buf)
4971 {
4972         uint32_t cksum;
4973         uint16_t iphl;
4974         uint16_t proto;
4975 
4976         /*
4977          * Calculate the LSO pseudo-header checksum.
4978          */
4979         iphl = (uint16_t)(4 * (buf[0] & 0xF));
4980         cksum = proto = buf[9];
4981         cksum += (((uint16_t)buf[12])<<8) + buf[13];
4982         cksum += (((uint16_t)buf[14])<<8) + buf[15];
4983         cksum += (((uint16_t)buf[16])<<8) + buf[17];
4984         cksum += (((uint16_t)buf[18])<<8) + buf[19];
4985         cksum = (cksum>>16) + (cksum & 0xFFFF);
4986         cksum = (cksum>>16) + (cksum & 0xFFFF);
4987 
4988         /*
4989          * Point it to the TCP/UDP header, and
4990          * update the checksum field.
4991          */
4992         buf += iphl + ((proto == IPPROTO_TCP) ?
4993             TCP_CKSUM_OFFSET : UDP_CKSUM_OFFSET);
4994 
4995         *(uint16_t *)(void *)buf = (uint16_t)htons((uint16_t)cksum);
4996 }
4997 
4998 /*
4999  * For IPv4 IP packets, distribute the tx packets evenly among tx rings
5000  */
5001 typedef uint32_t        ub4; /* unsigned 4-byte quantities */
5002 typedef uint8_t         ub1;
5003 
5004 #define hashsize(n)     ((ub4)1<<(n))
5005 #define hashmask(n)     (hashsize(n)-1)
5006 
5007 #define mix(a, b, c) \
5008 { \
5009         a -= b; a -= c; a ^= (c>>13); \
5010         b -= c; b -= a; b ^= (a<<8); \
5011         c -= a; c -= b; c ^= (b>>13); \
5012         a -= b; a -= c; a ^= (c>>12);  \
5013         b -= c; b -= a; b ^= (a<<16); \
5014         c -= a; c -= b; c ^= (b>>5); \
5015         a -= b; a -= c; a ^= (c>>3);  \
5016         b -= c; b -= a; b ^= (a<<10); \
5017         c -= a; c -= b; c ^= (b>>15); \
5018 }
5019 
5020 ub4
5021 hash(k, length, initval)
5022 register ub1 *k;        /* the key */
5023 register ub4 length;    /* the length of the key */
5024 register ub4 initval;   /* the previous hash, or an arbitrary value */
5025 {
5026         register ub4 a, b, c, len;
5027 
5028         /* Set up the internal state */
5029         len = length;
5030         a = b = 0x9e3779b9;     /* the golden ratio; an arbitrary value */
5031         c = initval;            /* the previous hash value */
5032 
5033         /* handle most of the key */
5034         while (len >= 12) {
5035                 a += (k[0] +((ub4)k[1]<<8) +((ub4)k[2]<<16) +((ub4)k[3]<<24));
5036                 b += (k[4] +((ub4)k[5]<<8) +((ub4)k[6]<<16) +((ub4)k[7]<<24));
5037                 c += (k[8] +((ub4)k[9]<<8) +((ub4)k[10]<<16)+((ub4)k[11]<<24));
5038                 mix(a, b, c);
5039                 k += 12;
5040                 len -= 12;
5041         }
5042 
5043         /* handle the last 11 bytes */
5044         c += length;
5045         /* all the case statements fall through */
5046         switch (len) {
5047                 /* FALLTHRU */
5048         case 11: c += ((ub4)k[10]<<24);
5049                 /* FALLTHRU */
5050         case 10: c += ((ub4)k[9]<<16);
5051                 /* FALLTHRU */
5052         case 9 : c += ((ub4)k[8]<<8);
5053         /* the first byte of c is reserved for the length */
5054                 /* FALLTHRU */
5055         case 8 : b += ((ub4)k[7]<<24);
5056                 /* FALLTHRU */
5057         case 7 : b += ((ub4)k[6]<<16);
5058                 /* FALLTHRU */
5059         case 6 : b += ((ub4)k[5]<<8);
5060                 /* FALLTHRU */
5061         case 5 : b += k[4];
5062                 /* FALLTHRU */
5063         case 4 : a += ((ub4)k[3]<<24);
5064                 /* FALLTHRU */
5065         case 3 : a += ((ub4)k[2]<<16);
5066                 /* FALLTHRU */
5067         case 2 : a += ((ub4)k[1]<<8);
5068                 /* FALLTHRU */
5069         case 1 : a += k[0];
5070         /* case 0: nothing left to add */
5071         }
5072         mix(a, b, c);
5073         /* report the result */
5074         return (c);
5075 }
5076 
5077 uint8_t
5078 ql_tx_hashing(qlge_t *qlge, caddr_t bp)
5079 {
5080         struct ip *iphdr = NULL;
5081         struct ether_header *ethhdr;
5082         struct ether_vlan_header *ethvhdr;
5083         struct tcphdr *tcp_hdr;
5084         struct udphdr *udp_hdr;
5085         uint32_t etherType;
5086         int mac_hdr_len, ip_hdr_len;
5087         uint32_t h = 0; /* 0 by default */
5088         uint8_t tx_ring_id = 0;
5089         uint32_t ip_src_addr = 0;
5090         uint32_t ip_desc_addr = 0;
5091         uint16_t src_port = 0;
5092         uint16_t dest_port = 0;
5093         uint8_t key[12];
5094         QL_PRINT(DBG_TX, ("%s(%d) entered \n", __func__, qlge->instance));
5095 
5096         ethhdr = (struct ether_header *)((void *)bp);
5097         ethvhdr = (struct ether_vlan_header *)((void *)bp);
5098 
5099         if (qlge->tx_ring_count == 1)
5100                 return (tx_ring_id);
5101 
5102         /* Is this vlan packet? */
5103         if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) {
5104                 mac_hdr_len = sizeof (struct ether_vlan_header);
5105                 etherType = ntohs(ethvhdr->ether_type);
5106         } else {
5107                 mac_hdr_len = sizeof (struct ether_header);
5108                 etherType = ntohs(ethhdr->ether_type);
5109         }
5110         /* Is this IPv4 or IPv6 packet? */
5111         if (etherType == ETHERTYPE_IP /* 0800 */) {
5112                 if (IPH_HDR_VERSION((ipha_t *)(void *)(bp+mac_hdr_len))
5113                     == IPV4_VERSION) {
5114                         iphdr = (struct ip *)(void *)(bp+mac_hdr_len);
5115                 }
5116                 if (((unsigned long)iphdr) & 0x3) {
5117                         /*  IP hdr not 4-byte aligned */
5118                         return (tx_ring_id);
5119                 }
5120         }
5121         /* ipV4 packets */
5122         if (iphdr) {
5123 
5124                 ip_hdr_len = IPH_HDR_LENGTH(iphdr);
5125                 ip_src_addr = iphdr->ip_src.s_addr;
5126                 ip_desc_addr = iphdr->ip_dst.s_addr;
5127 
5128                 if (iphdr->ip_p == IPPROTO_TCP) {
5129                         tcp_hdr = (struct tcphdr *)(void *)
5130                             ((uint8_t *)iphdr + ip_hdr_len);
5131                         src_port = tcp_hdr->th_sport;
5132                         dest_port = tcp_hdr->th_dport;
5133                 } else if (iphdr->ip_p == IPPROTO_UDP) {
5134                         udp_hdr = (struct udphdr *)(void *)
5135                             ((uint8_t *)iphdr + ip_hdr_len);
5136                         src_port = udp_hdr->uh_sport;
5137                         dest_port = udp_hdr->uh_dport;
5138                 }
5139                 key[0] = (uint8_t)((ip_src_addr) &0xFF);
5140                 key[1] = (uint8_t)((ip_src_addr >> 8) &0xFF);
5141                 key[2] = (uint8_t)((ip_src_addr >> 16) &0xFF);
5142                 key[3] = (uint8_t)((ip_src_addr >> 24) &0xFF);
5143                 key[4] = (uint8_t)((ip_desc_addr) &0xFF);
5144                 key[5] = (uint8_t)((ip_desc_addr >> 8) &0xFF);
5145                 key[6] = (uint8_t)((ip_desc_addr >> 16) &0xFF);
5146                 key[7] = (uint8_t)((ip_desc_addr >> 24) &0xFF);
5147                 key[8] = (uint8_t)((src_port) &0xFF);
5148                 key[9] = (uint8_t)((src_port >> 8) &0xFF);
5149                 key[10] = (uint8_t)((dest_port) &0xFF);
5150                 key[11] = (uint8_t)((dest_port >> 8) &0xFF);
5151                 h = hash(key, 12, 0); /* return 32 bit */
5152                 tx_ring_id = (h & (qlge->tx_ring_count - 1));
5153                 if (tx_ring_id >= qlge->tx_ring_count) {
5154                         cmn_err(CE_WARN, "%s bad tx_ring_id %d\n",
5155                             __func__, tx_ring_id);
5156                         tx_ring_id = 0;
5157                 }
5158         }
5159         return (tx_ring_id);
5160 }
5161 
5162 /*
5163  * Tell the hardware to do Large Send Offload (LSO)
5164  *
5165  * Some fields in ob_mac_iocb need to be set so hardware can know what is
5166  * the incoming packet, TCP or UDP, whether a VLAN tag needs to be inserted
5167  * in the right place of the packet etc, thus, hardware can process the
5168  * packet correctly.
5169  */
5170 static void
5171 ql_hw_lso_setup(qlge_t *qlge, uint32_t mss, caddr_t bp,
5172     struct ob_mac_iocb_req *mac_iocb_ptr)
5173 {
5174         struct ip *iphdr = NULL;
5175         struct ether_header *ethhdr;
5176         struct ether_vlan_header *ethvhdr;
5177         struct tcphdr *tcp_hdr;
5178         struct udphdr *udp_hdr;
5179         uint32_t etherType;
5180         uint16_t mac_hdr_len, ip_hdr_len, tcp_udp_hdr_len;
5181         uint16_t ip_hdr_off, tcp_udp_hdr_off, hdr_off;
5182 
5183         ethhdr = (struct ether_header *)(void *)bp;
5184         ethvhdr = (struct ether_vlan_header *)(void *)bp;
5185 
5186         /* Is this vlan packet? */
5187         if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) {
5188                 mac_hdr_len = sizeof (struct ether_vlan_header);
5189                 etherType = ntohs(ethvhdr->ether_type);
5190         } else {
5191                 mac_hdr_len = sizeof (struct ether_header);
5192                 etherType = ntohs(ethhdr->ether_type);
5193         }
5194         /* Is this IPv4 or IPv6 packet? */
5195         if (IPH_HDR_VERSION((ipha_t *)(void *)(bp + mac_hdr_len)) ==
5196             IPV4_VERSION) {
5197                 if (etherType == ETHERTYPE_IP /* 0800 */) {
5198                         iphdr   = (struct ip *)(void *)(bp+mac_hdr_len);
5199                 } else {
5200                         /* EMPTY */
5201                         QL_PRINT(DBG_TX, ("%s(%d) : IPv4 None IP packet"
5202                             " type 0x%x\n",
5203                             __func__, qlge->instance, etherType));
5204                 }
5205         }
5206 
5207         if (iphdr != NULL) { /* ipV4 packets */
5208                 ip_hdr_len = (uint16_t)IPH_HDR_LENGTH(iphdr);
5209                 QL_PRINT(DBG_TX,
5210                     ("%s(%d) : IPv4 header length using IPH_HDR_LENGTH: %d"
5211                     " bytes \n", __func__, qlge->instance, ip_hdr_len));
5212 
5213                 ip_hdr_off = mac_hdr_len;
5214                 QL_PRINT(DBG_TX, ("%s(%d) : ip_hdr_len=%d\n",
5215                     __func__, qlge->instance, ip_hdr_len));
5216 
5217                 mac_iocb_ptr->flag0 = (uint8_t)(mac_iocb_ptr->flag0 |
5218                     OB_MAC_IOCB_REQ_IPv4);
5219                 if (qlge->cfg_flags & CFG_CKSUM_FULL_IPv4) {
5220                         if (iphdr->ip_p == IPPROTO_TCP) {
5221                                 tcp_hdr = (struct tcphdr *)(void *)
5222                                     ((uint8_t *)(void *)iphdr +
5223                                     ip_hdr_len);
5224                                 QL_PRINT(DBG_TX, ("%s(%d) : Do TSO on TCP "
5225                                     "packet\n",
5226                                     __func__, qlge->instance));
5227                                 mac_iocb_ptr->opcode =
5228                                     OPCODE_OB_MAC_OFFLOAD_IOCB;
5229                                 mac_iocb_ptr->flag1 =
5230                                     (uint8_t)(mac_iocb_ptr->flag1 |
5231                                     OB_MAC_IOCB_REQ_LSO);
5232                                 iphdr->ip_sum = 0;
5233                                 tcp_udp_hdr_off =
5234                                     (uint16_t)(mac_hdr_len+ip_hdr_len);
5235                                 tcp_udp_hdr_len =
5236                                     (uint16_t)(tcp_hdr->th_off*4);
5237                                 QL_PRINT(DBG_TX, ("%s(%d): tcp header len:%d\n",
5238                                     __func__, qlge->instance, tcp_udp_hdr_len));
5239                                 hdr_off = ip_hdr_off;
5240                                 tcp_udp_hdr_off <<= 6;
5241                                 hdr_off |= tcp_udp_hdr_off;
5242                                 mac_iocb_ptr->hdr_off =
5243                                     (uint16_t)cpu_to_le16(hdr_off);
5244                                 mac_iocb_ptr->protocol_hdr_len = (uint16_t)
5245                                     cpu_to_le16(mac_hdr_len + ip_hdr_len +
5246                                     tcp_udp_hdr_len);
5247                                 mac_iocb_ptr->mss = (uint16_t)cpu_to_le16(mss);
5248 
5249                                 /*
5250                                  * if the chip is unable to calculate pseudo
5251                                  * header checksum, do it in then put the result
5252                                  * to the data passed to the chip
5253                                  */
5254                                 if (qlge->cfg_flags &
5255                                     CFG_HW_UNABLE_PSEUDO_HDR_CKSUM)
5256                                         ql_lso_pseudo_cksum((uint8_t *)iphdr);
5257                         } else if (iphdr->ip_p == IPPROTO_UDP) {
5258                                 udp_hdr = (struct udphdr *)(void *)
5259                                     ((uint8_t *)(void *)iphdr
5260                                     + ip_hdr_len);
5261                                 QL_PRINT(DBG_TX, ("%s(%d) : Do TSO on UDP "
5262                                     "packet\n",
5263                                     __func__, qlge->instance));
5264                                 mac_iocb_ptr->opcode =
5265                                     OPCODE_OB_MAC_OFFLOAD_IOCB;
5266                                 mac_iocb_ptr->flag1 =
5267                                     (uint8_t)(mac_iocb_ptr->flag1 |
5268                                     OB_MAC_IOCB_REQ_LSO);
5269                                 iphdr->ip_sum = 0;
5270                                 tcp_udp_hdr_off =
5271                                     (uint16_t)(mac_hdr_len+ip_hdr_len);
5272                                 tcp_udp_hdr_len =
5273                                     (uint16_t)(udp_hdr->uh_ulen*4);
5274                                 QL_PRINT(DBG_TX, ("%s(%d):udp header len:%d\n",
5275                                     __func__, qlge->instance, tcp_udp_hdr_len));
5276                                 hdr_off = ip_hdr_off;
5277                                 tcp_udp_hdr_off <<= 6;
5278                                 hdr_off |= tcp_udp_hdr_off;
5279                                 mac_iocb_ptr->hdr_off =
5280                                     (uint16_t)cpu_to_le16(hdr_off);
5281                                 mac_iocb_ptr->protocol_hdr_len = (uint16_t)
5282                                     cpu_to_le16(mac_hdr_len + ip_hdr_len +
5283                                     tcp_udp_hdr_len);
5284                                 mac_iocb_ptr->mss = (uint16_t)cpu_to_le16(mss);
5285 
5286                                 /*
5287                                  * if the chip is unable to do pseudo header
5288                                  * checksum calculation, do it here then put the
5289                                  * result to the data passed to the chip
5290                                  */
5291                                 if (qlge->cfg_flags &
5292                                     CFG_HW_UNABLE_PSEUDO_HDR_CKSUM)
5293                                         ql_lso_pseudo_cksum((uint8_t *)iphdr);
5294                         }
5295                 }
5296         }
5297 }
5298 
5299 /*
5300  * Generic packet sending function which is used to send one packet.
5301  */
5302 int
5303 ql_send_common(struct tx_ring *tx_ring, mblk_t *mp)
5304 {
5305         struct tx_ring_desc *tx_cb;
5306         struct ob_mac_iocb_req *mac_iocb_ptr;
5307         mblk_t *tp;
5308         size_t msg_len = 0;
5309         size_t off;
5310         caddr_t bp;
5311         size_t nbyte, total_len;
5312         uint_t i = 0;
5313         int j = 0, frags = 0;
5314         uint32_t phy_addr_low, phy_addr_high;
5315         uint64_t phys_addr;
5316         clock_t now;
5317         uint32_t pflags = 0;
5318         uint32_t mss = 0;
5319         enum tx_mode_t tx_mode;
5320         struct oal_entry *oal_entry;
5321         int status;
5322         uint_t ncookies, oal_entries, max_oal_entries;
5323         size_t max_seg_len = 0;
5324         boolean_t use_lso = B_FALSE;
5325         struct oal_entry *tx_entry = NULL;
5326         struct oal_entry *last_oal_entry;
5327         qlge_t *qlge = tx_ring->qlge;
5328         ddi_dma_cookie_t dma_cookie;
5329         size_t tx_buf_len = QL_MAX_COPY_LENGTH;
5330         int force_pullup = 0;
5331 
5332         tp = mp;
5333         total_len = msg_len = 0;
5334         max_oal_entries = TX_DESC_PER_IOCB + MAX_SG_ELEMENTS-1;
5335 
5336         /* Calculate number of data and segments in the incoming message */
5337         for (tp = mp; tp != NULL; tp = tp->b_cont) {
5338                 nbyte = MBLKL(tp);
5339                 total_len += nbyte;
5340                 max_seg_len = max(nbyte, max_seg_len);
5341                 QL_PRINT(DBG_TX, ("Requested sending data in %d segments, "
5342                     "total length: %d\n", frags, nbyte));
5343                 frags++;
5344         }
5345 
5346         if (total_len >= QL_LSO_MAX) {
5347                 freemsg(mp);
5348 #ifdef QLGE_LOAD_UNLOAD
5349                 cmn_err(CE_NOTE, "%s: quit, packet oversize %d\n",
5350                     __func__, (int)total_len);
5351 #endif
5352                 return (NULL);
5353         }
5354 
5355         bp = (caddr_t)mp->b_rptr;
5356         if (bp[0] & 1) {
5357                 if (bcmp(bp, ql_ether_broadcast_addr.ether_addr_octet,
5358                     ETHERADDRL) == 0) {
5359                         QL_PRINT(DBG_TX, ("Broadcast packet\n"));
5360                         tx_ring->brdcstxmt++;
5361                 } else {
5362                         QL_PRINT(DBG_TX, ("multicast packet\n"));
5363                         tx_ring->multixmt++;
5364                 }
5365         }
5366 
5367         tx_ring->obytes += total_len;
5368         tx_ring->opackets ++;
5369 
5370         QL_PRINT(DBG_TX, ("total requested sending data length: %d, in %d segs,"
5371             " max seg len: %d\n", total_len, frags, max_seg_len));
5372 
5373         /* claim a free slot in tx ring */
5374         tx_cb = &tx_ring->wq_desc[tx_ring->prod_idx];
5375 
5376         /* get the tx descriptor */
5377         mac_iocb_ptr = tx_cb->queue_entry;
5378 
5379         bzero((void *)mac_iocb_ptr, 20);
5380 
5381         ASSERT(tx_cb->mp == NULL);
5382 
5383         /*
5384          * Decide to use DMA map or copy mode.
5385          * DMA map mode must be used when the total msg length is more than the
5386          * tx buffer length.
5387          */
5388 
5389         if (total_len > tx_buf_len)
5390                 tx_mode = USE_DMA;
5391         else if (max_seg_len > QL_MAX_COPY_LENGTH)
5392                 tx_mode = USE_DMA;
5393         else
5394                 tx_mode = USE_COPY;
5395 
5396         if (qlge->chksum_cap) {
5397                 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags);
5398                 QL_PRINT(DBG_TX, ("checksum flag is :0x%x, card capability "
5399                     "is 0x%x \n", pflags, qlge->chksum_cap));
5400                 if (qlge->lso_enable) {
5401                         uint32_t lso_flags = 0;
5402                         mac_lso_get(mp, &mss, &lso_flags);
5403                         use_lso = (lso_flags == HW_LSO);
5404                 }
5405                 QL_PRINT(DBG_TX, ("mss :%d, use_lso %x \n",
5406                     mss, use_lso));
5407         }
5408 
5409 do_pullup:
5410 
5411         /* concatenate all frags into one large packet if too fragmented */
5412         if (((tx_mode == USE_DMA)&&(frags > QL_MAX_TX_DMA_HANDLES)) ||
5413             force_pullup) {
5414                 mblk_t *mp1;
5415                 if ((mp1 = msgpullup(mp, -1)) != NULL) {
5416                         freemsg(mp);
5417                         mp = mp1;
5418                         frags = 1;
5419                 } else {
5420                         tx_ring->tx_fail_dma_bind++;
5421                         goto bad;
5422                 }
5423         }
5424 
5425         tx_cb->tx_bytes = (uint32_t)total_len;
5426         tx_cb->mp = mp;
5427         tx_cb->tx_dma_handle_used = 0;
5428 
5429         if (tx_mode == USE_DMA) {
5430                 msg_len = total_len;
5431 
5432                 mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB;
5433                 mac_iocb_ptr->tid = tx_ring->prod_idx;
5434                 mac_iocb_ptr->frame_len = (uint32_t)cpu_to_le32(msg_len);
5435                 mac_iocb_ptr->txq_idx = tx_ring->wq_id;
5436 
5437                 tx_entry = &mac_iocb_ptr->oal_entry[0];
5438                 oal_entry = NULL;
5439 
5440                 for (tp = mp, oal_entries = j = 0; tp != NULL;
5441                     tp = tp->b_cont) {
5442                         /* if too many tx dma handles needed */
5443                         if (j >= QL_MAX_TX_DMA_HANDLES) {
5444                                 tx_ring->tx_no_dma_handle++;
5445                                 if (!force_pullup) {
5446                                         force_pullup = 1;
5447                                         goto do_pullup;
5448                                 } else {
5449                                         goto bad;
5450                                 }
5451                         }
5452                         nbyte = (uint16_t)MBLKL(tp);
5453                         if (nbyte == 0)
5454                                 continue;
5455 
5456                         status = ddi_dma_addr_bind_handle(
5457                             tx_cb->tx_dma_handle[j], NULL,
5458                             (caddr_t)tp->b_rptr, nbyte,
5459                             DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
5460                             0, &dma_cookie, &ncookies);
5461 
5462                         QL_PRINT(DBG_TX, ("map sending data segment: %d, "
5463                             "length: %d, spans in %d cookies\n",
5464                             j, nbyte, ncookies));
5465 
5466                         if (status != DDI_DMA_MAPPED) {
5467                                 goto bad;
5468                         }
5469                         /*
5470                          * Each fragment can span several cookies. One cookie
5471                          * will use one tx descriptor to transmit.
5472                          */
5473                         for (i = ncookies; i > 0; i--, tx_entry++,
5474                             oal_entries++) {
5475                                 /*
5476                                  * The number of TX descriptors that can be
5477                                  *  saved in tx iocb and oal list is limited
5478                                  */
5479                                 if (oal_entries > max_oal_entries) {
5480                                         tx_ring->tx_no_dma_cookie++;
5481                                         if (!force_pullup) {
5482                                                 force_pullup = 1;
5483                                                 goto do_pullup;
5484                                         } else {
5485                                                 goto bad;
5486                                         }
5487                                 }
5488 
5489                                 if ((oal_entries == TX_DESC_PER_IOCB) &&
5490                                     !oal_entry) {
5491                                         /*
5492                                          * Time to switch to an oal list
5493                                          * The last entry should be copied
5494                                          * to first entry in the oal list
5495                                          */
5496                                         oal_entry = tx_cb->oal;
5497                                         tx_entry =
5498                                             &mac_iocb_ptr->oal_entry[
5499                                             TX_DESC_PER_IOCB-1];
5500                                         bcopy(tx_entry, oal_entry,
5501                                             sizeof (*oal_entry));
5502 
5503                                         /*
5504                                          * last entry should be updated to
5505                                          * point to the extended oal list itself
5506                                          */
5507                                         tx_entry->buf_addr_low =
5508                                             cpu_to_le32(
5509                                             LS_64BITS(tx_cb->oal_dma_addr));
5510                                         tx_entry->buf_addr_high =
5511                                             cpu_to_le32(
5512                                             MS_64BITS(tx_cb->oal_dma_addr));
5513                                         /*
5514                                          * Point tx_entry to the oal list
5515                                          * second entry
5516                                          */
5517                                         tx_entry = &oal_entry[1];
5518                                 }
5519 
5520                                 tx_entry->buf_len =
5521                                     (uint32_t)cpu_to_le32(dma_cookie.dmac_size);
5522                                 phys_addr = dma_cookie.dmac_laddress;
5523                                 tx_entry->buf_addr_low =
5524                                     cpu_to_le32(LS_64BITS(phys_addr));
5525                                 tx_entry->buf_addr_high =
5526                                     cpu_to_le32(MS_64BITS(phys_addr));
5527 
5528                                 last_oal_entry = tx_entry;
5529 
5530                                 if (i > 1)
5531                                         ddi_dma_nextcookie(
5532                                             tx_cb->tx_dma_handle[j],
5533                                             &dma_cookie);
5534                         }
5535                         j++;
5536                 }
5537                 /*
5538                  * if OAL is used, the last oal entry in tx iocb indicates
5539                  * number of additional address/len pairs in OAL
5540                  */
5541                 if (oal_entries > TX_DESC_PER_IOCB) {
5542                         tx_entry = &mac_iocb_ptr->oal_entry[TX_DESC_PER_IOCB-1];
5543                         tx_entry->buf_len = (uint32_t)
5544                             (cpu_to_le32((sizeof (struct oal_entry) *
5545                             (oal_entries -TX_DESC_PER_IOCB+1))|OAL_CONT_ENTRY));
5546                 }
5547                 last_oal_entry->buf_len = cpu_to_le32(
5548                     le32_to_cpu(last_oal_entry->buf_len)|OAL_LAST_ENTRY);
5549 
5550                 tx_cb->tx_dma_handle_used = j;
5551                 QL_PRINT(DBG_TX, ("total tx_dma_handle_used %d cookies %d \n",
5552                     j, oal_entries));
5553 
5554                 bp = (caddr_t)mp->b_rptr;
5555         }
5556         if (tx_mode == USE_COPY) {
5557                 bp = tx_cb->copy_buffer;
5558                 off = 0;
5559                 nbyte = 0;
5560                 frags = 0;
5561                 /*
5562                  * Copy up to tx_buf_len of the transmit data
5563                  * from mp to tx buffer
5564                  */
5565                 for (tp = mp; tp != NULL; tp = tp->b_cont) {
5566                         nbyte = MBLKL(tp);
5567                         if ((off + nbyte) <= tx_buf_len) {
5568                                 bcopy(tp->b_rptr, &bp[off], nbyte);
5569                                 off += nbyte;
5570                                 frags ++;
5571                         }
5572                 }
5573 
5574                 msg_len = off;
5575 
5576                 mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB;
5577                 mac_iocb_ptr->tid = tx_ring->prod_idx;
5578                 mac_iocb_ptr->frame_len = (uint32_t)cpu_to_le32(msg_len);
5579                 mac_iocb_ptr->txq_idx = tx_ring->wq_id;
5580 
5581                 QL_PRINT(DBG_TX, ("Copy Mode:actual sent data length is: %d, "
5582                     "from %d segaments\n", msg_len, frags));
5583 
5584                 phys_addr = tx_cb->copy_buffer_dma_addr;
5585                 phy_addr_low = cpu_to_le32(LS_64BITS(phys_addr));
5586                 phy_addr_high = cpu_to_le32(MS_64BITS(phys_addr));
5587 
5588                 QL_DUMP(DBG_TX, "\t requested sending data:\n",
5589                     (uint8_t *)tx_cb->copy_buffer, 8, total_len);
5590 
5591                 mac_iocb_ptr->oal_entry[0].buf_len = (uint32_t)
5592                     cpu_to_le32(msg_len | OAL_LAST_ENTRY);
5593                 mac_iocb_ptr->oal_entry[0].buf_addr_low  = phy_addr_low;
5594                 mac_iocb_ptr->oal_entry[0].buf_addr_high = phy_addr_high;
5595 
5596                 freemsg(mp); /* no need, we have copied */
5597                 tx_cb->mp = NULL;
5598         } /* End of Copy Mode */
5599 
5600         /* Do TSO/LSO on TCP packet? */
5601         if (use_lso && mss) {
5602                 ql_hw_lso_setup(qlge, mss, bp, mac_iocb_ptr);
5603         } else if (pflags & qlge->chksum_cap) {
5604                 /* Do checksum offloading */
5605                 ql_hw_csum_setup(qlge, pflags, bp, mac_iocb_ptr);
5606         }
5607 
5608         /* let device know the latest outbound IOCB */
5609         (void) ddi_dma_sync(tx_ring->wq_dma.dma_handle,
5610             (off_t)((uintptr_t)mac_iocb_ptr - (uintptr_t)tx_ring->wq_dma.vaddr),
5611             (size_t)sizeof (*mac_iocb_ptr), DDI_DMA_SYNC_FORDEV);
5612 
5613         if (tx_mode == USE_DMA) {
5614                 /* let device know the latest outbound OAL if necessary */
5615                 if (oal_entries > TX_DESC_PER_IOCB) {
5616                         (void) ddi_dma_sync(tx_cb->oal_dma.dma_handle,
5617                             (off_t)0,
5618                             (sizeof (struct oal_entry) *
5619                             (oal_entries -TX_DESC_PER_IOCB+1)),
5620                             DDI_DMA_SYNC_FORDEV);
5621                 }
5622         } else { /* for USE_COPY mode, tx buffer has changed */
5623                 /* let device know the latest change */
5624                 (void) ddi_dma_sync(tx_cb->oal_dma.dma_handle,
5625                 /* copy buf offset */
5626                     (off_t)(sizeof (oal_entry) * MAX_SG_ELEMENTS),
5627                     msg_len, DDI_DMA_SYNC_FORDEV);
5628         }
5629 
5630         /* save how the packet was sent */
5631         tx_cb->tx_type = tx_mode;
5632 
5633         QL_DUMP_REQ_PKT(qlge, mac_iocb_ptr, tx_cb->oal, oal_entries);
5634         /* reduce the number of available tx slot */
5635         atomic_dec_32(&tx_ring->tx_free_count);
5636 
5637         tx_ring->prod_idx++;
5638         if (tx_ring->prod_idx >= tx_ring->wq_len)
5639                 tx_ring->prod_idx = 0;
5640 
5641         now = ddi_get_lbolt();
5642         qlge->last_tx_time = now;
5643 
5644         return (DDI_SUCCESS);
5645 
5646 bad:
5647         /*
5648          * if for any reason driver can not send, delete
5649          * the message pointer, mp
5650          */
5651         now = ddi_get_lbolt();
5652         freemsg(mp);
5653         mp = NULL;
5654         tx_cb->mp = NULL;
5655         for (i = 0; i < j; i++)
5656                 (void) ddi_dma_unbind_handle(tx_cb->tx_dma_handle[i]);
5657 
5658         QL_PRINT(DBG_TX, ("%s(%d) failed at 0x%x",
5659             __func__, qlge->instance, (int)now));
5660 
5661         return (DDI_SUCCESS);
5662 }
5663 
5664 
5665 /*
5666  * Initializes hardware and driver software flags before the driver
5667  * is finally ready to work.
5668  */
5669 int
5670 ql_do_start(qlge_t *qlge)
5671 {
5672         int i;
5673         struct rx_ring *rx_ring;
5674         uint16_t lbq_buf_size;
5675         int rings_done;
5676 
5677         ASSERT(qlge != NULL);
5678 
5679         mutex_enter(&qlge->hw_mutex);
5680 
5681         /* Reset adapter */
5682         (void) ql_asic_reset(qlge);
5683 
5684         lbq_buf_size = (uint16_t)
5685             ((qlge->mtu == ETHERMTU)? LRG_BUF_NORMAL_SIZE : LRG_BUF_JUMBO_SIZE);
5686         if (qlge->rx_ring[0].lbq_buf_size != lbq_buf_size) {
5687 #ifdef QLGE_LOAD_UNLOAD
5688                 cmn_err(CE_NOTE, "realloc buffers old: %d new: %d\n",
5689                     qlge->rx_ring[0].lbq_buf_size, lbq_buf_size);
5690 #endif
5691                 /*
5692                  * Check if any ring has buffers still with upper layers
5693                  * If buffers are pending with upper layers, we use the
5694                  * existing buffers and don't reallocate new ones
5695                  * Unfortunately there is no way to evict buffers from
5696                  * upper layers. Using buffers with the current size may
5697                  * cause slightly sub-optimal performance, but that seems
5698                  * to be the easiest way to handle this situation.
5699                  */
5700                 rings_done = 0;
5701                 for (i = 0; i < qlge->rx_ring_count; i++) {
5702                         rx_ring = &qlge->rx_ring[i];
5703                         if (rx_ring->rx_indicate == 0)
5704                                 rings_done++;
5705                         else
5706                                 break;
5707                 }
5708                 /*
5709                  * No buffers pending with upper layers;
5710                  * reallocte them for new MTU size
5711                  */
5712                 if (rings_done >= qlge->rx_ring_count) {
5713                         /* free large buffer pool */
5714                         for (i = 0; i < qlge->rx_ring_count; i++) {
5715                                 rx_ring = &qlge->rx_ring[i];
5716                                 if (rx_ring->type != TX_Q) {
5717                                         ql_free_sbq_buffers(rx_ring);
5718                                         ql_free_lbq_buffers(rx_ring);
5719                                 }
5720                         }
5721                         /* reallocate large buffer pool */
5722                         for (i = 0; i < qlge->rx_ring_count; i++) {
5723                                 rx_ring = &qlge->rx_ring[i];
5724                                 if (rx_ring->type != TX_Q) {
5725                                         (void) ql_alloc_sbufs(qlge, rx_ring);
5726                                         (void) ql_alloc_lbufs(qlge, rx_ring);
5727                                 }
5728                         }
5729                 }
5730         }
5731 
5732         if (ql_bringup_adapter(qlge) != DDI_SUCCESS) {
5733                 cmn_err(CE_WARN, "qlge bringup adapter failed");
5734                 mutex_exit(&qlge->hw_mutex);
5735                 if (qlge->fm_enable) {
5736                         atomic_or_32(&qlge->flags, ADAPTER_ERROR);
5737                         ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST);
5738                 }
5739                 return (DDI_FAILURE);
5740         }
5741 
5742         mutex_exit(&qlge->hw_mutex);
5743         /* if adapter is up successfully but was bad before */
5744         if (qlge->flags & ADAPTER_ERROR) {
5745                 atomic_and_32(&qlge->flags, ~ADAPTER_ERROR);
5746                 if (qlge->fm_enable) {
5747                         ddi_fm_service_impact(qlge->dip, DDI_SERVICE_RESTORED);
5748                 }
5749         }
5750 
5751         /* Get current link state */
5752         qlge->port_link_state = ql_get_link_state(qlge);
5753 
5754         if (qlge->port_link_state == LS_UP) {
5755                 QL_PRINT(DBG_GLD, ("%s(%d) Link UP !!\n",
5756                     __func__, qlge->instance));
5757                 /* If driver detects a carrier on */
5758                 CARRIER_ON(qlge);
5759         } else {
5760                 QL_PRINT(DBG_GLD, ("%s(%d) Link down\n",
5761                     __func__, qlge->instance));
5762                 /* If driver detects a lack of carrier */
5763                 CARRIER_OFF(qlge);
5764         }
5765         qlge->mac_flags = QL_MAC_STARTED;
5766         return (DDI_SUCCESS);
5767 }
5768 
5769 /*
5770  * Stop currently running driver
5771  * Driver needs to stop routing new packets to driver and wait until
5772  * all pending tx/rx buffers to be free-ed.
5773  */
5774 int
5775 ql_do_stop(qlge_t *qlge)
5776 {
5777         int rc = DDI_FAILURE;
5778         uint32_t i, j, k;
5779         struct bq_desc *sbq_desc, *lbq_desc;
5780         struct rx_ring *rx_ring;
5781 
5782         ASSERT(qlge != NULL);
5783 
5784         CARRIER_OFF(qlge);
5785 
5786         rc = ql_bringdown_adapter(qlge);
5787         if (rc != DDI_SUCCESS) {
5788                 cmn_err(CE_WARN, "qlge bringdown adapter failed.");
5789         } else
5790                 rc = DDI_SUCCESS;
5791 
5792         for (k = 0; k < qlge->rx_ring_count; k++) {
5793                 rx_ring = &qlge->rx_ring[k];
5794                 if (rx_ring->type != TX_Q) {
5795                         j = rx_ring->lbq_use_head;
5796 #ifdef QLGE_LOAD_UNLOAD
5797                         cmn_err(CE_NOTE, "ring %d: move %d lbufs in use list"
5798                             " to free list %d\n total %d\n",
5799                             k, rx_ring->lbuf_in_use_count,
5800                             rx_ring->lbuf_free_count,
5801                             rx_ring->lbuf_in_use_count +
5802                             rx_ring->lbuf_free_count);
5803 #endif
5804                         for (i = 0; i < rx_ring->lbuf_in_use_count; i++) {
5805                                 lbq_desc = rx_ring->lbuf_in_use[j];
5806                                 j++;
5807                                 if (j >= rx_ring->lbq_len) {
5808                                         j = 0;
5809                                 }
5810                                 if (lbq_desc->mp) {
5811                                         atomic_inc_32(&rx_ring->rx_indicate);
5812                                         freemsg(lbq_desc->mp);
5813                                 }
5814                         }
5815                         rx_ring->lbq_use_head = j;
5816                         rx_ring->lbq_use_tail = j;
5817                         rx_ring->lbuf_in_use_count = 0;
5818                         j = rx_ring->sbq_use_head;
5819 #ifdef QLGE_LOAD_UNLOAD
5820                         cmn_err(CE_NOTE, "ring %d: move %d sbufs in use list,"
5821                             " to free list %d\n total %d \n",
5822                             k, rx_ring->sbuf_in_use_count,
5823                             rx_ring->sbuf_free_count,
5824                             rx_ring->sbuf_in_use_count +
5825                             rx_ring->sbuf_free_count);
5826 #endif
5827                         for (i = 0; i < rx_ring->sbuf_in_use_count; i++) {
5828                                 sbq_desc = rx_ring->sbuf_in_use[j];
5829                                 j++;
5830                                 if (j >= rx_ring->sbq_len) {
5831                                         j = 0;
5832                                 }
5833                                 if (sbq_desc->mp) {
5834                                         atomic_inc_32(&rx_ring->rx_indicate);
5835                                         freemsg(sbq_desc->mp);
5836                                 }
5837                         }
5838                         rx_ring->sbq_use_head = j;
5839                         rx_ring->sbq_use_tail = j;
5840                         rx_ring->sbuf_in_use_count = 0;
5841                 }
5842         }
5843 
5844         qlge->mac_flags = QL_MAC_STOPPED;
5845 
5846         return (rc);
5847 }
5848 
5849 /*
5850  * Support
5851  */
5852 
5853 void
5854 ql_disable_isr(qlge_t *qlge)
5855 {
5856         /*
5857          * disable the hardware interrupt
5858          */
5859         ISP_DISABLE_GLOBAL_INTRS(qlge);
5860 
5861         qlge->flags &= ~INTERRUPTS_ENABLED;
5862 }
5863 
5864 
5865 
5866 /*
5867  * busy wait for 'usecs' microseconds.
5868  */
5869 void
5870 qlge_delay(clock_t usecs)
5871 {
5872         drv_usecwait(usecs);
5873 }
5874 
5875 /*
5876  * retrieve firmware details.
5877  */
5878 
5879 pci_cfg_t *
5880 ql_get_pci_config(qlge_t *qlge)
5881 {
5882         return (&(qlge->pci_cfg));
5883 }
5884 
5885 /*
5886  * Get current Link status
5887  */
5888 static uint32_t
5889 ql_get_link_state(qlge_t *qlge)
5890 {
5891         uint32_t bitToCheck = 0;
5892         uint32_t temp, linkState;
5893 
5894         if (qlge->func_number == qlge->fn0_net) {
5895                 bitToCheck = STS_PL0;
5896         } else {
5897                 bitToCheck = STS_PL1;
5898         }
5899         temp = ql_read_reg(qlge, REG_STATUS);
5900         QL_PRINT(DBG_GLD, ("%s(%d) chip status reg: 0x%x\n",
5901             __func__, qlge->instance, temp));
5902 
5903         if (temp & bitToCheck) {
5904                 linkState = LS_UP;
5905         } else {
5906                 linkState = LS_DOWN;
5907         }
5908         if (CFG_IST(qlge, CFG_CHIP_8100)) {
5909                 /* for Schultz, link Speed is fixed to 10G, full duplex */
5910                 qlge->speed  = SPEED_10G;
5911                 qlge->duplex = 1;
5912         }
5913         return (linkState);
5914 }
5915 /*
5916  * Get current link status and report to OS
5917  */
5918 static void
5919 ql_get_and_report_link_state(qlge_t *qlge)
5920 {
5921         uint32_t cur_link_state;
5922 
5923         /* Get current link state */
5924         cur_link_state = ql_get_link_state(qlge);
5925         /* if link state has changed */
5926         if (cur_link_state != qlge->port_link_state) {
5927 
5928                 qlge->port_link_state = cur_link_state;
5929 
5930                 if (qlge->port_link_state == LS_UP) {
5931                         QL_PRINT(DBG_GLD, ("%s(%d) Link UP !!\n",
5932                             __func__, qlge->instance));
5933                         /* If driver detects a carrier on */
5934                         CARRIER_ON(qlge);
5935                 } else {
5936                         QL_PRINT(DBG_GLD, ("%s(%d) Link down\n",
5937                             __func__, qlge->instance));
5938                         /* If driver detects a lack of carrier */
5939                         CARRIER_OFF(qlge);
5940                 }
5941         }
5942 }
5943 
5944 /*
5945  * timer callback function executed after timer expires
5946  */
5947 static void
5948 ql_timer(void* arg)
5949 {
5950         ql_get_and_report_link_state((qlge_t *)arg);
5951 }
5952 
5953 /*
5954  * stop the running timer if activated
5955  */
5956 static void
5957 ql_stop_timer(qlge_t *qlge)
5958 {
5959         timeout_id_t timer_id;
5960         /* Disable driver timer */
5961         if (qlge->ql_timer_timeout_id != NULL) {
5962                 timer_id = qlge->ql_timer_timeout_id;
5963                 qlge->ql_timer_timeout_id = NULL;
5964                 (void) untimeout(timer_id);
5965         }
5966 }
5967 
5968 /*
5969  * stop then restart timer
5970  */
5971 void
5972 ql_restart_timer(qlge_t *qlge)
5973 {
5974         ql_stop_timer(qlge);
5975         qlge->ql_timer_ticks = TICKS_PER_SEC / 4;
5976         qlge->ql_timer_timeout_id = timeout(ql_timer,
5977             (void *)qlge, qlge->ql_timer_ticks);
5978 }
5979 
5980 /* ************************************************************************* */
5981 /*
5982  *              Hardware K-Stats Data Structures and Subroutines
5983  */
5984 /* ************************************************************************* */
5985 static const ql_ksindex_t ql_kstats_hw[] = {
5986         /* PCI related hardware information */
5987         { 0, "Vendor Id"                        },
5988         { 1, "Device Id"                        },
5989         { 2, "Command"                          },
5990         { 3, "Status"                           },
5991         { 4, "Revision Id"                      },
5992         { 5, "Cache Line Size"                  },
5993         { 6, "Latency Timer"                    },
5994         { 7, "Header Type"                      },
5995         { 9, "I/O base addr"                    },
5996         { 10, "Control Reg Base addr low"       },
5997         { 11, "Control Reg Base addr high"      },
5998         { 12, "Doorbell Reg Base addr low"      },
5999         { 13, "Doorbell Reg Base addr high"     },
6000         { 14, "Subsystem Vendor Id"             },
6001         { 15, "Subsystem Device ID"             },
6002         { 16, "PCIe Device Control"             },
6003         { 17, "PCIe Link Status"                },
6004 
6005         { -1,   NULL                            },
6006 };
6007 
6008 /*
6009  * kstat update function for PCI registers
6010  */
6011 static int
6012 ql_kstats_get_pci_regs(kstat_t *ksp, int flag)
6013 {
6014         qlge_t *qlge;
6015         kstat_named_t *knp;
6016 
6017         if (flag != KSTAT_READ)
6018                 return (EACCES);
6019 
6020         qlge = ksp->ks_private;
6021         knp = ksp->ks_data;
6022         (knp++)->value.ui32 = qlge->pci_cfg.vendor_id;
6023         (knp++)->value.ui32 = qlge->pci_cfg.device_id;
6024         (knp++)->value.ui32 = qlge->pci_cfg.command;
6025         (knp++)->value.ui32 = qlge->pci_cfg.status;
6026         (knp++)->value.ui32 = qlge->pci_cfg.revision;
6027         (knp++)->value.ui32 = qlge->pci_cfg.cache_line_size;
6028         (knp++)->value.ui32 = qlge->pci_cfg.latency_timer;
6029         (knp++)->value.ui32 = qlge->pci_cfg.header_type;
6030         (knp++)->value.ui32 = qlge->pci_cfg.io_base_address;
6031         (knp++)->value.ui32 =
6032             qlge->pci_cfg.pci_cntl_reg_set_mem_base_address_lower;
6033         (knp++)->value.ui32 =
6034             qlge->pci_cfg.pci_cntl_reg_set_mem_base_address_upper;
6035         (knp++)->value.ui32 =
6036             qlge->pci_cfg.pci_doorbell_mem_base_address_lower;
6037         (knp++)->value.ui32 =
6038             qlge->pci_cfg.pci_doorbell_mem_base_address_upper;
6039         (knp++)->value.ui32 = qlge->pci_cfg.sub_vendor_id;
6040         (knp++)->value.ui32 = qlge->pci_cfg.sub_device_id;
6041         (knp++)->value.ui32 = qlge->pci_cfg.pcie_device_control;
6042         (knp++)->value.ui32 = qlge->pci_cfg.link_status;
6043 
6044         return (0);
6045 }
6046 
6047 static const ql_ksindex_t ql_kstats_mii[] = {
6048         /* MAC/MII related hardware information */
6049         { 0, "mtu"},
6050 
6051         { -1, NULL},
6052 };
6053 
6054 
6055 /*
6056  * kstat update function for MII related information.
6057  */
6058 static int
6059 ql_kstats_mii_update(kstat_t *ksp, int flag)
6060 {
6061         qlge_t *qlge;
6062         kstat_named_t *knp;
6063 
6064         if (flag != KSTAT_READ)
6065                 return (EACCES);
6066 
6067         qlge = ksp->ks_private;
6068         knp = ksp->ks_data;
6069 
6070         (knp++)->value.ui32 = qlge->mtu;
6071 
6072         return (0);
6073 }
6074 
6075 static const ql_ksindex_t ql_kstats_reg[] = {
6076         /* Register information */
6077         { 0, "System (0x08)"                    },
6078         { 1, "Reset/Fail Over(0x0Ch"            },
6079         { 2, "Function Specific Control(0x10)"  },
6080         { 3, "Status (0x30)"                    },
6081         { 4, "Intr Enable (0x34)"               },
6082         { 5, "Intr Status1 (0x3C)"              },
6083         { 6, "Error Status (0x54)"              },
6084         { 7, "XGMAC Flow Control(0x11C)"        },
6085         { 8, "XGMAC Tx Pause Frames(0x230)"     },
6086         { 9, "XGMAC Rx Pause Frames(0x388)"     },
6087         { 10, "XGMAC Rx FIFO Drop Count(0x5B8)" },
6088         { 11, "interrupts actually allocated"   },
6089         { 12, "interrupts on rx ring 0"         },
6090         { 13, "interrupts on rx ring 1"         },
6091         { 14, "interrupts on rx ring 2"         },
6092         { 15, "interrupts on rx ring 3"         },
6093         { 16, "interrupts on rx ring 4"         },
6094         { 17, "interrupts on rx ring 5"         },
6095         { 18, "interrupts on rx ring 6"         },
6096         { 19, "interrupts on rx ring 7"         },
6097         { 20, "polls on rx ring 0"              },
6098         { 21, "polls on rx ring 1"              },
6099         { 22, "polls on rx ring 2"              },
6100         { 23, "polls on rx ring 3"              },
6101         { 24, "polls on rx ring 4"              },
6102         { 25, "polls on rx ring 5"              },
6103         { 26, "polls on rx ring 6"              },
6104         { 27, "polls on rx ring 7"              },
6105         { 28, "tx no resource on ring 0"        },
6106         { 29, "tx dma bind fail on ring 0"      },
6107         { 30, "tx dma no handle on ring 0"      },
6108         { 31, "tx dma no cookie on ring 0"      },
6109         { 32, "MPI firmware major version"      },
6110         { 33, "MPI firmware minor version"      },
6111         { 34, "MPI firmware sub version"        },
6112         { 35, "rx no resource"                  },
6113 
6114         { -1, NULL},
6115 };
6116 
6117 
6118 /*
6119  * kstat update function for device register set
6120  */
6121 static int
6122 ql_kstats_get_reg_and_dev_stats(kstat_t *ksp, int flag)
6123 {
6124         qlge_t *qlge;
6125         kstat_named_t *knp;
6126         uint32_t val32;
6127         int i = 0;
6128         struct tx_ring *tx_ring;
6129         struct rx_ring *rx_ring;
6130 
6131         if (flag != KSTAT_READ)
6132                 return (EACCES);
6133 
6134         qlge = ksp->ks_private;
6135         knp = ksp->ks_data;
6136 
6137         (knp++)->value.ui32 = ql_read_reg(qlge, REG_SYSTEM);
6138         (knp++)->value.ui32 = ql_read_reg(qlge, REG_RESET_FAILOVER);
6139         (knp++)->value.ui32 = ql_read_reg(qlge, REG_FUNCTION_SPECIFIC_CONTROL);
6140         (knp++)->value.ui32 = ql_read_reg(qlge, REG_STATUS);
6141         (knp++)->value.ui32 = ql_read_reg(qlge, REG_INTERRUPT_ENABLE);
6142         (knp++)->value.ui32 = ql_read_reg(qlge, REG_INTERRUPT_STATUS_1);
6143         (knp++)->value.ui32 = ql_read_reg(qlge, REG_ERROR_STATUS);
6144 
6145         if (ql_sem_spinlock(qlge, qlge->xgmac_sem_mask)) {
6146                 return (0);
6147         }
6148         (void) ql_read_xgmac_reg(qlge, REG_XGMAC_FLOW_CONTROL, &val32);
6149         (knp++)->value.ui32 = val32;
6150 
6151         (void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_TX_PAUSE_PKTS, &val32);
6152         (knp++)->value.ui32 = val32;
6153 
6154         (void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_RX_PAUSE_PKTS, &val32);
6155         (knp++)->value.ui32 = val32;
6156 
6157         (void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_RX_FIFO_DROPS, &val32);
6158         (knp++)->value.ui32 = val32;
6159 
6160         ql_sem_unlock(qlge, qlge->xgmac_sem_mask);
6161 
6162         (knp++)->value.ui32 = qlge->intr_cnt;
6163 
6164         for (i = 0; i < 8; i++) {
6165                 (knp++)->value.ui32 = qlge->rx_interrupts[i];
6166         }
6167 
6168         for (i = 0; i < 8; i++) {
6169                 (knp++)->value.ui32 = qlge->rx_polls[i];
6170         }
6171 
6172         tx_ring = &qlge->tx_ring[0];
6173         (knp++)->value.ui32 = tx_ring->defer;
6174         (knp++)->value.ui32 = tx_ring->tx_fail_dma_bind;
6175         (knp++)->value.ui32 = tx_ring->tx_no_dma_handle;
6176         (knp++)->value.ui32 = tx_ring->tx_no_dma_cookie;
6177 
6178         (knp++)->value.ui32 = qlge->fw_version_info.major_version;
6179         (knp++)->value.ui32 = qlge->fw_version_info.minor_version;
6180         (knp++)->value.ui32 = qlge->fw_version_info.sub_minor_version;
6181 
6182         for (i = 0; i < qlge->rx_ring_count; i++) {
6183                 rx_ring = &qlge->rx_ring[i];
6184                 val32 += rx_ring->rx_packets_dropped_no_buffer;
6185         }
6186         (knp++)->value.ui32 = val32;
6187 
6188         return (0);
6189 }
6190 
6191 
6192 static kstat_t *
6193 ql_setup_named_kstat(qlge_t *qlge, int instance, char *name,
6194     const ql_ksindex_t *ksip, size_t size, int (*update)(kstat_t *, int))
6195 {
6196         kstat_t *ksp;
6197         kstat_named_t *knp;
6198         char *np;
6199         int type;
6200 
6201         size /= sizeof (ql_ksindex_t);
6202         ksp = kstat_create(ADAPTER_NAME, instance, name, "net",
6203             KSTAT_TYPE_NAMED, ((uint32_t)size) - 1, KSTAT_FLAG_PERSISTENT);
6204         if (ksp == NULL)
6205                 return (NULL);
6206 
6207         ksp->ks_private = qlge;
6208         ksp->ks_update = update;
6209         for (knp = ksp->ks_data; (np = ksip->name) != NULL; ++knp, ++ksip) {
6210                 switch (*np) {
6211                 default:
6212                         type = KSTAT_DATA_UINT32;
6213                         break;
6214                 case '&':
6215                         np += 1;
6216                         type = KSTAT_DATA_CHAR;
6217                         break;
6218                 }
6219                 kstat_named_init(knp, np, (uint8_t)type);
6220         }
6221         kstat_install(ksp);
6222 
6223         return (ksp);
6224 }
6225 
6226 /*
6227  * Setup various kstat
6228  */
6229 int
6230 ql_init_kstats(qlge_t *qlge)
6231 {
6232         /* Hardware KStats */
6233         qlge->ql_kstats[QL_KSTAT_CHIP] = ql_setup_named_kstat(qlge,
6234             qlge->instance, "chip", ql_kstats_hw,
6235             sizeof (ql_kstats_hw), ql_kstats_get_pci_regs);
6236         if (qlge->ql_kstats[QL_KSTAT_CHIP] == NULL) {
6237                 return (DDI_FAILURE);
6238         }
6239 
6240         /* MII KStats */
6241         qlge->ql_kstats[QL_KSTAT_LINK] = ql_setup_named_kstat(qlge,
6242             qlge->instance, "mii", ql_kstats_mii,
6243             sizeof (ql_kstats_mii), ql_kstats_mii_update);
6244         if (qlge->ql_kstats[QL_KSTAT_LINK] == NULL) {
6245                 return (DDI_FAILURE);
6246         }
6247 
6248         /* REG KStats */
6249         qlge->ql_kstats[QL_KSTAT_REG] = ql_setup_named_kstat(qlge,
6250             qlge->instance, "reg", ql_kstats_reg,
6251             sizeof (ql_kstats_reg), ql_kstats_get_reg_and_dev_stats);
6252         if (qlge->ql_kstats[QL_KSTAT_REG] == NULL) {
6253                 return (DDI_FAILURE);
6254         }
6255         return (DDI_SUCCESS);
6256 }
6257 
6258 /*
6259  * delete all kstat
6260  */
6261 void
6262 ql_fini_kstats(qlge_t *qlge)
6263 {
6264         int i;
6265 
6266         for (i = 0; i < QL_KSTAT_COUNT; i++) {
6267                 if (qlge->ql_kstats[i] != NULL)
6268                         kstat_delete(qlge->ql_kstats[i]);
6269         }
6270 }
6271 
6272 /* ************************************************************************* */
6273 /*
6274  *                                 kstat end
6275  */
6276 /* ************************************************************************* */
6277 
6278 /*
6279  * Setup the parameters for receive and transmit rings including buffer sizes
6280  * and completion queue sizes
6281  */
6282 static int
6283 ql_setup_rings(qlge_t *qlge)
6284 {
6285         uint8_t i;
6286         struct rx_ring *rx_ring;
6287         struct tx_ring *tx_ring;
6288         uint16_t lbq_buf_size;
6289 
6290         lbq_buf_size = (uint16_t)
6291             ((qlge->mtu == ETHERMTU)? LRG_BUF_NORMAL_SIZE : LRG_BUF_JUMBO_SIZE);
6292 
6293         /*
6294          * rx_ring[0] is always the default queue.
6295          */
6296         /*
6297          * qlge->rx_ring_count:
6298          * Total number of rx_rings. This includes a number
6299          * of outbound completion handler rx_rings, and a
6300          * number of inbound completion handler rx_rings.
6301          * rss is only enabled if we have more than 1 rx completion
6302          * queue. If we have a single rx completion queue
6303          * then all rx completions go to this queue and
6304          * the last completion queue
6305          */
6306 
6307         qlge->tx_ring_first_cq_id = qlge->rss_ring_count;
6308 
6309         for (i = 0; i < qlge->tx_ring_count; i++) {
6310                 tx_ring = &qlge->tx_ring[i];
6311                 bzero((void *)tx_ring, sizeof (*tx_ring));
6312                 tx_ring->qlge = qlge;
6313                 tx_ring->wq_id = i;
6314                 tx_ring->wq_len = qlge->tx_ring_size;
6315                 tx_ring->wq_size = (uint32_t)(
6316                     tx_ring->wq_len * sizeof (struct ob_mac_iocb_req));
6317 
6318                 /*
6319                  * The completion queue ID for the tx rings start
6320                  * immediately after the last rss completion queue.
6321                  */
6322                 tx_ring->cq_id = (uint16_t)(i + qlge->tx_ring_first_cq_id);
6323         }
6324 
6325         for (i = 0; i < qlge->rx_ring_count; i++) {
6326                 rx_ring = &qlge->rx_ring[i];
6327                 bzero((void *)rx_ring, sizeof (*rx_ring));
6328                 rx_ring->qlge = qlge;
6329                 rx_ring->cq_id = i;
6330                 if (i != 0)
6331                         rx_ring->cpu = (i) % qlge->rx_ring_count;
6332                 else
6333                         rx_ring->cpu = 0;
6334 
6335                 if (i < qlge->rss_ring_count) {
6336                         /*
6337                          * Inbound completions (RSS) queues
6338                          * Default queue is queue 0 which handles
6339                          * unicast plus bcast/mcast and async events.
6340                          * Other inbound queues handle unicast frames only.
6341                          */
6342                         rx_ring->cq_len = qlge->rx_ring_size;
6343                         rx_ring->cq_size = (uint32_t)
6344                             (rx_ring->cq_len * sizeof (struct net_rsp_iocb));
6345                         rx_ring->lbq_len = NUM_LARGE_BUFFERS;
6346                         rx_ring->lbq_size = (uint32_t)
6347                             (rx_ring->lbq_len * sizeof (uint64_t));
6348                         rx_ring->lbq_buf_size = lbq_buf_size;
6349                         rx_ring->sbq_len = NUM_SMALL_BUFFERS;
6350                         rx_ring->sbq_size = (uint32_t)
6351                             (rx_ring->sbq_len * sizeof (uint64_t));
6352                         rx_ring->sbq_buf_size = SMALL_BUFFER_SIZE * 2;
6353                         rx_ring->type = RX_Q;
6354 
6355                         QL_PRINT(DBG_GLD,
6356                             ("%s(%d)Allocating rss completion queue %d "
6357                             "on cpu %d\n", __func__, qlge->instance,
6358                             rx_ring->cq_id, rx_ring->cpu));
6359                 } else {
6360                         /*
6361                          * Outbound queue handles outbound completions only
6362                          */
6363                         /* outbound cq is same size as tx_ring it services. */
6364                         QL_PRINT(DBG_INIT, ("rx_ring 0x%p i %d\n", rx_ring, i));
6365                         rx_ring->cq_len = qlge->tx_ring_size;
6366                         rx_ring->cq_size = (uint32_t)
6367                             (rx_ring->cq_len * sizeof (struct net_rsp_iocb));
6368                         rx_ring->lbq_len = 0;
6369                         rx_ring->lbq_size = 0;
6370                         rx_ring->lbq_buf_size = 0;
6371                         rx_ring->sbq_len = 0;
6372                         rx_ring->sbq_size = 0;
6373                         rx_ring->sbq_buf_size = 0;
6374                         rx_ring->type = TX_Q;
6375 
6376                         QL_PRINT(DBG_GLD,
6377                             ("%s(%d)Allocating TX completion queue %d on"
6378                             " cpu %d\n", __func__, qlge->instance,
6379                             rx_ring->cq_id, rx_ring->cpu));
6380                 }
6381         }
6382 
6383         return (DDI_SUCCESS);
6384 }
6385 
6386 static int
6387 ql_start_rx_ring(qlge_t *qlge, struct rx_ring *rx_ring)
6388 {
6389         struct cqicb_t *cqicb = (struct cqicb_t *)rx_ring->cqicb_dma.vaddr;
6390         void *shadow_reg = (uint8_t *)qlge->host_copy_shadow_dma_attr.vaddr +
6391             (rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE)
6392         /* first shadow area is used by wqicb's host copy of consumer index */
6393             + sizeof (uint64_t);
6394         uint64_t shadow_reg_dma = qlge->host_copy_shadow_dma_attr.dma_addr +
6395             (rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE)
6396             + sizeof (uint64_t);
6397         /* lrg/sml bufq pointers */
6398         uint8_t *buf_q_base_reg =
6399             (uint8_t *)qlge->buf_q_ptr_base_addr_dma_attr.vaddr +
6400             (rx_ring->cq_id * sizeof (uint64_t) * BUF_Q_PTR_SPACE);
6401         uint64_t buf_q_base_reg_dma =
6402             qlge->buf_q_ptr_base_addr_dma_attr.dma_addr +
6403             (rx_ring->cq_id * sizeof (uint64_t) * BUF_Q_PTR_SPACE);
6404         caddr_t doorbell_area =
6405             qlge->doorbell_reg_iobase + (VM_PAGE_SIZE * (128 + rx_ring->cq_id));
6406         int err = 0;
6407         uint16_t bq_len;
6408         uint64_t tmp;
6409         uint64_t *base_indirect_ptr;
6410         int page_entries;
6411 
6412         /* Set up the shadow registers for this ring. */
6413         rx_ring->prod_idx_sh_reg = shadow_reg;
6414         rx_ring->prod_idx_sh_reg_dma = shadow_reg_dma;
6415         rx_ring->prod_idx_sh_reg_offset = (off_t)(((rx_ring->cq_id *
6416             sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE) + sizeof (uint64_t)));
6417 
6418         rx_ring->lbq_base_indirect = (uint64_t *)(void *)buf_q_base_reg;
6419         rx_ring->lbq_base_indirect_dma = buf_q_base_reg_dma;
6420 
6421         QL_PRINT(DBG_INIT, ("%s rx ring(%d): prod_idx virtual addr = 0x%lx,"
6422             " phys_addr 0x%lx\n", __func__, rx_ring->cq_id,
6423             rx_ring->prod_idx_sh_reg, rx_ring->prod_idx_sh_reg_dma));
6424 
6425         buf_q_base_reg += ((BUF_Q_PTR_SPACE / 2) * sizeof (uint64_t));
6426         buf_q_base_reg_dma += ((BUF_Q_PTR_SPACE / 2) * sizeof (uint64_t));
6427         rx_ring->sbq_base_indirect = (uint64_t *)(void *)buf_q_base_reg;
6428         rx_ring->sbq_base_indirect_dma = buf_q_base_reg_dma;
6429 
6430         /* PCI doorbell mem area + 0x00 for consumer index register */
6431         rx_ring->cnsmr_idx_db_reg = (uint32_t *)(void *)doorbell_area;
6432         rx_ring->cnsmr_idx = 0;
6433         *rx_ring->prod_idx_sh_reg = 0;
6434         rx_ring->curr_entry = rx_ring->cq_dma.vaddr;
6435 
6436         /* PCI doorbell mem area + 0x04 for valid register */
6437         rx_ring->valid_db_reg = (uint32_t *)(void *)
6438             ((uint8_t *)(void *)doorbell_area + 0x04);
6439 
6440         /* PCI doorbell mem area + 0x18 for large buffer consumer */
6441         rx_ring->lbq_prod_idx_db_reg = (uint32_t *)(void *)
6442             ((uint8_t *)(void *)doorbell_area + 0x18);
6443 
6444         /* PCI doorbell mem area + 0x1c */
6445         rx_ring->sbq_prod_idx_db_reg = (uint32_t *)(void *)
6446             ((uint8_t *)(void *)doorbell_area + 0x1c);
6447 
6448         bzero((void *)cqicb, sizeof (*cqicb));
6449 
6450         cqicb->msix_vect = (uint8_t)rx_ring->irq;
6451 
6452         bq_len = (uint16_t)((rx_ring->cq_len == 65536) ?
6453             (uint16_t)0 : (uint16_t)rx_ring->cq_len);
6454         cqicb->len = (uint16_t)cpu_to_le16(bq_len | LEN_V | LEN_CPP_CONT);
6455 
6456         cqicb->cq_base_addr_lo =
6457             cpu_to_le32(LS_64BITS(rx_ring->cq_dma.dma_addr));
6458         cqicb->cq_base_addr_hi =
6459             cpu_to_le32(MS_64BITS(rx_ring->cq_dma.dma_addr));
6460 
6461         cqicb->prod_idx_addr_lo =
6462             cpu_to_le32(LS_64BITS(rx_ring->prod_idx_sh_reg_dma));
6463         cqicb->prod_idx_addr_hi =
6464             cpu_to_le32(MS_64BITS(rx_ring->prod_idx_sh_reg_dma));
6465 
6466         /*
6467          * Set up the control block load flags.
6468          */
6469         cqicb->flags = FLAGS_LC | /* Load queue base address */
6470             FLAGS_LV | /* Load MSI-X vector */
6471             FLAGS_LI;  /* Load irq delay values */
6472         if (rx_ring->lbq_len) {
6473                 /* Load lbq values */
6474                 cqicb->flags = (uint8_t)(cqicb->flags | FLAGS_LL);
6475                 tmp = (uint64_t)rx_ring->lbq_dma.dma_addr;
6476                 base_indirect_ptr = (uint64_t *)rx_ring->lbq_base_indirect;
6477                 page_entries = 0;
6478                 do {
6479                         *base_indirect_ptr = cpu_to_le64(tmp);
6480                         tmp += VM_PAGE_SIZE;
6481                         base_indirect_ptr++;
6482                         page_entries++;
6483                 } while (page_entries < (int)(
6484                     ((rx_ring->lbq_len * sizeof (uint64_t)) / VM_PAGE_SIZE)));
6485 
6486                 cqicb->lbq_addr_lo =
6487                     cpu_to_le32(LS_64BITS(rx_ring->lbq_base_indirect_dma));
6488                 cqicb->lbq_addr_hi =
6489                     cpu_to_le32(MS_64BITS(rx_ring->lbq_base_indirect_dma));
6490                 bq_len = (uint16_t)((rx_ring->lbq_buf_size == 65536) ?
6491                     (uint16_t)0 : (uint16_t)rx_ring->lbq_buf_size);
6492                 cqicb->lbq_buf_size = (uint16_t)cpu_to_le16(bq_len);
6493                 bq_len = (uint16_t)((rx_ring->lbq_len == 65536) ? (uint16_t)0 :
6494                     (uint16_t)rx_ring->lbq_len);
6495                 cqicb->lbq_len = (uint16_t)cpu_to_le16(bq_len);
6496                 rx_ring->lbq_prod_idx = 0;
6497                 rx_ring->lbq_curr_idx = 0;
6498         }
6499         if (rx_ring->sbq_len) {
6500                 /* Load sbq values */
6501                 cqicb->flags = (uint8_t)(cqicb->flags | FLAGS_LS);
6502                 tmp = (uint64_t)rx_ring->sbq_dma.dma_addr;
6503                 base_indirect_ptr = (uint64_t *)rx_ring->sbq_base_indirect;
6504                 page_entries = 0;
6505 
6506                 do {
6507                         *base_indirect_ptr = cpu_to_le64(tmp);
6508                         tmp += VM_PAGE_SIZE;
6509                         base_indirect_ptr++;
6510                         page_entries++;
6511                 } while (page_entries < (uint32_t)
6512                     (((rx_ring->sbq_len * sizeof (uint64_t)) / VM_PAGE_SIZE)));
6513 
6514                 cqicb->sbq_addr_lo =
6515                     cpu_to_le32(LS_64BITS(rx_ring->sbq_base_indirect_dma));
6516                 cqicb->sbq_addr_hi =
6517                     cpu_to_le32(MS_64BITS(rx_ring->sbq_base_indirect_dma));
6518                 cqicb->sbq_buf_size = (uint16_t)
6519                     cpu_to_le16((uint16_t)(rx_ring->sbq_buf_size/2));
6520                 bq_len = (uint16_t)((rx_ring->sbq_len == 65536) ?
6521                     (uint16_t)0 : (uint16_t)rx_ring->sbq_len);
6522                 cqicb->sbq_len = (uint16_t)cpu_to_le16(bq_len);
6523                 rx_ring->sbq_prod_idx = 0;
6524                 rx_ring->sbq_curr_idx = 0;
6525         }
6526         switch (rx_ring->type) {
6527         case TX_Q:
6528                 cqicb->irq_delay = (uint16_t)
6529                     cpu_to_le16(qlge->tx_coalesce_usecs);
6530                 cqicb->pkt_delay = (uint16_t)
6531                     cpu_to_le16(qlge->tx_max_coalesced_frames);
6532                 break;
6533 
6534         case DEFAULT_Q:
6535                 cqicb->irq_delay = (uint16_t)
6536                     cpu_to_le16(qlge->rx_coalesce_usecs);
6537                 cqicb->pkt_delay = (uint16_t)
6538                     cpu_to_le16(qlge->rx_max_coalesced_frames);
6539                 break;
6540 
6541         case RX_Q:
6542                 /*
6543                  * Inbound completion handling rx_rings run in
6544                  * separate NAPI contexts.
6545                  */
6546                 cqicb->irq_delay = (uint16_t)
6547                     cpu_to_le16(qlge->rx_coalesce_usecs);
6548                 cqicb->pkt_delay = (uint16_t)
6549                     cpu_to_le16(qlge->rx_max_coalesced_frames);
6550                 break;
6551         default:
6552                 cmn_err(CE_WARN, "Invalid rx_ring->type = %d.",
6553                     rx_ring->type);
6554         }
6555         QL_PRINT(DBG_INIT, ("Initializing rx completion queue %d.\n",
6556             rx_ring->cq_id));
6557         /* QL_DUMP_CQICB(qlge, cqicb); */
6558         err = ql_write_cfg(qlge, CFG_LCQ, rx_ring->cqicb_dma.dma_addr,
6559             rx_ring->cq_id);
6560         if (err) {
6561                 cmn_err(CE_WARN, "Failed to load CQICB.");
6562                 return (err);
6563         }
6564 
6565         rx_ring->rx_packets_dropped_no_buffer = 0;
6566         rx_ring->rx_pkt_dropped_mac_unenabled = 0;
6567         rx_ring->rx_failed_sbq_allocs = 0;
6568         rx_ring->rx_failed_lbq_allocs = 0;
6569         rx_ring->rx_packets = 0;
6570         rx_ring->rx_bytes = 0;
6571         rx_ring->frame_too_long = 0;
6572         rx_ring->frame_too_short = 0;
6573         rx_ring->fcs_err = 0;
6574 
6575         return (err);
6576 }
6577 
6578 /*
6579  * start RSS
6580  */
6581 static int
6582 ql_start_rss(qlge_t *qlge)
6583 {
6584         struct ricb *ricb = (struct ricb *)qlge->ricb_dma.vaddr;
6585         int status = 0;
6586         int i;
6587         uint8_t *hash_id = (uint8_t *)ricb->hash_cq_id;
6588 
6589         bzero((void *)ricb, sizeof (*ricb));
6590 
6591         ricb->base_cq = RSS_L4K;
6592         ricb->flags =
6593             (RSS_L6K | RSS_LI | RSS_LB | RSS_LM | RSS_RI4 | RSS_RI6 | RSS_RT4 |
6594             RSS_RT6);
6595         ricb->mask = (uint16_t)cpu_to_le16(RSS_HASH_CQ_ID_MAX - 1);
6596 
6597         /*
6598          * Fill out the Indirection Table.
6599          */
6600         for (i = 0; i < RSS_HASH_CQ_ID_MAX; i++)
6601                 hash_id[i] = (uint8_t)(i & (qlge->rss_ring_count - 1));
6602 
6603         (void) memcpy(&ricb->ipv6_hash_key[0], key_data, 40);
6604         (void) memcpy(&ricb->ipv4_hash_key[0], key_data, 16);
6605 
6606         QL_PRINT(DBG_INIT, ("Initializing RSS.\n"));
6607 
6608         status = ql_write_cfg(qlge, CFG_LR, qlge->ricb_dma.dma_addr, 0);
6609         if (status) {
6610                 cmn_err(CE_WARN, "Failed to load RICB.");
6611                 return (status);
6612         }
6613 
6614         return (status);
6615 }
6616 
6617 /*
6618  * load a tx ring control block to hw and start this ring
6619  */
6620 static int
6621 ql_start_tx_ring(qlge_t *qlge, struct tx_ring *tx_ring)
6622 {
6623         struct wqicb_t *wqicb = (struct wqicb_t *)tx_ring->wqicb_dma.vaddr;
6624         caddr_t doorbell_area =
6625             qlge->doorbell_reg_iobase + (VM_PAGE_SIZE * tx_ring->wq_id);
6626         void *shadow_reg = (uint8_t *)qlge->host_copy_shadow_dma_attr.vaddr +
6627             (tx_ring->wq_id * sizeof (uint64_t)) * RX_TX_RING_SHADOW_SPACE;
6628         uint64_t shadow_reg_dma = qlge->host_copy_shadow_dma_attr.dma_addr +
6629             (tx_ring->wq_id * sizeof (uint64_t)) * RX_TX_RING_SHADOW_SPACE;
6630         int err = 0;
6631 
6632         /*
6633          * Assign doorbell registers for this tx_ring.
6634          */
6635 
6636         /* TX PCI doorbell mem area for tx producer index */
6637         tx_ring->prod_idx_db_reg = (uint32_t *)(void *)doorbell_area;
6638         tx_ring->prod_idx = 0;
6639         /* TX PCI doorbell mem area + 0x04 */
6640         tx_ring->valid_db_reg = (uint32_t *)(void *)
6641             ((uint8_t *)(void *)doorbell_area + 0x04);
6642 
6643         /*
6644          * Assign shadow registers for this tx_ring.
6645          */
6646         tx_ring->cnsmr_idx_sh_reg = shadow_reg;
6647         tx_ring->cnsmr_idx_sh_reg_dma = shadow_reg_dma;
6648         *tx_ring->cnsmr_idx_sh_reg = 0;
6649 
6650         QL_PRINT(DBG_INIT, ("%s tx ring(%d): cnsmr_idx virtual addr = 0x%lx,"
6651             " phys_addr 0x%lx\n",
6652             __func__, tx_ring->wq_id, tx_ring->cnsmr_idx_sh_reg,
6653             tx_ring->cnsmr_idx_sh_reg_dma));
6654 
6655         wqicb->len =
6656             (uint16_t)cpu_to_le16(tx_ring->wq_len | Q_LEN_V | Q_LEN_CPP_CONT);
6657         wqicb->flags = cpu_to_le16(Q_FLAGS_LC |
6658             Q_FLAGS_LB | Q_FLAGS_LI | Q_FLAGS_LO);
6659         wqicb->cq_id_rss = (uint16_t)cpu_to_le16(tx_ring->cq_id);
6660         wqicb->rid = 0;
6661         wqicb->wq_addr_lo = cpu_to_le32(LS_64BITS(tx_ring->wq_dma.dma_addr));
6662         wqicb->wq_addr_hi = cpu_to_le32(MS_64BITS(tx_ring->wq_dma.dma_addr));
6663         wqicb->cnsmr_idx_addr_lo =
6664             cpu_to_le32(LS_64BITS(tx_ring->cnsmr_idx_sh_reg_dma));
6665         wqicb->cnsmr_idx_addr_hi =
6666             cpu_to_le32(MS_64BITS(tx_ring->cnsmr_idx_sh_reg_dma));
6667 
6668         ql_init_tx_ring(tx_ring);
6669         /* QL_DUMP_WQICB(qlge, wqicb); */
6670         err = ql_write_cfg(qlge, CFG_LRQ, tx_ring->wqicb_dma.dma_addr,
6671             tx_ring->wq_id);
6672 
6673         if (err) {
6674                 cmn_err(CE_WARN, "Failed to load WQICB.");
6675                 return (err);
6676         }
6677         return (err);
6678 }
6679 
6680 /*
6681  * Set up a MAC, multicast or VLAN address for the
6682  * inbound frame matching.
6683  */
6684 int
6685 ql_set_mac_addr_reg(qlge_t *qlge, uint8_t *addr, uint32_t type,
6686     uint16_t index)
6687 {
6688         uint32_t offset = 0;
6689         int status = DDI_SUCCESS;
6690 
6691         switch (type) {
6692         case MAC_ADDR_TYPE_MULTI_MAC:
6693         case MAC_ADDR_TYPE_CAM_MAC: {
6694                 uint32_t cam_output;
6695                 uint32_t upper = (addr[0] << 8) | addr[1];
6696                 uint32_t lower =
6697                     (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
6698                     (addr[5]);
6699 
6700                 QL_PRINT(DBG_INIT, ("Adding %s ", (type ==
6701                     MAC_ADDR_TYPE_MULTI_MAC) ?
6702                     "MULTICAST" : "UNICAST"));
6703                 QL_PRINT(DBG_INIT,
6704                     ("addr %02x %02x %02x %02x %02x %02x at index %d in "
6705                     "the CAM.\n",
6706                     addr[0], addr[1], addr[2], addr[3], addr[4],
6707                     addr[5], index));
6708 
6709                 status = ql_wait_reg_rdy(qlge,
6710                     REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0);
6711                 if (status)
6712                         goto exit;
6713                 /* offset 0 - lower 32 bits of the MAC address */
6714                 ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
6715                     (offset++) |
6716                     (index << MAC_ADDR_IDX_SHIFT) | /* index */
6717                     type);      /* type */
6718                 ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, lower);
6719                 status = ql_wait_reg_rdy(qlge,
6720                     REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0);
6721                 if (status)
6722                         goto exit;
6723                 /* offset 1 - upper 16 bits of the MAC address */
6724                 ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
6725                     (offset++) |
6726                     (index << MAC_ADDR_IDX_SHIFT) | /* index */
6727                     type);      /* type */
6728                 ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, upper);
6729                 status = ql_wait_reg_rdy(qlge,
6730                     REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0);
6731                 if (status)
6732                         goto exit;
6733                 /* offset 2 - CQ ID associated with this MAC address */
6734                 ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
6735                     (offset) | (index << MAC_ADDR_IDX_SHIFT) |    /* index */
6736                     type);      /* type */
6737                 /*
6738                  * This field should also include the queue id
6739                  * and possibly the function id.  Right now we hardcode
6740                  * the route field to NIC core.
6741                  */
6742                 if (type == MAC_ADDR_TYPE_CAM_MAC) {
6743                         cam_output = (CAM_OUT_ROUTE_NIC |
6744                             (qlge->func_number << CAM_OUT_FUNC_SHIFT) |
6745                             (0 <<
6746                             CAM_OUT_CQ_ID_SHIFT));
6747 
6748                         /* route to NIC core */
6749                         ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA,
6750                             cam_output);
6751                         }
6752                 break;
6753                 }
6754         default:
6755                 cmn_err(CE_WARN,
6756                     "Address type %d not yet supported.", type);
6757                 status = DDI_FAILURE;
6758         }
6759 exit:
6760         return (status);
6761 }
6762 
6763 /*
6764  * The NIC function for this chip has 16 routing indexes.  Each one can be used
6765  * to route different frame types to various inbound queues.  We send broadcast
6766  * multicast/error frames to the default queue for slow handling,
6767  * and CAM hit/RSS frames to the fast handling queues.
6768  */
6769 static int
6770 ql_set_routing_reg(qlge_t *qlge, uint32_t index, uint32_t mask, int enable)
6771 {
6772         int status;
6773         uint32_t value = 0;
6774 
6775         QL_PRINT(DBG_INIT,
6776             ("%s %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s mask %s the routing reg.\n",
6777             (enable ? "Adding" : "Removing"),
6778             ((index == RT_IDX_ALL_ERR_SLOT) ? "MAC ERROR/ALL ERROR" : ""),
6779             ((index == RT_IDX_IP_CSUM_ERR_SLOT) ? "IP CSUM ERROR" : ""),
6780             ((index ==
6781             RT_IDX_TCP_UDP_CSUM_ERR_SLOT) ? "TCP/UDP CSUM ERROR" : ""),
6782             ((index == RT_IDX_BCAST_SLOT) ? "BROADCAST" : ""),
6783             ((index == RT_IDX_MCAST_MATCH_SLOT) ? "MULTICAST MATCH" : ""),
6784             ((index == RT_IDX_ALLMULTI_SLOT) ? "ALL MULTICAST MATCH" : ""),
6785             ((index == RT_IDX_UNUSED6_SLOT) ? "UNUSED6" : ""),
6786             ((index == RT_IDX_UNUSED7_SLOT) ? "UNUSED7" : ""),
6787             ((index == RT_IDX_RSS_MATCH_SLOT) ? "RSS ALL/IPV4 MATCH" : ""),
6788             ((index == RT_IDX_RSS_IPV6_SLOT) ? "RSS IPV6" : ""),
6789             ((index == RT_IDX_RSS_TCP4_SLOT) ? "RSS TCP4" : ""),
6790             ((index == RT_IDX_RSS_TCP6_SLOT) ? "RSS TCP6" : ""),
6791             ((index == RT_IDX_CAM_HIT_SLOT) ? "CAM HIT" : ""),
6792             ((index == RT_IDX_UNUSED013) ? "UNUSED13" : ""),
6793             ((index == RT_IDX_UNUSED014) ? "UNUSED14" : ""),
6794             ((index == RT_IDX_PROMISCUOUS_SLOT) ? "PROMISCUOUS" : ""),
6795             (enable ? "to" : "from")));
6796 
6797         switch (mask) {
6798         case RT_IDX_CAM_HIT:
6799                 value = RT_IDX_DST_CAM_Q | /* dest */
6800                     RT_IDX_TYPE_NICQ | /* type */
6801                     (RT_IDX_CAM_HIT_SLOT << RT_IDX_IDX_SHIFT); /* index */
6802                 break;
6803 
6804         case RT_IDX_VALID: /* Promiscuous Mode frames. */
6805                 value = RT_IDX_DST_DFLT_Q |     /* dest */
6806                     RT_IDX_TYPE_NICQ |  /* type */
6807                     (RT_IDX_PROMISCUOUS_SLOT << RT_IDX_IDX_SHIFT); /* index */
6808                 break;
6809 
6810         case RT_IDX_ERR:        /* Pass up MAC,IP,TCP/UDP error frames. */
6811                 value = RT_IDX_DST_DFLT_Q |     /* dest */
6812                     RT_IDX_TYPE_NICQ |  /* type */
6813                     (RT_IDX_ALL_ERR_SLOT << RT_IDX_IDX_SHIFT); /* index */
6814                 break;
6815 
6816         case RT_IDX_BCAST:      /* Pass up Broadcast frames to default Q. */
6817                 value = RT_IDX_DST_DFLT_Q |     /* dest */
6818                     RT_IDX_TYPE_NICQ |  /* type */
6819                     (RT_IDX_BCAST_SLOT << RT_IDX_IDX_SHIFT); /* index */
6820                 break;
6821 
6822         case RT_IDX_MCAST:      /* Pass up All Multicast frames. */
6823                 value = RT_IDX_DST_CAM_Q |      /* dest */
6824                     RT_IDX_TYPE_NICQ |  /* type */
6825                     (RT_IDX_ALLMULTI_SLOT << RT_IDX_IDX_SHIFT); /* index */
6826                 break;
6827 
6828         case RT_IDX_MCAST_MATCH:        /* Pass up matched Multicast frames. */
6829                 value = RT_IDX_DST_CAM_Q |      /* dest */
6830                     RT_IDX_TYPE_NICQ |  /* type */
6831                     (RT_IDX_MCAST_MATCH_SLOT << RT_IDX_IDX_SHIFT); /* index */
6832                 break;
6833 
6834         case RT_IDX_RSS_MATCH:  /* Pass up matched RSS frames. */
6835                 value = RT_IDX_DST_RSS |        /* dest */
6836                     RT_IDX_TYPE_NICQ |  /* type */
6837                     (RT_IDX_RSS_MATCH_SLOT << RT_IDX_IDX_SHIFT); /* index */
6838                 break;
6839 
6840         case 0: /* Clear the E-bit on an entry. */
6841                 value = RT_IDX_DST_DFLT_Q |     /* dest */
6842                     RT_IDX_TYPE_NICQ |  /* type */
6843                     (index << RT_IDX_IDX_SHIFT); /* index */
6844                 break;
6845 
6846         default:
6847                 cmn_err(CE_WARN, "Mask type %d not yet supported.",
6848                     mask);
6849                 status = -EPERM;
6850                 goto exit;
6851         }
6852 
6853         if (value != 0) {
6854                 status = ql_wait_reg_rdy(qlge, REG_ROUTING_INDEX, RT_IDX_MW, 0);
6855                 if (status)
6856                         goto exit;
6857                 value |= (enable ? RT_IDX_E : 0);
6858                 ql_write_reg(qlge, REG_ROUTING_INDEX, value);
6859                 ql_write_reg(qlge, REG_ROUTING_DATA, enable ? mask : 0);
6860         }
6861 
6862 exit:
6863         return (status);
6864 }
6865 
6866 /*
6867  * Clear all the entries in the routing table.
6868  * Caller must get semaphore in advance.
6869  */
6870 
6871 static int
6872 ql_stop_routing(qlge_t *qlge)
6873 {
6874         int status = 0;
6875         int i;
6876         /* Clear all the entries in the routing table. */
6877         for (i = 0; i < 16; i++) {
6878                 status = ql_set_routing_reg(qlge, i, 0, 0);
6879                 if (status) {
6880                         cmn_err(CE_WARN, "Stop routing failed. ");
6881                 }
6882         }
6883         return (status);
6884 }
6885 
6886 /* Initialize the frame-to-queue routing. */
6887 int
6888 ql_route_initialize(qlge_t *qlge)
6889 {
6890         int status = 0;
6891 
6892         status = ql_sem_spinlock(qlge, SEM_RT_IDX_MASK);
6893         if (status != DDI_SUCCESS)
6894                 return (status);
6895 
6896         /* Clear all the entries in the routing table. */
6897         status = ql_stop_routing(qlge);
6898         if (status) {
6899                 goto exit;
6900         }
6901         status = ql_set_routing_reg(qlge, RT_IDX_BCAST_SLOT, RT_IDX_BCAST, 1);
6902         if (status) {
6903                 cmn_err(CE_WARN,
6904                     "Failed to init routing register for broadcast packets.");
6905                 goto exit;
6906         }
6907         /*
6908          * If we have more than one inbound queue, then turn on RSS in the
6909          * routing block.
6910          */
6911         if (qlge->rss_ring_count > 1) {
6912                 status = ql_set_routing_reg(qlge, RT_IDX_RSS_MATCH_SLOT,
6913                     RT_IDX_RSS_MATCH, 1);
6914                 if (status) {
6915                         cmn_err(CE_WARN,
6916                             "Failed to init routing register for MATCH RSS "
6917                             "packets.");
6918                         goto exit;
6919                 }
6920         }
6921 
6922         status = ql_set_routing_reg(qlge, RT_IDX_CAM_HIT_SLOT,
6923             RT_IDX_CAM_HIT, 1);
6924         if (status) {
6925                 cmn_err(CE_WARN,
6926                     "Failed to init routing register for CAM packets.");
6927                 goto exit;
6928         }
6929 
6930         status = ql_set_routing_reg(qlge, RT_IDX_MCAST_MATCH_SLOT,
6931             RT_IDX_MCAST_MATCH, 1);
6932         if (status) {
6933                 cmn_err(CE_WARN,
6934                     "Failed to init routing register for Multicast "
6935                     "packets.");
6936         }
6937 
6938 exit:
6939         ql_sem_unlock(qlge, SEM_RT_IDX_MASK);
6940         return (status);
6941 }
6942 
6943 /*
6944  * Initialize hardware
6945  */
6946 static int
6947 ql_device_initialize(qlge_t *qlge)
6948 {
6949         uint32_t value, mask;
6950         int i;
6951         int status = 0;
6952         uint16_t pause = PAUSE_MODE_DISABLED;
6953         boolean_t update_port_config = B_FALSE;
6954         uint32_t pause_bit_mask;
6955         boolean_t dcbx_enable = B_FALSE;
6956         uint32_t dcbx_bit_mask = 0x10;
6957         /*
6958          * Set up the System register to halt on errors.
6959          */
6960         value = SYS_EFE | SYS_FAE;
6961         mask = value << 16;
6962         ql_write_reg(qlge, REG_SYSTEM, mask | value);
6963 
6964         /* Set the default queue. */
6965         value = NIC_RCV_CFG_DFQ;
6966         mask = NIC_RCV_CFG_DFQ_MASK;
6967 
6968         ql_write_reg(qlge, REG_NIC_RECEIVE_CONFIGURATION, mask | value);
6969 
6970         /* Enable the MPI interrupt. */
6971         ql_write_reg(qlge, REG_INTERRUPT_MASK, (INTR_MASK_PI << 16)
6972             | INTR_MASK_PI);
6973         /* Enable the function, set pagesize, enable error checking. */
6974         value = FSC_FE | FSC_EPC_INBOUND | FSC_EPC_OUTBOUND |
6975             FSC_EC | FSC_VM_PAGE_4K | FSC_DBRST_1024;
6976         /* Set/clear header splitting. */
6977         if (CFG_IST(qlge, CFG_ENABLE_SPLIT_HEADER)) {
6978                 value |= FSC_SH;
6979                 ql_write_reg(qlge, REG_SPLIT_HEADER, SMALL_BUFFER_SIZE);
6980         }
6981         mask = FSC_VM_PAGESIZE_MASK |
6982             FSC_DBL_MASK | FSC_DBRST_MASK | (value << 16);
6983         ql_write_reg(qlge, REG_FUNCTION_SPECIFIC_CONTROL, mask | value);
6984         /*
6985          * check current port max frame size, if different from OS setting,
6986          * then we need to change
6987          */
6988         qlge->max_frame_size =
6989             (qlge->mtu == ETHERMTU)? NORMAL_FRAME_SIZE : JUMBO_FRAME_SIZE;
6990 
6991         mutex_enter(&qlge->mbx_mutex);
6992         status = ql_get_port_cfg(qlge);
6993         mutex_exit(&qlge->mbx_mutex);
6994 
6995         if (status == DDI_SUCCESS) {
6996                 /* if current frame size is smaller than required size */
6997                 if (qlge->port_cfg_info.max_frame_size <
6998                     qlge->max_frame_size) {
6999                         QL_PRINT(DBG_MBX,
7000                             ("update frame size, current %d, new %d\n",
7001                             qlge->port_cfg_info.max_frame_size,
7002                             qlge->max_frame_size));
7003                         qlge->port_cfg_info.max_frame_size =
7004                             qlge->max_frame_size;
7005                         qlge->port_cfg_info.link_cfg |= ENABLE_JUMBO;
7006                         update_port_config = B_TRUE;
7007                 }
7008 
7009                 if (qlge->port_cfg_info.link_cfg & STD_PAUSE)
7010                         pause = PAUSE_MODE_STANDARD;
7011                 else if (qlge->port_cfg_info.link_cfg & PP_PAUSE)
7012                         pause = PAUSE_MODE_PER_PRIORITY;
7013 
7014                 if (pause != qlge->pause) {
7015                         pause_bit_mask = 0x60;  /* bit 5-6 */
7016                         /* clear pause bits */
7017                         qlge->port_cfg_info.link_cfg &= ~pause_bit_mask;
7018                         if (qlge->pause == PAUSE_MODE_STANDARD)
7019                                 qlge->port_cfg_info.link_cfg |= STD_PAUSE;
7020                         else if (qlge->pause == PAUSE_MODE_PER_PRIORITY)
7021                                 qlge->port_cfg_info.link_cfg |= PP_PAUSE;
7022                         update_port_config = B_TRUE;
7023                 }
7024 
7025                 if (qlge->port_cfg_info.link_cfg & DCBX_ENABLE)
7026                         dcbx_enable = B_TRUE;
7027                 if (dcbx_enable != qlge->dcbx_enable) {
7028                         qlge->port_cfg_info.link_cfg &= ~dcbx_bit_mask;
7029                         if (qlge->dcbx_enable)
7030                                 qlge->port_cfg_info.link_cfg |= DCBX_ENABLE;
7031                 }
7032 
7033                 update_port_config = B_TRUE;
7034 
7035                 /* if need to update port configuration */
7036                 if (update_port_config) {
7037                         mutex_enter(&qlge->mbx_mutex);
7038                         (void) ql_set_mpi_port_config(qlge,
7039                             qlge->port_cfg_info);
7040                         mutex_exit(&qlge->mbx_mutex);
7041                 }
7042         } else
7043                 cmn_err(CE_WARN, "ql_get_port_cfg failed");
7044 
7045         /* Start up the rx queues. */
7046         for (i = 0; i < qlge->rx_ring_count; i++) {
7047                 status = ql_start_rx_ring(qlge, &qlge->rx_ring[i]);
7048                 if (status) {
7049                         cmn_err(CE_WARN,
7050                             "Failed to start rx ring[%d]", i);
7051                         return (status);
7052                 }
7053         }
7054 
7055         /*
7056          * If there is more than one inbound completion queue
7057          * then download a RICB to configure RSS.
7058          */
7059         if (qlge->rss_ring_count > 1) {
7060                 status = ql_start_rss(qlge);
7061                 if (status) {
7062                         cmn_err(CE_WARN, "Failed to start RSS.");
7063                         return (status);
7064                 }
7065         }
7066 
7067         /* Start up the tx queues. */
7068         for (i = 0; i < qlge->tx_ring_count; i++) {
7069                 status = ql_start_tx_ring(qlge, &qlge->tx_ring[i]);
7070                 if (status) {
7071                         cmn_err(CE_WARN,
7072                             "Failed to start tx ring[%d]", i);
7073                         return (status);
7074                 }
7075         }
7076         qlge->selected_tx_ring = 0;
7077         /* Set the frame routing filter. */
7078         status = ql_route_initialize(qlge);
7079         if (status) {
7080                 cmn_err(CE_WARN,
7081                     "Failed to init CAM/Routing tables.");
7082                 return (status);
7083         }
7084 
7085         return (status);
7086 }
7087 /*
7088  * Issue soft reset to chip.
7089  */
7090 static int
7091 ql_asic_reset(qlge_t *qlge)
7092 {
7093         int status = DDI_SUCCESS;
7094 
7095         ql_write_reg(qlge, REG_RESET_FAILOVER, FUNCTION_RESET_MASK
7096             |FUNCTION_RESET);
7097 
7098         if (ql_wait_reg_bit(qlge, REG_RESET_FAILOVER, FUNCTION_RESET,
7099             BIT_RESET, 0) != DDI_SUCCESS) {
7100                 cmn_err(CE_WARN,
7101                     "TIMEOUT!!! errored out of resetting the chip!");
7102                 status = DDI_FAILURE;
7103         }
7104 
7105         return (status);
7106 }
7107 
7108 /*
7109  * If there are more than MIN_BUFFERS_ARM_COUNT small buffer descriptors in
7110  * its free list, move xMIN_BUFFERS_ARM_COUNT descriptors to its in use list
7111  * to be used by hardware.
7112  */
7113 static void
7114 ql_arm_sbuf(qlge_t *qlge, struct rx_ring *rx_ring)
7115 {
7116         struct bq_desc *sbq_desc;
7117         int i;
7118         uint64_t *sbq_entry = rx_ring->sbq_dma.vaddr;
7119         uint32_t arm_count;
7120 
7121         if (rx_ring->sbuf_free_count > rx_ring->sbq_len-MIN_BUFFERS_ARM_COUNT)
7122                 arm_count = (rx_ring->sbq_len-MIN_BUFFERS_ARM_COUNT);
7123         else {
7124                 /* Adjust to a multiple of 16 */
7125                 arm_count = (rx_ring->sbuf_free_count / 16) * 16;
7126 #ifdef QLGE_LOAD_UNLOAD
7127                 cmn_err(CE_NOTE, "adjust sbuf arm_count %d\n", arm_count);
7128 #endif
7129         }
7130         for (i = 0; i < arm_count; i++) {
7131                 sbq_desc = ql_get_sbuf_from_free_list(rx_ring);
7132                 if (sbq_desc == NULL)
7133                         break;
7134                 /* Arm asic */
7135                 *sbq_entry = cpu_to_le64(sbq_desc->bd_dma.dma_addr);
7136                 sbq_entry++;
7137 
7138                 /* link the descriptors to in_use_list */
7139                 ql_add_sbuf_to_in_use_list(rx_ring, sbq_desc);
7140                 rx_ring->sbq_prod_idx++;
7141         }
7142         ql_update_sbq_prod_idx(qlge, rx_ring);
7143 }
7144 
7145 /*
7146  * If there are more than MIN_BUFFERS_ARM_COUNT large buffer descriptors in
7147  * its free list, move xMIN_BUFFERS_ARM_COUNT descriptors to its in use list
7148  * to be used by hardware.
7149  */
7150 static void
7151 ql_arm_lbuf(qlge_t *qlge, struct rx_ring *rx_ring)
7152 {
7153         struct bq_desc *lbq_desc;
7154         int i;
7155         uint64_t *lbq_entry = rx_ring->lbq_dma.vaddr;
7156         uint32_t arm_count;
7157 
7158         if (rx_ring->lbuf_free_count > rx_ring->lbq_len-MIN_BUFFERS_ARM_COUNT)
7159                 arm_count = (rx_ring->lbq_len-MIN_BUFFERS_ARM_COUNT);
7160         else {
7161                 /* Adjust to a multiple of 16 */
7162                 arm_count = (rx_ring->lbuf_free_count / 16) * 16;
7163 #ifdef QLGE_LOAD_UNLOAD
7164                 cmn_err(CE_NOTE, "adjust lbuf arm_count %d\n", arm_count);
7165 #endif
7166         }
7167         for (i = 0; i < arm_count; i++) {
7168                 lbq_desc = ql_get_lbuf_from_free_list(rx_ring);
7169                 if (lbq_desc == NULL)
7170                         break;
7171                 /* Arm asic */
7172                 *lbq_entry = cpu_to_le64(lbq_desc->bd_dma.dma_addr);
7173                 lbq_entry++;
7174 
7175                 /* link the descriptors to in_use_list */
7176                 ql_add_lbuf_to_in_use_list(rx_ring, lbq_desc);
7177                 rx_ring->lbq_prod_idx++;
7178         }
7179         ql_update_lbq_prod_idx(qlge, rx_ring);
7180 }
7181 
7182 
7183 /*
7184  * Initializes the adapter by configuring request and response queues,
7185  * allocates and ARMs small and large receive buffers to the
7186  * hardware
7187  */
7188 static int
7189 ql_bringup_adapter(qlge_t *qlge)
7190 {
7191         int i;
7192 
7193         if (ql_device_initialize(qlge) != DDI_SUCCESS) {
7194                 cmn_err(CE_WARN, "?%s(%d): ql_device_initialize failed",
7195                     __func__, qlge->instance);
7196                 goto err_bringup;
7197         }
7198         qlge->sequence |= INIT_ADAPTER_UP;
7199 
7200 #ifdef QLGE_TRACK_BUFFER_USAGE
7201         for (i = 0; i < qlge->rx_ring_count; i++) {
7202                 if (qlge->rx_ring[i].type != TX_Q) {
7203                         qlge->rx_sb_low_count[i] = NUM_SMALL_BUFFERS;
7204                         qlge->rx_lb_low_count[i] = NUM_LARGE_BUFFERS;
7205                 }
7206                 qlge->cq_low_count[i] = NUM_RX_RING_ENTRIES;
7207         }
7208 #endif
7209         /* Arm buffers */
7210         for (i = 0; i < qlge->rx_ring_count; i++) {
7211                 if (qlge->rx_ring[i].type != TX_Q) {
7212                         ql_arm_sbuf(qlge, &qlge->rx_ring[i]);
7213                         ql_arm_lbuf(qlge, &qlge->rx_ring[i]);
7214                 }
7215         }
7216 
7217         /* Enable work/request queues */
7218         for (i = 0; i < qlge->tx_ring_count; i++) {
7219                 if (qlge->tx_ring[i].valid_db_reg)
7220                         ql_write_doorbell_reg(qlge,
7221                             qlge->tx_ring[i].valid_db_reg,
7222                             REQ_Q_VALID);
7223         }
7224 
7225         /* Enable completion queues */
7226         for (i = 0; i < qlge->rx_ring_count; i++) {
7227                 if (qlge->rx_ring[i].valid_db_reg)
7228                         ql_write_doorbell_reg(qlge,
7229                             qlge->rx_ring[i].valid_db_reg,
7230                             RSP_Q_VALID);
7231         }
7232 
7233         for (i = 0; i < qlge->tx_ring_count; i++) {
7234                 mutex_enter(&qlge->tx_ring[i].tx_lock);
7235                 qlge->tx_ring[i].mac_flags = QL_MAC_STARTED;
7236                 mutex_exit(&qlge->tx_ring[i].tx_lock);
7237         }
7238 
7239         for (i = 0; i < qlge->rx_ring_count; i++) {
7240                 mutex_enter(&qlge->rx_ring[i].rx_lock);
7241                 qlge->rx_ring[i].mac_flags = QL_MAC_STARTED;
7242                 mutex_exit(&qlge->rx_ring[i].rx_lock);
7243         }
7244 
7245         /* This mutex will get re-acquired in enable_completion interrupt */
7246         mutex_exit(&qlge->hw_mutex);
7247         /* Traffic can start flowing now */
7248         ql_enable_all_completion_interrupts(qlge);
7249         mutex_enter(&qlge->hw_mutex);
7250 
7251         ql_enable_global_interrupt(qlge);
7252 
7253         qlge->sequence |= ADAPTER_INIT;
7254         return (DDI_SUCCESS);
7255 
7256 err_bringup:
7257         (void) ql_asic_reset(qlge);
7258         return (DDI_FAILURE);
7259 }
7260 
7261 /*
7262  * Initialize mutexes of each rx/tx rings
7263  */
7264 static int
7265 ql_init_rx_tx_locks(qlge_t *qlge)
7266 {
7267         struct tx_ring *tx_ring;
7268         struct rx_ring *rx_ring;
7269         int i;
7270 
7271         for (i = 0; i < qlge->tx_ring_count; i++) {
7272                 tx_ring = &qlge->tx_ring[i];
7273                 mutex_init(&tx_ring->tx_lock, NULL, MUTEX_DRIVER,
7274                     DDI_INTR_PRI(qlge->intr_pri));
7275         }
7276 
7277         for (i = 0; i < qlge->rx_ring_count; i++) {
7278                 rx_ring = &qlge->rx_ring[i];
7279                 mutex_init(&rx_ring->rx_lock, NULL, MUTEX_DRIVER,
7280                     DDI_INTR_PRI(qlge->intr_pri));
7281                 mutex_init(&rx_ring->sbq_lock, NULL, MUTEX_DRIVER,
7282                     DDI_INTR_PRI(qlge->intr_pri));
7283                 mutex_init(&rx_ring->lbq_lock, NULL, MUTEX_DRIVER,
7284                     DDI_INTR_PRI(qlge->intr_pri));
7285         }
7286 
7287         return (DDI_SUCCESS);
7288 }
7289 
7290 /*ARGSUSED*/
7291 /*
7292  * Simply call pci_ereport_post which generates ereports for errors
7293  * that occur in the PCI local bus configuration status registers.
7294  */
7295 static int
7296 ql_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
7297 {
7298         pci_ereport_post(dip, err, NULL);
7299         return (err->fme_status);
7300 }
7301 
7302 static void
7303 ql_fm_init(qlge_t *qlge)
7304 {
7305         ddi_iblock_cookie_t iblk;
7306 
7307         QL_PRINT(DBG_INIT, ("ql_fm_init(%d) entered, FMA capability %x\n",
7308             qlge->instance, qlge->fm_capabilities));
7309         /*
7310          * Register capabilities with IO Fault Services. The capabilities
7311          * set above may not be supported by the parent nexus, in that case
7312          * some capability bits may be cleared.
7313          */
7314         if (qlge->fm_capabilities)
7315                 ddi_fm_init(qlge->dip, &qlge->fm_capabilities, &iblk);
7316 
7317         /*
7318          * Initialize pci ereport capabilities if ereport capable
7319          */
7320         if (DDI_FM_EREPORT_CAP(qlge->fm_capabilities) ||
7321             DDI_FM_ERRCB_CAP(qlge->fm_capabilities)) {
7322                 pci_ereport_setup(qlge->dip);
7323         }
7324 
7325         /* Register error callback if error callback capable */
7326         if (DDI_FM_ERRCB_CAP(qlge->fm_capabilities)) {
7327                 ddi_fm_handler_register(qlge->dip,
7328                     ql_fm_error_cb, (void*) qlge);
7329         }
7330 
7331         /*
7332          * DDI_FLGERR_ACC indicates:
7333          *  Driver will check its access handle(s) for faults on
7334          *   a regular basis by calling ddi_fm_acc_err_get
7335          *  Driver is able to cope with incorrect results of I/O
7336          *   operations resulted from an I/O fault
7337          */
7338         if (DDI_FM_ACC_ERR_CAP(qlge->fm_capabilities)) {
7339                 ql_dev_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
7340         }
7341 
7342         /*
7343          * DDI_DMA_FLAGERR indicates:
7344          *  Driver will check its DMA handle(s) for faults on a
7345          *   regular basis using ddi_fm_dma_err_get
7346          *  Driver is able to cope with incorrect results of DMA
7347          *   operations resulted from an I/O fault
7348          */
7349         if (DDI_FM_DMA_ERR_CAP(qlge->fm_capabilities)) {
7350                 tx_mapping_dma_attr.dma_attr_flags = DDI_DMA_FLAGERR;
7351                 dma_attr.dma_attr_flags = DDI_DMA_FLAGERR;
7352         }
7353         QL_PRINT(DBG_INIT, ("ql_fm_init(%d) done\n",
7354             qlge->instance));
7355 }
7356 
7357 static void
7358 ql_fm_fini(qlge_t *qlge)
7359 {
7360         QL_PRINT(DBG_INIT, ("ql_fm_fini(%d) entered\n",
7361             qlge->instance));
7362         /* Only unregister FMA capabilities if we registered some */
7363         if (qlge->fm_capabilities) {
7364 
7365                 /*
7366                  * Release any resources allocated by pci_ereport_setup()
7367                  */
7368                 if (DDI_FM_EREPORT_CAP(qlge->fm_capabilities) ||
7369                     DDI_FM_ERRCB_CAP(qlge->fm_capabilities))
7370                         pci_ereport_teardown(qlge->dip);
7371 
7372                 /*
7373                  * Un-register error callback if error callback capable
7374                  */
7375                 if (DDI_FM_ERRCB_CAP(qlge->fm_capabilities))
7376                         ddi_fm_handler_unregister(qlge->dip);
7377 
7378                 /* Unregister from IO Fault Services */
7379                 ddi_fm_fini(qlge->dip);
7380         }
7381         QL_PRINT(DBG_INIT, ("ql_fm_fini(%d) done\n",
7382             qlge->instance));
7383 }
7384 /*
7385  * ql_attach - Driver attach.
7386  */
7387 static int
7388 ql_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
7389 {
7390         int instance;
7391         qlge_t *qlge = NULL;
7392         int rval;
7393         uint16_t w;
7394         mac_register_t *macp = NULL;
7395         uint32_t data;
7396 
7397         rval = DDI_FAILURE;
7398 
7399         /* first get the instance */
7400         instance = ddi_get_instance(dip);
7401 
7402         switch (cmd) {
7403         case DDI_ATTACH:
7404                 /*
7405                  * Allocate our per-device-instance structure
7406                  */
7407                 qlge = (qlge_t *)kmem_zalloc(sizeof (*qlge), KM_SLEEP);
7408                 ASSERT(qlge != NULL);
7409                 qlge->sequence |= INIT_SOFTSTATE_ALLOC;
7410 
7411                 qlge->dip = dip;
7412                 qlge->instance = instance;
7413                 /* Set up the coalescing parameters. */
7414                 qlge->ql_dbgprnt = 0;
7415 #if QL_DEBUG
7416                 qlge->ql_dbgprnt = QL_DEBUG;
7417 #endif /* QL_DEBUG */
7418 
7419                 /*
7420                  * Initialize for fma support
7421                  */
7422                 /* fault management (fm) capabilities. */
7423                 qlge->fm_capabilities =
7424                     DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE;
7425                 data = ql_get_prop(qlge, "fm-capable");
7426                 if (data <= 0xf) {
7427                         qlge->fm_capabilities = data;
7428                 }
7429                 ql_fm_init(qlge);
7430                 qlge->sequence |= INIT_FM;
7431                 QL_PRINT(DBG_INIT, ("ql_attach(%d): fma init done\n",
7432                     qlge->instance));
7433 
7434                 /*
7435                  * Setup the ISP8x00 registers address mapping to be
7436                  * accessed by this particular driver.
7437                  * 0x0   Configuration Space
7438                  * 0x1   I/O Space
7439                  * 0x2   1st Memory Space address - Control Register Set
7440                  * 0x3   2nd Memory Space address - Doorbell Memory Space
7441                  */
7442                 w = 2;
7443                 if (ddi_regs_map_setup(dip, w, (caddr_t *)&qlge->iobase, 0,
7444                     sizeof (dev_reg_t), &ql_dev_acc_attr,
7445                     &qlge->dev_handle) != DDI_SUCCESS) {
7446                         cmn_err(CE_WARN, "%s(%d): Unable to map device "
7447                             "registers", ADAPTER_NAME, instance);
7448                         break;
7449                 }
7450                 QL_PRINT(DBG_GLD, ("ql_attach: I/O base = 0x%x\n",
7451                     qlge->iobase));
7452                 qlge->sequence |= INIT_REGS_SETUP;
7453 
7454                 /* map Doorbell memory space */
7455                 w = 3;
7456                 if (ddi_regs_map_setup(dip, w,
7457                     (caddr_t *)&qlge->doorbell_reg_iobase, 0,
7458                     0x100000 /* sizeof (dev_doorbell_reg_t) */,
7459                     &ql_dev_acc_attr,
7460                     &qlge->dev_doorbell_reg_handle) != DDI_SUCCESS) {
7461                         cmn_err(CE_WARN, "%s(%d): Unable to map Doorbell "
7462                             "registers",
7463                             ADAPTER_NAME, instance);
7464                         break;
7465                 }
7466                 QL_PRINT(DBG_GLD, ("ql_attach: Doorbell I/O base = 0x%x\n",
7467                     qlge->doorbell_reg_iobase));
7468                 qlge->sequence |= INIT_DOORBELL_REGS_SETUP;
7469 
7470                 /*
7471                  * Allocate a macinfo structure for this instance
7472                  */
7473                 if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
7474                         cmn_err(CE_WARN, "%s(%d): mac_alloc failed",
7475                             __func__, instance);
7476                         break;
7477                 }
7478                 /* save adapter status to dip private data */
7479                 ddi_set_driver_private(dip, qlge);
7480                 QL_PRINT(DBG_INIT, ("%s(%d): Allocate macinfo structure done\n",
7481                     ADAPTER_NAME, instance));
7482                 qlge->sequence |= INIT_MAC_ALLOC;
7483 
7484                 /*
7485                  * Attach this instance of the device
7486                  */
7487                 /* Setup PCI Local Bus Configuration resource. */
7488                 if (pci_config_setup(dip, &qlge->pci_handle) != DDI_SUCCESS) {
7489                         cmn_err(CE_WARN, "%s(%d):Unable to get PCI resources",
7490                             ADAPTER_NAME, instance);
7491                         if (qlge->fm_enable) {
7492                                 ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
7493                                 ddi_fm_service_impact(qlge->dip,
7494                                     DDI_SERVICE_LOST);
7495                         }
7496                         break;
7497                 }
7498                 qlge->sequence |= INIT_PCI_CONFIG_SETUP;
7499                 QL_PRINT(DBG_GLD, ("ql_attach(%d): pci_config_setup done\n",
7500                     instance));
7501 
7502                 if (ql_init_instance(qlge) != DDI_SUCCESS) {
7503                         cmn_err(CE_WARN, "%s(%d): Unable to initialize device "
7504                             "instance", ADAPTER_NAME, instance);
7505                         if (qlge->fm_enable) {
7506                                 ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE);
7507                                 ddi_fm_service_impact(qlge->dip,
7508                                     DDI_SERVICE_LOST);
7509                         }
7510                         break;
7511                 }
7512                 QL_PRINT(DBG_GLD, ("ql_attach(%d): ql_init_instance done\n",
7513                     instance));
7514 
7515                 /* Setup interrupt vectors */
7516                 if (ql_alloc_irqs(qlge) != DDI_SUCCESS) {
7517                         break;
7518                 }
7519                 qlge->sequence |= INIT_INTR_ALLOC;
7520                 QL_PRINT(DBG_GLD, ("ql_attach(%d): ql_alloc_irqs done\n",
7521                     instance));
7522 
7523                 /* Configure queues */
7524                 if (ql_setup_rings(qlge) != DDI_SUCCESS) {
7525                         break;
7526                 }
7527                 qlge->sequence |= INIT_SETUP_RINGS;
7528                 QL_PRINT(DBG_GLD, ("ql_attach(%d): setup rings done\n",
7529                     instance));
7530 
7531                 /*
7532                  * Allocate memory resources
7533                  */
7534                 if (ql_alloc_mem_resources(qlge) != DDI_SUCCESS) {
7535                         cmn_err(CE_WARN, "%s(%d): memory allocation failed",
7536                             __func__, qlge->instance);
7537                         break;
7538                 }
7539                 qlge->sequence |= INIT_MEMORY_ALLOC;
7540                 QL_PRINT(DBG_GLD, ("ql_alloc_mem_resources(%d) done\n",
7541                     instance));
7542 
7543                 /*
7544                  * Map queues to interrupt vectors
7545                  */
7546                 ql_resolve_queues_to_irqs(qlge);
7547 
7548                 /* Initialize mutex, need the interrupt priority */
7549                 (void) ql_init_rx_tx_locks(qlge);
7550                 qlge->sequence |= INIT_LOCKS_CREATED;
7551                 QL_PRINT(DBG_INIT, ("%s(%d): ql_init_rx_tx_locks done\n",
7552                     ADAPTER_NAME, instance));
7553 
7554                 /*
7555                  * Use a soft interrupt to do something that we do not want
7556                  * to do in regular network functions or with mutexs being held
7557                  */
7558                 if (ddi_intr_add_softint(qlge->dip, &qlge->mpi_event_intr_hdl,
7559                     DDI_INTR_SOFTPRI_MIN, ql_mpi_event_work, (caddr_t)qlge)
7560                     != DDI_SUCCESS) {
7561                         break;
7562                 }
7563 
7564                 if (ddi_intr_add_softint(qlge->dip, &qlge->asic_reset_intr_hdl,
7565                     DDI_INTR_SOFTPRI_MIN, ql_asic_reset_work, (caddr_t)qlge)
7566                     != DDI_SUCCESS) {
7567                         break;
7568                 }
7569 
7570                 if (ddi_intr_add_softint(qlge->dip, &qlge->mpi_reset_intr_hdl,
7571                     DDI_INTR_SOFTPRI_MIN, ql_mpi_reset_work, (caddr_t)qlge)
7572                     != DDI_SUCCESS) {
7573                         break;
7574                 }
7575                 qlge->sequence |= INIT_ADD_SOFT_INTERRUPT;
7576                 QL_PRINT(DBG_INIT, ("%s(%d): ddi_intr_add_softint done\n",
7577                     ADAPTER_NAME, instance));
7578 
7579                 /*
7580                  * mutex to protect the adapter state structure.
7581                  * initialize mutexes according to the interrupt priority
7582                  */
7583                 mutex_init(&qlge->gen_mutex, NULL, MUTEX_DRIVER,
7584                     DDI_INTR_PRI(qlge->intr_pri));
7585                 mutex_init(&qlge->hw_mutex, NULL, MUTEX_DRIVER,
7586                     DDI_INTR_PRI(qlge->intr_pri));
7587                 mutex_init(&qlge->mbx_mutex, NULL, MUTEX_DRIVER,
7588                     DDI_INTR_PRI(qlge->intr_pri));
7589 
7590                 /* Mailbox wait and interrupt conditional variable. */
7591                 cv_init(&qlge->cv_mbx_intr, NULL, CV_DRIVER, NULL);
7592                 qlge->sequence |= INIT_MUTEX;
7593                 QL_PRINT(DBG_INIT, ("%s(%d): mutex_init done\n",
7594                     ADAPTER_NAME, instance));
7595 
7596                 /*
7597                  * KStats
7598                  */
7599                 if (ql_init_kstats(qlge) != DDI_SUCCESS) {
7600                         cmn_err(CE_WARN, "%s(%d): KState initialization failed",
7601                             ADAPTER_NAME, instance);
7602                         break;
7603                 }
7604                 qlge->sequence |= INIT_KSTATS;
7605                 QL_PRINT(DBG_INIT, ("%s(%d): ql_init_kstats done\n",
7606                     ADAPTER_NAME, instance));
7607 
7608                 /*
7609                  * Initialize gld macinfo structure
7610                  */
7611                 ql_gld3_init(qlge, macp);
7612                 /*
7613                  * Add interrupt handlers
7614                  */
7615                 if (ql_add_intr_handlers(qlge) != DDI_SUCCESS) {
7616                         cmn_err(CE_WARN, "Failed to add interrupt "
7617                             "handlers");
7618                         break;
7619                 }
7620                 qlge->sequence |= INIT_ADD_INTERRUPT;
7621                 QL_PRINT(DBG_INIT, ("%s(%d): Add interrupt handler done\n",
7622                     ADAPTER_NAME, instance));
7623 
7624                 /*
7625                  * MAC Register
7626                  */
7627                 if (mac_register(macp, &qlge->mh) != DDI_SUCCESS) {
7628                         cmn_err(CE_WARN, "%s(%d): mac_register failed",
7629                             __func__, instance);
7630                         break;
7631                 }
7632                 qlge->sequence |= INIT_MAC_REGISTERED;
7633                 QL_PRINT(DBG_GLD, ("%s(%d): mac_register done\n",
7634                     ADAPTER_NAME, instance));
7635 
7636                 mac_free(macp);
7637                 macp = NULL;
7638 
7639                 qlge->mac_flags = QL_MAC_ATTACHED;
7640 
7641                 ddi_report_dev(dip);
7642 
7643                 rval = DDI_SUCCESS;
7644 
7645         break;
7646 /*
7647  * DDI_RESUME
7648  * When called  with  cmd  set  to  DDI_RESUME,  attach()  must
7649  * restore  the hardware state of a device (power may have been
7650  * removed from the device), allow  pending  requests  to  con-
7651  * tinue,  and  service  new requests. In this case, the driver
7652  * must not  make  any  assumptions  about  the  state  of  the
7653  * hardware,  but  must  restore the state of the device except
7654  * for the power level of components.
7655  *
7656  */
7657         case DDI_RESUME:
7658 
7659                 if ((qlge = (qlge_t *)QL_GET_DEV(dip)) == NULL)
7660                         return (DDI_FAILURE);
7661 
7662                 QL_PRINT(DBG_GLD, ("%s(%d)-DDI_RESUME\n",
7663                     __func__, qlge->instance));
7664 
7665                 mutex_enter(&qlge->gen_mutex);
7666                 rval = ql_do_start(qlge);
7667                 mutex_exit(&qlge->gen_mutex);
7668                 break;
7669 
7670         default:
7671                 break;
7672         }
7673 
7674         /* if failed to attach */
7675         if ((cmd == DDI_ATTACH) && (rval != DDI_SUCCESS) && (qlge != NULL)) {
7676                 cmn_err(CE_WARN, "qlge driver attach failed, sequence %x",
7677                     qlge->sequence);
7678                 ql_free_resources(qlge);
7679         }
7680 
7681         return (rval);
7682 }
7683 
7684 /*
7685  * Unbind all pending tx dma handles during driver bring down
7686  */
7687 static void
7688 ql_unbind_pending_tx_dma_handle(struct tx_ring *tx_ring)
7689 {
7690         struct tx_ring_desc *tx_ring_desc;
7691         int i, j;
7692 
7693         if (tx_ring->wq_desc) {
7694                 tx_ring_desc = tx_ring->wq_desc;
7695                 for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) {
7696                         for (j = 0; j < tx_ring_desc->tx_dma_handle_used; j++) {
7697                                 if (tx_ring_desc->tx_dma_handle[j]) {
7698                                         (void) ddi_dma_unbind_handle(
7699                                             tx_ring_desc->tx_dma_handle[j]);
7700                                 }
7701                         }
7702                         tx_ring_desc->tx_dma_handle_used = 0;
7703                 } /* end of for loop */
7704         }
7705 }
7706 /*
7707  * Wait for all the packets sent to the chip to finish transmission
7708  * to prevent buffers to be unmapped before or during a transmit operation
7709  */
7710 static int
7711 ql_wait_tx_quiesce(qlge_t *qlge)
7712 {
7713         int count = MAX_TX_WAIT_COUNT, i;
7714         int rings_done;
7715         volatile struct tx_ring *tx_ring;
7716         uint32_t consumer_idx;
7717         uint32_t producer_idx;
7718         uint32_t temp;
7719         int done = 0;
7720         int rval = DDI_FAILURE;
7721 
7722         while (!done) {
7723                 rings_done = 0;
7724 
7725                 for (i = 0; i < qlge->tx_ring_count; i++) {
7726                         tx_ring = &qlge->tx_ring[i];
7727                         temp = ql_read_doorbell_reg(qlge,
7728                             tx_ring->prod_idx_db_reg);
7729                         producer_idx = temp & 0x0000ffff;
7730                         consumer_idx = (temp >> 16);
7731 
7732                         if (qlge->isr_stride) {
7733                                 struct rx_ring *ob_ring;
7734                                 ob_ring = &qlge->rx_ring[tx_ring->cq_id];
7735                                 if (producer_idx != ob_ring->cnsmr_idx) {
7736                                         cmn_err(CE_NOTE, " force clean \n");
7737                                         (void) ql_clean_outbound_rx_ring(
7738                                             ob_ring);
7739                                 }
7740                         }
7741                         /*
7742                          * Get the pending iocb count, ones which have not been
7743                          * pulled down by the chip
7744                          */
7745                         if (producer_idx >= consumer_idx)
7746                                 temp = (producer_idx - consumer_idx);
7747                         else
7748                                 temp = (tx_ring->wq_len - consumer_idx) +
7749                                     producer_idx;
7750 
7751                         if ((tx_ring->tx_free_count + temp) >= tx_ring->wq_len)
7752                                 rings_done++;
7753                         else {
7754                                 done = 1;
7755                                 break;
7756                         }
7757                 }
7758 
7759                 /* If all the rings are done */
7760                 if (rings_done >= qlge->tx_ring_count) {
7761 #ifdef QLGE_LOAD_UNLOAD
7762                         cmn_err(CE_NOTE, "%s(%d) done successfully \n",
7763                             __func__, qlge->instance);
7764 #endif
7765                         rval = DDI_SUCCESS;
7766                         break;
7767                 }
7768 
7769                 qlge_delay(100);
7770 
7771                 count--;
7772                 if (!count) {
7773 
7774                         count = MAX_TX_WAIT_COUNT;
7775 #ifdef QLGE_LOAD_UNLOAD
7776                         volatile struct rx_ring *rx_ring;
7777                         cmn_err(CE_NOTE, "%s(%d): Waiting for %d pending"
7778                             " Transmits on queue %d to complete .\n",
7779                             __func__, qlge->instance,
7780                             (qlge->tx_ring[i].wq_len -
7781                             qlge->tx_ring[i].tx_free_count),
7782                             i);
7783 
7784                         rx_ring = &qlge->rx_ring[i+1];
7785                         temp = ql_read_doorbell_reg(qlge,
7786                             rx_ring->cnsmr_idx_db_reg);
7787                         consumer_idx = temp & 0x0000ffff;
7788                         producer_idx = (temp >> 16);
7789                         cmn_err(CE_NOTE, "%s(%d): Transmit completion queue %d,"
7790                             " Producer %d, Consumer %d\n",
7791                             __func__, qlge->instance,
7792                             i+1,
7793                             producer_idx, consumer_idx);
7794 
7795                         temp = ql_read_doorbell_reg(qlge,
7796                             tx_ring->prod_idx_db_reg);
7797                         producer_idx = temp & 0x0000ffff;
7798                         consumer_idx = (temp >> 16);
7799                         cmn_err(CE_NOTE, "%s(%d): Transmit request queue %d,"
7800                             " Producer %d, Consumer %d\n",
7801                             __func__, qlge->instance, i,
7802                             producer_idx, consumer_idx);
7803 #endif
7804 
7805                         /* For now move on */
7806                         break;
7807                 }
7808         }
7809         /* Stop the request queue */
7810         mutex_enter(&qlge->hw_mutex);
7811         for (i = 0; i < qlge->tx_ring_count; i++) {
7812                 if (qlge->tx_ring[i].valid_db_reg) {
7813                         ql_write_doorbell_reg(qlge,
7814                             qlge->tx_ring[i].valid_db_reg, 0);
7815                 }
7816         }
7817         mutex_exit(&qlge->hw_mutex);
7818         return (rval);
7819 }
7820 
7821 /*
7822  * Wait for all the receives indicated to the stack to come back
7823  */
7824 static int
7825 ql_wait_rx_complete(qlge_t *qlge)
7826 {
7827         int i;
7828         /* Disable all the completion queues */
7829         mutex_enter(&qlge->hw_mutex);
7830         for (i = 0; i < qlge->rx_ring_count; i++) {
7831                 if (qlge->rx_ring[i].valid_db_reg) {
7832                         ql_write_doorbell_reg(qlge,
7833                             qlge->rx_ring[i].valid_db_reg, 0);
7834                 }
7835         }
7836         mutex_exit(&qlge->hw_mutex);
7837 
7838         /* Wait for OS to return all rx buffers */
7839         qlge_delay(QL_ONE_SEC_DELAY);
7840         return (DDI_SUCCESS);
7841 }
7842 
7843 /*
7844  * stop the driver
7845  */
7846 static int
7847 ql_bringdown_adapter(qlge_t *qlge)
7848 {
7849         int i;
7850         int status = DDI_SUCCESS;
7851 
7852         qlge->mac_flags = QL_MAC_BRINGDOWN;
7853         if (qlge->sequence & ADAPTER_INIT) {
7854                 /* stop forwarding external packets to driver */
7855                 status = ql_sem_spinlock(qlge, SEM_RT_IDX_MASK);
7856                 if (status)
7857                         return (status);
7858                 (void) ql_stop_routing(qlge);
7859                 ql_sem_unlock(qlge, SEM_RT_IDX_MASK);
7860                 /*
7861                  * Set the flag for receive and transmit
7862                  * operations to cease
7863                  */
7864                 for (i = 0; i < qlge->tx_ring_count; i++) {
7865                         mutex_enter(&qlge->tx_ring[i].tx_lock);
7866                         qlge->tx_ring[i].mac_flags = QL_MAC_STOPPED;
7867                         mutex_exit(&qlge->tx_ring[i].tx_lock);
7868                 }
7869 
7870                 for (i = 0; i < qlge->rx_ring_count; i++) {
7871                         mutex_enter(&qlge->rx_ring[i].rx_lock);
7872                         qlge->rx_ring[i].mac_flags = QL_MAC_STOPPED;
7873                         mutex_exit(&qlge->rx_ring[i].rx_lock);
7874                 }
7875 
7876                 /*
7877                  * Need interrupts to be running while the transmit
7878                  * completions are cleared. Wait for the packets
7879                  * queued to the chip to be sent out
7880                  */
7881                 (void) ql_wait_tx_quiesce(qlge);
7882                 /* Interrupts not needed from now */
7883                 ql_disable_all_completion_interrupts(qlge);
7884 
7885                 mutex_enter(&qlge->hw_mutex);
7886                 /* Disable Global interrupt */
7887                 ql_disable_global_interrupt(qlge);
7888                 mutex_exit(&qlge->hw_mutex);
7889 
7890                 /* Wait for all the indicated packets to come back */
7891                 status = ql_wait_rx_complete(qlge);
7892 
7893                 mutex_enter(&qlge->hw_mutex);
7894                 /* Reset adapter */
7895                 (void) ql_asic_reset(qlge);
7896                 /*
7897                  * Unbind all tx dma handles to prevent pending tx descriptors'
7898                  * dma handles from being re-used.
7899                  */
7900                 for (i = 0; i < qlge->tx_ring_count; i++) {
7901                         ql_unbind_pending_tx_dma_handle(&qlge->tx_ring[i]);
7902                 }
7903 
7904                 qlge->sequence &= ~ADAPTER_INIT;
7905 
7906                 mutex_exit(&qlge->hw_mutex);
7907         }
7908         return (status);
7909 }
7910 
7911 /*
7912  * ql_detach
7913  * Used to remove all the states associated with a given
7914  * instances of a device node prior to the removal of that
7915  * instance from the system.
7916  */
7917 static int
7918 ql_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
7919 {
7920         qlge_t *qlge;
7921         int rval;
7922 
7923         rval = DDI_SUCCESS;
7924 
7925         switch (cmd) {
7926         case DDI_DETACH:
7927 
7928                 if ((qlge = QL_GET_DEV(dip)) == NULL)
7929                         return (DDI_FAILURE);
7930                 rval = ql_bringdown_adapter(qlge);
7931                 if (rval != DDI_SUCCESS)
7932                         break;
7933 
7934                 qlge->mac_flags = QL_MAC_DETACH;
7935 
7936                 /* free memory resources */
7937                 if (qlge->sequence & INIT_MEMORY_ALLOC) {
7938                         ql_free_mem_resources(qlge);
7939                         qlge->sequence &= ~INIT_MEMORY_ALLOC;
7940                 }
7941                 ql_free_resources(qlge);
7942 
7943                 break;
7944 
7945         case DDI_SUSPEND:
7946                 if ((qlge = QL_GET_DEV(dip)) == NULL)
7947                         return (DDI_FAILURE);
7948 
7949                 mutex_enter(&qlge->gen_mutex);
7950                 if ((qlge->mac_flags == QL_MAC_ATTACHED) ||
7951                     (qlge->mac_flags == QL_MAC_STARTED)) {
7952                         (void) ql_do_stop(qlge);
7953                 }
7954                 qlge->mac_flags = QL_MAC_SUSPENDED;
7955                 mutex_exit(&qlge->gen_mutex);
7956 
7957                 break;
7958         default:
7959                 rval = DDI_FAILURE;
7960                 break;
7961         }
7962 
7963         return (rval);
7964 }
7965 
7966 /*
7967  * quiesce(9E) entry point.
7968  *
7969  * This function is called when the system is single-threaded at high
7970  * PIL with preemption disabled. Therefore, this function must not be
7971  * blocked.
7972  *
7973  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
7974  */
7975 int
7976 ql_quiesce(dev_info_t *dip)
7977 {
7978         qlge_t *qlge;
7979         int i;
7980 
7981         if ((qlge = QL_GET_DEV(dip)) == NULL)
7982                 return (DDI_FAILURE);
7983 
7984         if (CFG_IST(qlge, CFG_CHIP_8100)) {
7985                 /* stop forwarding external packets to driver */
7986                 (void) ql_sem_spinlock(qlge, SEM_RT_IDX_MASK);
7987                 (void) ql_stop_routing(qlge);
7988                 ql_sem_unlock(qlge, SEM_RT_IDX_MASK);
7989                 /* Stop all the request queues */
7990                 for (i = 0; i < qlge->tx_ring_count; i++) {
7991                         if (qlge->tx_ring[i].valid_db_reg) {
7992                                 ql_write_doorbell_reg(qlge,
7993                                     qlge->tx_ring[i].valid_db_reg, 0);
7994                         }
7995                 }
7996                 qlge_delay(QL_ONE_SEC_DELAY/4);
7997                 /* Interrupts not needed from now */
7998                 /* Disable MPI interrupt */
7999                 ql_write_reg(qlge, REG_INTERRUPT_MASK,
8000                     (INTR_MASK_PI << 16));
8001                 ql_disable_global_interrupt(qlge);
8002 
8003                 /* Disable all the rx completion queues */
8004                 for (i = 0; i < qlge->rx_ring_count; i++) {
8005                         if (qlge->rx_ring[i].valid_db_reg) {
8006                                 ql_write_doorbell_reg(qlge,
8007                                     qlge->rx_ring[i].valid_db_reg, 0);
8008                         }
8009                 }
8010                 qlge_delay(QL_ONE_SEC_DELAY/4);
8011                 qlge->mac_flags = QL_MAC_STOPPED;
8012                 /* Reset adapter */
8013                 (void) ql_asic_reset(qlge);
8014                 qlge_delay(100);
8015         }
8016 
8017         return (DDI_SUCCESS);
8018 }
8019 
8020 QL_STREAM_OPS(ql_ops, ql_attach, ql_detach);
8021 
8022 /*
8023  * Loadable Driver Interface Structures.
8024  * Declare and initialize the module configuration section...
8025  */
8026 static struct modldrv modldrv = {
8027         &mod_driverops,             /* type of module: driver */
8028         version,                /* name of module */
8029         &ql_ops                     /* driver dev_ops */
8030 };
8031 
8032 static struct modlinkage modlinkage = {
8033         MODREV_1,       &modldrv,   NULL
8034 };
8035 
8036 /*
8037  * Loadable Module Routines
8038  */
8039 
8040 /*
8041  * _init
8042  * Initializes a loadable module. It is called before any other
8043  * routine in a loadable module.
8044  */
8045 int
8046 _init(void)
8047 {
8048         int rval;
8049 
8050         mac_init_ops(&ql_ops, ADAPTER_NAME);
8051         rval = mod_install(&modlinkage);
8052         if (rval != DDI_SUCCESS) {
8053                 mac_fini_ops(&ql_ops);
8054                 cmn_err(CE_WARN, "?Unable to install/attach driver '%s'",
8055                     ADAPTER_NAME);
8056         }
8057 
8058         return (rval);
8059 }
8060 
8061 /*
8062  * _fini
8063  * Prepares a module for unloading. It is called when the system
8064  * wants to unload a module. If the module determines that it can
8065  * be unloaded, then _fini() returns the value returned by
8066  * mod_remove(). Upon successful return from _fini() no other
8067  * routine in the module will be called before _init() is called.
8068  */
8069 int
8070 _fini(void)
8071 {
8072         int rval;
8073 
8074         rval = mod_remove(&modlinkage);
8075         if (rval == DDI_SUCCESS) {
8076                 mac_fini_ops(&ql_ops);
8077         }
8078 
8079         return (rval);
8080 }
8081 
8082 /*
8083  * _info
8084  * Returns information about loadable module.
8085  */
8086 int
8087 _info(struct modinfo *modinfop)
8088 {
8089         return (mod_info(&modlinkage, modinfop));
8090 }