1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 /*
  26  * The following notice accompanied the original version of this file:
  27  *
  28  * BSD LICENSE
  29  *
  30  * Copyright(c) 2007 Intel Corporation. All rights reserved.
  31  * All rights reserved.
  32  *
  33  * Redistribution and use in source and binary forms, with or without
  34  * modification, are permitted provided that the following conditions
  35  * are met:
  36  *
  37  *   * Redistributions of source code must retain the above copyright
  38  *     notice, this list of conditions and the following disclaimer.
  39  *   * Redistributions in binary form must reproduce the above copyright
  40  *     notice, this list of conditions and the following disclaimer in
  41  *     the documentation and/or other materials provided with the
  42  *     distribution.
  43  *   * Neither the name of Intel Corporation nor the names of its
  44  *     contributors may be used to endorse or promote products derived
  45  *     from this software without specific prior written permission.
  46  *
  47  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  48  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  49  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  50  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  51  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  52  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  53  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  54  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  55  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  56  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  57  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  58  */
  59 
  60 /*
  61  * Driver kernel header files
  62  */
  63 #include <sys/conf.h>
  64 #include <sys/ddi.h>
  65 #include <sys/stat.h>
  66 #include <sys/pci.h>
  67 #include <sys/sunddi.h>
  68 #include <sys/modctl.h>
  69 #include <sys/file.h>
  70 #include <sys/cred.h>
  71 #include <sys/byteorder.h>
  72 #include <sys/atomic.h>
  73 #include <sys/modhash.h>
  74 #include <sys/scsi/scsi.h>
  75 #include <sys/ethernet.h>
  76 
  77 /*
  78  * COMSTAR header files
  79  */
  80 #include <sys/stmf_defines.h>
  81 #include <sys/fct_defines.h>
  82 #include <sys/stmf.h>
  83 #include <sys/portif.h>
  84 #include <sys/fct.h>
  85 
  86 /*
  87  * FCoE header files
  88  */
  89 #include <sys/fcoe/fcoe_common.h>
  90 
  91 /*
  92  * Driver's own header files
  93  */
  94 #include "fcoet.h"
  95 #include "fcoet_eth.h"
  96 #include "fcoet_fc.h"
  97 
  98 /*
  99  * static function forward declaration
 100  */
 101 static int fcoet_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
 102 static int fcoet_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
 103 static int fcoet_open(dev_t *devp, int flag, int otype, cred_t *credp);
 104 static int fcoet_close(dev_t dev, int flag, int otype, cred_t *credp);
 105 static int fcoet_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
 106     cred_t *credp, int *rval);
 107 static fct_status_t fcoet_attach_init(fcoet_soft_state_t *ss);
 108 static fct_status_t fcoet_detach_uninit(fcoet_soft_state_t *ss);
 109 static void fcoet_watchdog(void *arg);
 110 static void fcoet_handle_sol_flogi(fcoet_soft_state_t *ss);
 111 static stmf_data_buf_t *fcoet_dbuf_alloc(fct_local_port_t *port,
 112     uint32_t size, uint32_t *pminsize, uint32_t flags);
 113 static void fcoet_dbuf_free(fct_dbuf_store_t *fds, stmf_data_buf_t *dbuf);
 114 static int fcoet_dbuf_init(fcoet_soft_state_t *ss);
 115 static void fcoet_dbuf_destroy(fcoet_soft_state_t *ss);
 116 static uint_t
 117 fcoet_sol_oxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg);
 118 static uint_t
 119 fcoet_unsol_rxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg);
 120 
 121 /*
 122  * Driver identificaton stuff
 123  */
 124 static struct cb_ops fcoet_cb_ops = {
 125         fcoet_open,
 126         fcoet_close,
 127         nodev,
 128         nodev,
 129         nodev,
 130         nodev,
 131         nodev,
 132         fcoet_ioctl,
 133         nodev,
 134         nodev,
 135         nodev,
 136         nochpoll,
 137         ddi_prop_op,
 138         0,
 139         D_MP | D_NEW
 140 };
 141 
 142 static struct dev_ops fcoet_ops = {
 143         DEVO_REV,
 144         0,
 145         nodev,
 146         nulldev,
 147         nulldev,
 148         fcoet_attach,
 149         fcoet_detach,
 150         nodev,
 151         &fcoet_cb_ops,
 152         NULL,
 153         ddi_power,
 154         ddi_quiesce_not_needed
 155 };
 156 
 157 static struct modldrv modldrv = {
 158         &mod_driverops,
 159         FCOET_MOD_NAME,
 160         &fcoet_ops,
 161 };
 162 
 163 static struct modlinkage modlinkage = {
 164         MODREV_1, &modldrv, NULL
 165 };
 166 
 167 /*
 168  * Driver's global variables
 169  */
 170 static kmutex_t  fcoet_mutex;
 171 static void     *fcoet_state = NULL;
 172 
 173 int fcoet_use_ext_log = 1;
 174 static char                              fcoet_provider_name[] = "fcoet";
 175 static struct stmf_port_provider        *fcoet_pp       = NULL;
 176 
 177 /*
 178  * Common loadable module entry points _init, _fini, _info
 179  */
 180 
 181 int
 182 _init(void)
 183 {
 184         int ret;
 185 
 186         ret = ddi_soft_state_init(&fcoet_state, sizeof (fcoet_soft_state_t), 0);
 187         if (ret == 0) {
 188                 fcoet_pp = (stmf_port_provider_t *)
 189                     stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0);
 190                 fcoet_pp->pp_portif_rev = PORTIF_REV_1;
 191                 fcoet_pp->pp_name = fcoet_provider_name;
 192                 if (stmf_register_port_provider(fcoet_pp) != STMF_SUCCESS) {
 193                         stmf_free(fcoet_pp);
 194                         ddi_soft_state_fini(&fcoet_state);
 195                         return (EIO);
 196                 }
 197 
 198                 mutex_init(&fcoet_mutex, 0, MUTEX_DRIVER, 0);
 199                 ret = mod_install(&modlinkage);
 200                 if (ret) {
 201                         (void) stmf_deregister_port_provider(fcoet_pp);
 202                         stmf_free(fcoet_pp);
 203                         mutex_destroy(&fcoet_mutex);
 204                         ddi_soft_state_fini(&fcoet_state);
 205                 }
 206         }
 207 
 208         FCOET_LOG("_init", "exit _init with %x", ret);
 209         return (ret);
 210 }
 211 
 212 int
 213 _fini(void)
 214 {
 215         int ret;
 216 
 217         ret = mod_remove(&modlinkage);
 218         if (ret == 0) {
 219                 (void) stmf_deregister_port_provider(fcoet_pp);
 220                 stmf_free(fcoet_pp);
 221                 mutex_destroy(&fcoet_mutex);
 222                 ddi_soft_state_fini(&fcoet_state);
 223         }
 224 
 225         FCOET_LOG("_fini", "exit _fini with %x", ret);
 226         return (ret);
 227 }
 228 
 229 int
 230 _info(struct modinfo *modinfop)
 231 {
 232         return (mod_info(&modlinkage, modinfop));
 233 }
 234 
 235 /*
 236  * Autoconfiguration entry points: attach, detach, getinfo
 237  */
 238 
 239 static int
 240 fcoet_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 241 {
 242         int                      ret = DDI_FAILURE;
 243         int                      instance;
 244         fcoet_soft_state_t      *ss;
 245 
 246         instance = ddi_get_instance(dip);
 247         FCOET_LOG("fcoet_attach", "get instance %d", instance);
 248 
 249         switch (cmd) {
 250         case DDI_ATTACH:
 251                 ret = ddi_soft_state_zalloc(fcoet_state, instance);
 252                 if (ret != DDI_SUCCESS) {
 253                         return (ret);
 254                 }
 255 
 256                 ss = ddi_get_soft_state(fcoet_state, instance);
 257                 ss->ss_instance = instance;
 258                 ss->ss_dip = dip;
 259 
 260                 ret = fcoet_attach_init(ss);
 261                 if (ret != FCOE_SUCCESS) {
 262                         ddi_soft_state_free(fcoet_state, instance);
 263                         ret = DDI_FAILURE;
 264                 }
 265 
 266                 FCOET_LOG("fcoet_attach", "end with-%x", ret);
 267                 break;
 268 
 269         case DDI_RESUME:
 270                 ret = DDI_SUCCESS;
 271                 break;
 272 
 273         default:
 274                 FCOET_LOG("fcoet_attach", "unspported attach cmd-%x", cmd);
 275                 break;
 276         }
 277 
 278         return (ret);
 279 }
 280 
 281 static int
 282 fcoet_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 283 {
 284         int                      ret = DDI_FAILURE;
 285         int                      fcoe_ret;
 286         int                      instance;
 287         fcoet_soft_state_t      *ss;
 288 
 289         instance = ddi_get_instance(dip);
 290         ss = ddi_get_soft_state(fcoet_state, instance);
 291         if (ss == NULL) {
 292                 return (ret);
 293         }
 294 
 295         switch (cmd) {
 296         case DDI_DETACH:
 297                 fcoe_ret = fcoet_detach_uninit(ss);
 298                 if (fcoe_ret == FCOE_SUCCESS) {
 299                         ret = DDI_SUCCESS;
 300                 }
 301 
 302                 FCOET_LOG("fcoet_detach", "fcoet_detach_uninit end with-%x",
 303                     fcoe_ret);
 304                 break;
 305 
 306         case DDI_SUSPEND:
 307                 ret = DDI_SUCCESS;
 308                 break;
 309 
 310         default:
 311                 FCOET_LOG("fcoet_detach", "unsupported detach cmd-%x", cmd);
 312                 break;
 313         }
 314 
 315         return (ret);
 316 }
 317 
 318 /*
 319  * Device access entry points
 320  */
 321 static int
 322 fcoet_open(dev_t *devp, int flag, int otype, cred_t *credp)
 323 {
 324         int                      instance;
 325         fcoet_soft_state_t      *ss;
 326 
 327         if (otype != OTYP_CHR) {
 328                 return (EINVAL);
 329         }
 330 
 331         /*
 332          * Since this is for debugging only, only allow root to issue ioctl now
 333          */
 334         if (drv_priv(credp)) {
 335                 return (EPERM);
 336         }
 337 
 338         instance = (int)getminor(*devp);
 339         ss = ddi_get_soft_state(fcoet_state, instance);
 340         if (ss == NULL) {
 341                 return (ENXIO);
 342         }
 343 
 344         mutex_enter(&ss->ss_ioctl_mutex);
 345         if (ss->ss_ioctl_flags & FCOET_IOCTL_FLAG_EXCL) {
 346                 /*
 347                  * It is already open for exclusive access.
 348                  * So shut the door on this caller.
 349                  */
 350                 mutex_exit(&ss->ss_ioctl_mutex);
 351                 return (EBUSY);
 352         }
 353 
 354         if (flag & FEXCL) {
 355                 if (ss->ss_ioctl_flags & FCOET_IOCTL_FLAG_OPEN) {
 356                         /*
 357                          * Exclusive operation not possible
 358                          * as it is already opened
 359                          */
 360                         mutex_exit(&ss->ss_ioctl_mutex);
 361                         return (EBUSY);
 362                 }
 363                 ss->ss_ioctl_flags |= FCOET_IOCTL_FLAG_EXCL;
 364         }
 365         ss->ss_ioctl_flags |= FCOET_IOCTL_FLAG_OPEN;
 366         mutex_exit(&ss->ss_ioctl_mutex);
 367 
 368         return (0);
 369 }
 370 
 371 /* ARGSUSED */
 372 static int
 373 fcoet_close(dev_t dev, int flag, int otype, cred_t *credp)
 374 {
 375         int                      instance;
 376         fcoet_soft_state_t      *ss;
 377 
 378         if (otype != OTYP_CHR) {
 379                 return (EINVAL);
 380         }
 381 
 382         instance = (int)getminor(dev);
 383         ss = ddi_get_soft_state(fcoet_state, instance);
 384         if (ss == NULL) {
 385                 return (ENXIO);
 386         }
 387 
 388         mutex_enter(&ss->ss_ioctl_mutex);
 389         if ((ss->ss_ioctl_flags & FCOET_IOCTL_FLAG_OPEN) == 0) {
 390                 mutex_exit(&ss->ss_ioctl_mutex);
 391                 return (ENODEV);
 392         }
 393 
 394         /*
 395          * It looks there's one hole here, maybe there could several concurrent
 396          * shareed open session, but we never check this case.
 397          * But it will not hurt too much, disregard it now.
 398          */
 399         ss->ss_ioctl_flags &= ~FCOET_IOCTL_FLAG_MASK;
 400         mutex_exit(&ss->ss_ioctl_mutex);
 401 
 402         return (0);
 403 }
 404 
 405 /* ARGSUSED */
 406 static int
 407 fcoet_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
 408     cred_t *credp, int *rval)
 409 {
 410         fcoet_soft_state_t      *ss;
 411         int              ret = 0;
 412 
 413         if (drv_priv(credp) != 0) {
 414                 return (EPERM);
 415         }
 416 
 417         ss = ddi_get_soft_state(fcoet_state, (int32_t)getminor(dev));
 418         if (ss == NULL) {
 419                 return (ENXIO);
 420         }
 421 
 422         switch (cmd) {
 423         default:
 424                 FCOET_LOG("fcoet_ioctl", "ioctl-0x%02X", cmd);
 425                 ret = ENOTTY;
 426                 break;
 427         }
 428 
 429         *rval = ret;
 430         return (ret);
 431 }
 432 
 433 static fct_status_t
 434 fcoet_attach_init(fcoet_soft_state_t *ss)
 435 {
 436         fcoe_client_t            client_fcoet;
 437         fcoe_port_t             *eport;
 438         fct_local_port_t        *port;
 439         fct_dbuf_store_t        *fds;
 440         char                     taskq_name[FCOET_TASKQ_NAME_LEN];
 441         int                      ret;
 442 
 443         /*
 444          * FCoE (fcoe is fcoet's dependent driver)
 445          * First we need register fcoet to FCoE as one client
 446          */
 447         client_fcoet.ect_eport_flags = EPORT_FLAG_TGT_MODE |
 448             EPORT_FLAG_IS_DIRECT_P2P;
 449         client_fcoet.ect_max_fc_frame_size = 2136;
 450         client_fcoet.ect_private_frame_struct_size = sizeof (fcoet_frame_t);
 451         client_fcoet.ect_rx_frame = fcoet_rx_frame;
 452         client_fcoet.ect_port_event = fcoet_port_event;
 453         client_fcoet.ect_release_sol_frame = fcoet_release_sol_frame;
 454         client_fcoet.ect_client_port_struct = ss;
 455         client_fcoet.ect_fcoe_ver = FCOE_VER_NOW;
 456         FCOET_LOG(__FUNCTION__, "version: %x %x", FCOE_VER_NOW, fcoe_ver_now);
 457         ret = ddi_prop_get_int(DDI_DEV_T_ANY, ss->ss_dip,
 458             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "mac_id", -1);
 459         if (ret == -1) {
 460                 FCOET_LOG("fcoet_attach_init", "get mac_id failed");
 461                 return (DDI_FAILURE);
 462         } else {
 463                 client_fcoet.ect_channelid = ret;
 464         }
 465         FCOET_LOG("fcoet_attach_init", "channel_id is %d",
 466             client_fcoet.ect_channelid);
 467 
 468         /*
 469          * It's FCoE's responsiblity to initialize eport's all elements
 470          */
 471         eport = fcoe_register_client(&client_fcoet);
 472         if (eport == NULL) {
 473                 goto fail_register_client;
 474         }
 475 
 476         /*
 477          * Now it's time to register local port to FCT
 478          */
 479         if (fcoet_dbuf_init(ss) != FCOE_SUCCESS) {
 480                 goto fail_init_dbuf;
 481         }
 482 
 483         fds = (fct_dbuf_store_t *)fct_alloc(FCT_STRUCT_DBUF_STORE, 0, 0);
 484         if (fds == NULL) {
 485                 goto fail_alloc_dbuf;
 486         } else {
 487                 fds->fds_alloc_data_buf = fcoet_dbuf_alloc;
 488                 fds->fds_free_data_buf = fcoet_dbuf_free;
 489                 fds->fds_fca_private = (void *)ss;
 490         }
 491 
 492         port = (fct_local_port_t *)fct_alloc(FCT_STRUCT_LOCAL_PORT, 0, 0);
 493         if (port == NULL) {
 494                 goto fail_alloc_port;
 495         } else {
 496                 /*
 497                  * Do ss's initialization now
 498                  */
 499                 (void) snprintf(ss->ss_alias, sizeof (ss->ss_alias), "fcoet%d",
 500                     ss->ss_instance);
 501                 ret = ddi_create_minor_node(ss->ss_dip, "admin",
 502                     S_IFCHR, ss->ss_instance, DDI_NT_STMF_PP, 0);
 503                 if (ret != DDI_SUCCESS) {
 504                         goto fail_minor_node;
 505                 }
 506 
 507                 ss->ss_state = FCT_STATE_OFFLINE;
 508                 ss->ss_state_not_acked = 1;
 509                 ss->ss_flags = 0;
 510                 ss->ss_port = port;
 511                 ss->ss_eport = eport;
 512                 FCOE_SET_DEFAULT_FPORT_ADDR(eport->eport_efh_dst);
 513 
 514                 ss->ss_rportid_in_dereg = 0;
 515                 ss->ss_rport_dereg_state = 0;
 516 
 517                 ss->ss_next_sol_oxid = 0xFFFF;
 518                 ss->ss_next_unsol_rxid = 0xFFFF;
 519                 ss->ss_sol_oxid_hash = mod_hash_create_idhash(
 520                     "ss_sol_oxid_hash", FCOET_SOL_HASH_SIZE,
 521                     mod_hash_null_valdtor);
 522                 ss->ss_unsol_rxid_hash = mod_hash_create_idhash(
 523                     "ss_unsol_rxid_hash", FCOET_SOL_HASH_SIZE,
 524                     mod_hash_null_valdtor);
 525 
 526                 ss->ss_watch_count = 0;
 527                 mutex_init(&ss->ss_watch_mutex, 0, MUTEX_DRIVER, 0);
 528                 cv_init(&ss->ss_watch_cv, NULL, CV_DRIVER, NULL);
 529 
 530                 list_create(&ss->ss_abort_xchg_list, sizeof (fcoet_exchange_t),
 531                     offsetof(fcoet_exchange_t, xch_abort_node));
 532 
 533                 ss->ss_sol_flogi = NULL;
 534                 ss->ss_sol_flogi_state = SFS_WAIT_LINKUP;
 535 
 536                 bzero(&ss->ss_link_info, sizeof (fct_link_info_t));
 537 
 538                 ss->ss_ioctl_flags = 0;
 539                 mutex_init(&ss->ss_ioctl_mutex, 0, MUTEX_DRIVER, 0);
 540 
 541                 ss->ss_change_state_flags = 0;
 542         }
 543 
 544         /*
 545          * Do port's initialization
 546          *
 547          * port_fct_private and port_lport have been initialized by fct_alloc
 548          */
 549         port->port_fca_private = ss;
 550         port->port_fca_version = FCT_FCA_MODREV_1;
 551         bcopy(ss->ss_eport->eport_nodewwn, port->port_nwwn, 8);
 552         bcopy(ss->ss_eport->eport_portwwn, port->port_pwwn, 8);
 553         port->port_default_alias = ss->ss_alias;
 554         port->port_sym_node_name = NULL;
 555         port->port_sym_port_name = NULL;
 556 
 557         port->port_pp = fcoet_pp;
 558 
 559         port->port_hard_address = 0;
 560         port->port_max_logins = FCOET_MAX_LOGINS;
 561         port->port_max_xchges = FCOET_MAX_XCHGES;
 562         port->port_fca_fcp_cmd_size = sizeof (fcoet_exchange_t);
 563         port->port_fca_rp_private_size = 0;
 564         port->port_fca_sol_els_private_size = sizeof (fcoet_exchange_t);
 565         port->port_fca_sol_ct_private_size = sizeof (fcoet_exchange_t);
 566 
 567         port->port_fca_abort_timeout = 5 * 1000;     /* 5 seconds */
 568         port->port_fds = fds;
 569 
 570         port->port_get_link_info = fcoet_get_link_info;
 571         port->port_register_remote_port = fcoet_register_remote_port;
 572         port->port_deregister_remote_port = fcoet_deregister_remote_port;
 573         port->port_send_cmd = fcoet_send_cmd;
 574         port->port_xfer_scsi_data = fcoet_xfer_scsi_data;
 575         port->port_send_cmd_response = fcoet_send_cmd_response;
 576         port->port_abort_cmd = fcoet_abort_cmd;
 577         port->port_ctl = fcoet_ctl;
 578         port->port_flogi_xchg = fcoet_do_flogi;
 579         port->port_populate_hba_details = fcoet_populate_hba_fru_details;
 580         if (fct_register_local_port(port) != FCT_SUCCESS) {
 581                 goto fail_register_port;
 582         }
 583 
 584         /*
 585          * Start watchdog thread
 586          */
 587         (void) snprintf(taskq_name, sizeof (taskq_name),
 588             "stmf_fct_fcoet_%d_taskq", ss->ss_instance);
 589         if ((ss->ss_watchdog_taskq = ddi_taskq_create(NULL,
 590             taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
 591                 goto fail_create_taskq;
 592         }
 593 
 594         atomic_and_32(&ss->ss_flags, ~SS_FLAG_TERMINATE_WATCHDOG);
 595         (void) ddi_taskq_dispatch(ss->ss_watchdog_taskq,
 596             fcoet_watchdog, ss, DDI_SLEEP);
 597         while ((ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) == 0) {
 598                 delay(10);
 599         }
 600 
 601         ddi_report_dev(ss->ss_dip);
 602         return (DDI_SUCCESS);
 603 
 604 fail_create_taskq:
 605         if (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
 606                 atomic_or_32(&ss->ss_flags, SS_FLAG_TERMINATE_WATCHDOG);
 607                 cv_broadcast(&ss->ss_watch_cv);
 608                 while (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
 609                         delay(10);
 610                 }
 611         }
 612 
 613         ddi_taskq_destroy(ss->ss_watchdog_taskq);
 614         FCOET_LOG("fcoet_attach_init", "fail_register_port");
 615 
 616 fail_register_port:
 617         mutex_destroy(&ss->ss_ioctl_mutex);
 618         mutex_destroy(&ss->ss_watch_mutex);
 619         cv_destroy(&ss->ss_watch_cv);
 620         mod_hash_destroy_hash(ss->ss_sol_oxid_hash);
 621         mod_hash_destroy_hash(ss->ss_unsol_rxid_hash);
 622         list_destroy(&ss->ss_abort_xchg_list);
 623         FCOET_LOG("fcoet_attach_init", "fail_create_taskq");
 624 
 625 fail_minor_node:
 626         fct_free(port);
 627         FCOET_LOG("fcoet_attach_init", "fail_minor_node");
 628 
 629 fail_alloc_port:
 630         fct_free(fds);
 631         FCOET_LOG("fcoet_attach_init", "fail_alloc_port");
 632 
 633 fail_alloc_dbuf:
 634         fcoet_dbuf_destroy(ss);
 635         FCOET_LOG("fcoet_attach_init", "fail_alloc_dbuf");
 636 
 637 fail_init_dbuf:
 638         ss->ss_eport->eport_deregister_client(ss->ss_eport);
 639         FCOET_LOG("fcoet_attach_init", "fail_init_dbuf");
 640 
 641 fail_register_client:
 642         FCOET_LOG("fcoet_attach_init", "fail_register_client");
 643         return (DDI_FAILURE);
 644 }
 645 
 646 static fct_status_t
 647 fcoet_detach_uninit(fcoet_soft_state_t *ss)
 648 {
 649         if ((ss->ss_state != FCT_STATE_OFFLINE) ||
 650             ss->ss_state_not_acked) {
 651                 return (FCOE_FAILURE);
 652         }
 653 
 654         /*
 655          * Avoid modunload before running fcinfo remove-target-port
 656          */
 657         if (ss->ss_eport != NULL &&
 658             ss->ss_eport->eport_flags & EPORT_FLAG_MAC_IN_USE) {
 659                 return (FCOE_FAILURE);
 660         }
 661 
 662         if (ss->ss_port == NULL) {
 663                 return (FCOE_SUCCESS);
 664         }
 665 
 666         ss->ss_sol_oxid_hash_empty = 1;
 667         ss->ss_unsol_rxid_hash_empty = 1;
 668         mod_hash_walk(ss->ss_sol_oxid_hash, fcoet_sol_oxid_hash_empty, ss);
 669         mod_hash_walk(ss->ss_unsol_rxid_hash, fcoet_unsol_rxid_hash_empty, ss);
 670         if ((!ss->ss_sol_oxid_hash_empty) || (!ss->ss_unsol_rxid_hash_empty)) {
 671                 return (FCOE_FAILURE);
 672         }
 673 
 674         /*
 675          * We need offline the port manually, before we want to detach it
 676          * or it will not succeed.
 677          */
 678         if (fct_deregister_local_port(ss->ss_port) != FCT_SUCCESS) {
 679                 FCOET_LOG("fcoet_detach_uninit",
 680                     "fct_deregister_local_port failed");
 681                 return (FCOE_FAILURE);
 682         }
 683 
 684         /*
 685          * Stop watchdog
 686          */
 687         if (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
 688                 atomic_or_32(&ss->ss_flags, SS_FLAG_TERMINATE_WATCHDOG);
 689                 cv_broadcast(&ss->ss_watch_cv);
 690                 while (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
 691                         delay(10);
 692                 }
 693         }
 694 
 695         ddi_taskq_destroy(ss->ss_watchdog_taskq);
 696 
 697         /*
 698          * Release all resources
 699          */
 700         mutex_destroy(&ss->ss_ioctl_mutex);
 701         mutex_destroy(&ss->ss_watch_mutex);
 702         cv_destroy(&ss->ss_watch_cv);
 703         mod_hash_destroy_hash(ss->ss_sol_oxid_hash);
 704         mod_hash_destroy_hash(ss->ss_unsol_rxid_hash);
 705         list_destroy(&ss->ss_abort_xchg_list);
 706 
 707         fct_free(ss->ss_port->port_fds);
 708         fct_free(ss->ss_port);
 709         ss->ss_port = NULL;
 710 
 711         fcoet_dbuf_destroy(ss);
 712 
 713         if (ss->ss_eport != NULL &&
 714             ss->ss_eport->eport_deregister_client != NULL) {
 715                 ss->ss_eport->eport_deregister_client(ss->ss_eport);
 716         }
 717         ddi_soft_state_free(fcoet_state, ss->ss_instance);
 718         return (FCOE_SUCCESS);
 719 }
 720 
 721 static void
 722 fcoet_watchdog(void *arg)
 723 {
 724         fcoet_soft_state_t      *ss = (fcoet_soft_state_t *)arg;
 725         clock_t                  tmp_delay = 0;
 726         fcoet_exchange_t        *xchg, *xchg_next;
 727 
 728         FCOET_LOG("fcoet_watchdog", "fcoet_soft_state is %p", ss);
 729 
 730         mutex_enter(&ss->ss_watch_mutex);
 731         atomic_or_32(&ss->ss_flags, SS_FLAG_WATCHDOG_RUNNING);
 732         tmp_delay = STMF_SEC2TICK(1)/2;
 733 
 734         while ((ss->ss_flags & SS_FLAG_TERMINATE_WATCHDOG) == 0) {
 735                 ss->ss_watch_count++;
 736 
 737                 if (ss->ss_sol_flogi_state != SFS_FLOGI_DONE) {
 738                         fcoet_handle_sol_flogi(ss);
 739                 }
 740                 for (xchg = list_head(&ss->ss_abort_xchg_list); xchg; ) {
 741                         xchg_next = list_next(&ss->ss_abort_xchg_list, xchg);
 742                         if (xchg->xch_ref == 0) {
 743                                 list_remove(&ss->ss_abort_xchg_list, xchg);
 744                                 mutex_exit(&ss->ss_watch_mutex);
 745                                 /* xchg abort done */
 746                                 if (xchg->xch_dbuf_num) {
 747                                         kmem_free((void*)xchg->xch_dbufs,
 748                                             xchg->xch_dbuf_num *
 749                                             sizeof (void *));
 750                                         xchg->xch_dbufs = NULL;
 751                                         xchg->xch_dbuf_num = 0;
 752                                 }
 753                                 fct_cmd_fca_aborted(xchg->xch_cmd,
 754                                     FCT_ABORT_SUCCESS, FCT_IOF_FCA_DONE);
 755                                 mutex_enter(&ss->ss_watch_mutex);
 756                         }
 757                         xchg = xchg_next;
 758                 }
 759 
 760                 atomic_or_32(&ss->ss_flags, SS_FLAG_DOG_WAITING);
 761                 (void) cv_reltimedwait(&ss->ss_watch_cv, &ss->ss_watch_mutex,
 762                     (clock_t)tmp_delay, TR_CLOCK_TICK);
 763                 atomic_and_32(&ss->ss_flags, ~SS_FLAG_DOG_WAITING);
 764         }
 765 
 766         /*
 767          * Ensure no ongoing FLOGI, before terminate the watchdog
 768          */
 769         if (ss->ss_sol_flogi) {
 770                 fcoet_clear_sol_exchange(ss->ss_sol_flogi);
 771                 fct_free(ss->ss_sol_flogi->xch_cmd);
 772                 ss->ss_sol_flogi = NULL;
 773         }
 774 
 775         atomic_and_32(&ss->ss_flags, ~SS_FLAG_WATCHDOG_RUNNING);
 776         mutex_exit(&ss->ss_watch_mutex);
 777 }
 778 
 779 static void
 780 fcoet_handle_sol_flogi(fcoet_soft_state_t *ss)
 781 {
 782         clock_t                 twosec = STMF_SEC2TICK(2);
 783 
 784 check_state_again:
 785         if (ss->ss_flags & SS_FLAG_PORT_DISABLED) {
 786                 ss->ss_sol_flogi_state = SFS_WAIT_LINKUP;
 787         }
 788 
 789         switch (ss->ss_sol_flogi_state) {
 790         case SFS_WAIT_LINKUP:
 791                 if (ss->ss_sol_flogi) {
 792                         if (ss->ss_sol_flogi->xch_ref == 0) {
 793                                 fcoet_clear_sol_exchange(ss->ss_sol_flogi);
 794                                 fct_free(ss->ss_sol_flogi->xch_cmd);
 795                                 ss->ss_sol_flogi = NULL;
 796                         }
 797                 }
 798                 break;
 799 
 800         case SFS_FLOGI_INIT:
 801                 if (ss->ss_sol_flogi) {
 802                         /*
 803                          * wait for the response to finish
 804                          */
 805                         ss->ss_sol_flogi_state = SFS_CLEAR_FLOGI;
 806                         break;
 807                 }
 808                 fcoet_send_sol_flogi(ss);
 809                 ss->ss_sol_flogi_state++;
 810                 break;
 811 
 812         case SFS_FLOGI_CHECK_TIMEOUT:
 813                 if ((ss->ss_sol_flogi->xch_start_time + twosec) <
 814                     ddi_get_lbolt()) {
 815                         ss->ss_sol_flogi_state++;
 816                 }
 817                 break;
 818 
 819         case SFS_ABTS_INIT:
 820                 fcoet_send_sol_abts(ss->ss_sol_flogi);
 821                 ss->ss_sol_flogi_state++;
 822                 break;
 823 
 824         case SFS_CLEAR_FLOGI:
 825                 if (ss->ss_sol_flogi) {
 826                         if (ss->ss_sol_flogi->xch_ref) {
 827                                 break;
 828                         }
 829                         fcoet_clear_sol_exchange(ss->ss_sol_flogi);
 830                         fct_free(ss->ss_sol_flogi->xch_cmd);
 831                         ss->ss_sol_flogi = NULL;
 832                 }
 833                 ss->ss_sol_flogi_state = SFS_FLOGI_INIT;
 834                 goto check_state_again;
 835 
 836         case SFS_FLOGI_ACC:
 837                 ss->ss_sol_flogi_state++;
 838                 goto check_state_again;
 839 
 840         case SFS_FLOGI_DONE:
 841                 if (!(ss->ss_flags & SS_FLAG_PORT_DISABLED) &&
 842                     ss->ss_sol_flogi) {
 843                         fcoet_clear_sol_exchange(ss->ss_sol_flogi);
 844                         fct_free(ss->ss_sol_flogi->xch_cmd);
 845                         ss->ss_sol_flogi = NULL;
 846                 }
 847 
 848                 /*
 849                  * We'd better to offline it first, and delay 0.1 seconds,
 850                  * before we say it's on again.
 851                  */
 852                 fct_handle_event(ss->ss_port,
 853                     FCT_EVENT_LINK_DOWN, 0, NULL);
 854                 delay(STMF_SEC2TICK(1)/10);
 855                 fct_handle_event(ss->ss_port,
 856                     FCT_EVENT_LINK_UP, 0, NULL);
 857                 break;
 858 
 859         default:
 860                 ASSERT(0);
 861                 break;
 862         }
 863 }
 864 
 865 /* ARGSUSED */
 866 static int
 867 fcoet_dbuf_init(fcoet_soft_state_t *ss)
 868 {
 869         return (FCOE_SUCCESS);
 870 }
 871 
 872 /* ARGSUSED */
 873 static void
 874 fcoet_dbuf_destroy(fcoet_soft_state_t *ss)
 875 {
 876 
 877 }
 878 
 879 /* ARGSUSED */
 880 static stmf_data_buf_t *
 881 fcoet_dbuf_alloc(fct_local_port_t *port, uint32_t size, uint32_t *pminsize,
 882     uint32_t flags)
 883 {
 884         stmf_data_buf_t *dbuf;
 885         int              add_size;
 886         int              sge_num;
 887         int              sge_size;
 888         int              idx;
 889         int              ii;
 890         void            *netb;
 891         uint8_t         *fc_buf;
 892         fcoet_soft_state_t      *ss =
 893             (fcoet_soft_state_t *)port->port_fca_private;
 894 
 895         if (size > FCOET_MAX_DBUF_LEN) {
 896                 if (*pminsize > FCOET_MAX_DBUF_LEN) {
 897                         return (NULL);
 898                 }
 899 
 900                 size = FCOET_MAX_DBUF_LEN;
 901         }
 902 
 903         sge_num = (size - 1) / ss->ss_fcp_data_payload_size + 1;
 904         add_size = (sge_num - 1) * sizeof (struct stmf_sglist_ent) +
 905             sge_num * sizeof (mblk_t *);
 906         dbuf = stmf_alloc(STMF_STRUCT_DATA_BUF, add_size, 0);
 907         if (dbuf == NULL) {
 908                 return (NULL);
 909         }
 910         dbuf->db_buf_size = size;
 911         dbuf->db_data_size = size;
 912         dbuf->db_sglist_length = 0;
 913         dbuf->db_flags |= DB_DONT_REUSE;
 914         FCOET_SET_SEG_NUM(dbuf, sge_num);
 915 
 916         /*
 917          * Initialize non-last sg entries
 918          */
 919         for (idx = 0; idx < sge_num - 1; idx++) {
 920                 sge_size = ss->ss_fcp_data_payload_size;
 921                 netb = ss->ss_eport->eport_alloc_netb(
 922                     ss->ss_eport, sizeof (fcoe_fc_frame_header_t) +
 923                     sge_size, &fc_buf);
 924                 if (netb == NULL) {
 925                         for (ii = 0; ii < idx; ii++) {
 926                                 ss->ss_eport->eport_free_netb(
 927                                     FCOET_GET_NETB(dbuf, ii));
 928                         }
 929                         stmf_free(dbuf);
 930                         FCOET_LOG("fcoe_dbuf_alloc", "no netb");
 931                         return (NULL);
 932                 }
 933                 FCOET_SET_NETB(dbuf, idx, netb);
 934                 dbuf->db_sglist[idx].seg_addr = fc_buf +
 935                     sizeof (fcoe_fc_frame_header_t);
 936                 dbuf->db_sglist[idx].seg_length = sge_size;
 937         }
 938 
 939         /*
 940          * Initialize the last sg entry
 941          */
 942         if (size % ss->ss_fcp_data_payload_size) {
 943                 sge_size = P2ROUNDUP(size % ss->ss_fcp_data_payload_size, 4);
 944         } else {
 945                 sge_size = ss->ss_fcp_data_payload_size;
 946         }
 947 
 948         netb = ss->ss_eport->eport_alloc_netb(
 949             ss->ss_eport,
 950             sizeof (fcoe_fc_frame_header_t) +
 951             sge_size, &fc_buf);
 952         if (netb == NULL) {
 953                 for (ii = 0; ii < idx; ii++) {
 954                         ss->ss_eport->eport_free_netb(
 955                             FCOET_GET_NETB(dbuf, ii));
 956                 }
 957                 stmf_free(dbuf);
 958                 FCOET_LOG("fcoe_dbuf_alloc", "no netb");
 959                 return (NULL);
 960         }
 961 
 962         FCOET_SET_NETB(dbuf, idx, netb);
 963         dbuf->db_sglist[idx].seg_addr = fc_buf +
 964             sizeof (fcoe_fc_frame_header_t);
 965         dbuf->db_sglist[idx].seg_length = sge_size;
 966 
 967         /*
 968          * Let COMSTAR know how many sg entries we will use
 969          */
 970         dbuf->db_sglist_length = idx + 1;
 971 
 972         return (dbuf);
 973 }
 974 
 975 static void
 976 fcoet_dbuf_free(fct_dbuf_store_t *fds, stmf_data_buf_t *dbuf)
 977 {
 978         int     idx;
 979         fcoet_soft_state_t      *ss =
 980             (fcoet_soft_state_t *)fds->fds_fca_private;
 981 
 982         for (idx = 0; idx < FCOET_GET_SEG_NUM(dbuf); idx++) {
 983                 if (FCOET_GET_NETB(dbuf, idx)) {
 984                         ss->ss_eport->eport_free_netb(
 985                             FCOET_GET_NETB(dbuf, idx));
 986                 }
 987         }
 988 
 989         stmf_free(dbuf);
 990 }
 991 
 992 /*
 993  * We should have initialized fcoe_frame_t before
 994  */
 995 void
 996 fcoet_init_tfm(fcoe_frame_t *frm, fcoet_exchange_t *xch)
 997 {
 998         FRM2TFM(frm)->tfm_fcoe_frame = frm;
 999         FRM2TFM(frm)->tfm_xch = xch;
1000         FRM2TFM(frm)->tfm_seq = NULL;
1001 }
1002 
1003 /* ARGSUSED */
1004 static uint_t
1005 fcoet_sol_oxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
1006 {
1007         fcoet_soft_state_t      *ss = (fcoet_soft_state_t *)arg;
1008 
1009         ss->ss_sol_oxid_hash_empty = 0;
1010         FCOET_LOG("fcoet_sol_oxid_hash_empty", "one ongoing xch: %p", val);
1011         return (MH_WALK_CONTINUE);
1012 }
1013 
1014 /* ARGSUSED */
1015 static uint_t
1016 fcoet_unsol_rxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
1017 {
1018         fcoet_soft_state_t      *ss = (fcoet_soft_state_t *)arg;
1019 
1020         ss->ss_sol_oxid_hash_empty = 0;
1021         FCOET_LOG("fcoet_unsol_rxid_hash_empty", "one ongoing xch: %p", val);
1022         return (MH_WALK_CONTINUE);
1023 }
1024 
1025 /* ARGSUSED */
1026 void
1027 fcoet_modhash_find_cb(mod_hash_key_t key, mod_hash_val_t val)
1028 {
1029         ASSERT(val != NULL);
1030         fcoet_exchange_t *xch = (fcoet_exchange_t *)val;
1031         FCOET_BUSY_XCHG(xch);
1032 }