1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2013 Nexenta Inc. All rights reserved.
14 */
15
16 /* Based on the NetBSD virtio driver by Minoura Makoto. */
17 /*
18 * Copyright (c) 2010 Minoura Makoto.
19 * All rights reserved.
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 * 1. Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 *
30 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
31 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
33 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
34 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
35 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
39 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 */
41
42 #include <sys/types.h>
43 #include <sys/errno.h>
44 #include <sys/param.h>
45 #include <sys/stropts.h>
46 #include <sys/stream.h>
47 #include <sys/strsubr.h>
48 #include <sys/kmem.h>
49 #include <sys/conf.h>
50 #include <sys/devops.h>
51 #include <sys/ksynch.h>
52 #include <sys/stat.h>
53 #include <sys/modctl.h>
54 #include <sys/debug.h>
55 #include <sys/pci.h>
56 #include <sys/ethernet.h>
57
58 #define VLAN_TAGSZ 4
59
60 #include <sys/dlpi.h>
61 #include <sys/taskq.h>
62 #include <sys/cyclic.h>
63
64 #include <sys/pattr.h>
65 #include <sys/strsun.h>
66
67 #include <sys/random.h>
68 #include <sys/sysmacros.h>
69 #include <sys/stream.h>
70
71 #include <sys/mac.h>
72 #include <sys/mac_provider.h>
73 #include <sys/mac_ether.h>
74
75 #include "virtiovar.h"
76 #include "virtioreg.h"
77
78 #if !defined(__packed)
79 #define __packed __attribute__((packed))
80 #endif /* __packed */
81
82 /* Configuration registers */
83 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
84 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
85
86 /* Feature bits */
87 #define VIRTIO_NET_F_CSUM (1 << 0) /* Host handles pkts w/ partial csum */
88 #define VIRTIO_NET_F_GUEST_CSUM (1 << 1) /* Guest handles pkts w/ part csum */
89 #define VIRTIO_NET_F_MAC (1 << 5) /* Host has given MAC address. */
90 #define VIRTIO_NET_F_GSO (1 << 6) /* Host handles pkts w/ any GSO type */
91 #define VIRTIO_NET_F_GUEST_TSO4 (1 << 7) /* Guest can handle TSOv4 in. */
92 #define VIRTIO_NET_F_GUEST_TSO6 (1 << 8) /* Guest can handle TSOv6 in. */
93 #define VIRTIO_NET_F_GUEST_ECN (1 << 9) /* Guest can handle TSO[6] w/ ECN in */
94 #define VIRTIO_NET_F_GUEST_UFO (1 << 10) /* Guest can handle UFO in. */
95 #define VIRTIO_NET_F_HOST_TSO4 (1 << 11) /* Host can handle TSOv4 in. */
96 #define VIRTIO_NET_F_HOST_TSO6 (1 << 12) /* Host can handle TSOv6 in. */
97 #define VIRTIO_NET_F_HOST_ECN (1 << 13) /* Host can handle TSO[6] w/ ECN in */
98 #define VIRTIO_NET_F_HOST_UFO (1 << 14) /* Host can handle UFO in. */
99 #define VIRTIO_NET_F_MRG_RXBUF (1 << 15) /* Host can merge receive buffers. */
100 #define VIRTIO_NET_F_STATUS (1 << 16) /* Config.status available */
101 #define VIRTIO_NET_F_CTRL_VQ (1 << 17) /* Control channel available */
102 #define VIRTIO_NET_F_CTRL_RX (1 << 18) /* Control channel RX mode support */
103 #define VIRTIO_NET_F_CTRL_VLAN (1 << 19) /* Control channel VLAN filtering */
104 #define VIRTIO_NET_F_CTRL_RX_EXTRA (1 << 20) /* Extra RX mode control support */
105
106 /* Status */
107 #define VIRTIO_NET_S_LINK_UP 1
108
109 /* Packet header structure */
110 struct virtio_net_hdr {
111 uint8_t flags;
112 uint8_t gso_type;
113 uint16_t hdr_len;
114 uint16_t gso_size;
115 uint16_t csum_start;
116 uint16_t csum_offset;
117 };
118
119 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
120 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
121 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
122 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
123 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
124 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
125
126
127 /* Control virtqueue */
128 struct virtio_net_ctrl_cmd {
129 uint8_t class;
130 uint8_t command;
131 } __packed;
132
133 #define VIRTIO_NET_CTRL_RX 0
134 #define VIRTIO_NET_CTRL_RX_PROMISC 0
135 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1
136
137 #define VIRTIO_NET_CTRL_MAC 1
138 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
139
140 #define VIRTIO_NET_CTRL_VLAN 2
141 #define VIRTIO_NET_CTRL_VLAN_ADD 0
142 #define VIRTIO_NET_CTRL_VLAN_DEL 1
143
144 struct virtio_net_ctrl_status {
145 uint8_t ack;
146 } __packed;
147
148 struct virtio_net_ctrl_rx {
149 uint8_t onoff;
150 } __packed;
151
152 struct virtio_net_ctrl_mac_tbl {
153 uint32_t nentries;
154 uint8_t macs[][ETHERADDRL];
155 } __packed;
156
157 struct virtio_net_ctrl_vlan {
158 uint16_t id;
159 } __packed;
160
161 static int vioif_quiesce(dev_info_t *);
162 static int vioif_attach(dev_info_t *, ddi_attach_cmd_t);
163 static int vioif_detach(dev_info_t *, ddi_detach_cmd_t);
164
165 DDI_DEFINE_STREAM_OPS(vioif_ops,
166 nulldev, /* identify */
167 nulldev, /* probe */
168 vioif_attach, /* attach */
169 vioif_detach, /* detach */
170 nodev, /* reset */
171 NULL, /* cb_ops */
172 D_MP, /* bus_ops */
173 NULL, /* power */
174 vioif_quiesce /* quiesce */
175 );
176
177 static char vioif_ident[] = "VirtIO ethernet driver";
178
179 /* Standard Module linkage initialization for a Streams driver */
180 extern struct mod_ops mod_driverops;
181
182 static struct modldrv modldrv = {
183 &mod_driverops, /* Type of module. This one is a driver */
184 vioif_ident, /* short description */
185 &vioif_ops /* driver specific ops */
186 };
187
188 static struct modlinkage modlinkage = {
189 MODREV_1,
190 {
191 (void *)&modldrv,
192 NULL,
193 },
194 };
195
196 ddi_device_acc_attr_t vioif_attr = {
197 DDI_DEVICE_ATTR_V0,
198 DDI_NEVERSWAP_ACC, /* virtio is always native byte order */
199 DDI_STORECACHING_OK_ACC,
200 DDI_DEFAULT_ACC
201 };
202
203 /*
204 * A mapping represents a binding for a single buffer that is contiguous in the
205 * virtual address space.
206 */
207 struct vioif_buf_mapping {
208 caddr_t vbm_buf;
209 ddi_dma_handle_t vbm_dmah;
210 ddi_acc_handle_t vbm_acch;
211 ddi_dma_cookie_t vbm_dmac;
212 unsigned int vbm_ncookies;
213 };
214
215 /*
216 * Rx buffers can be loaned upstream, so the code has
217 * to allocate them dynamically.
218 */
219 struct vioif_rx_buf {
220 struct vioif_softc *rb_sc;
221 frtn_t rb_frtn;
222
223 struct vioif_buf_mapping rb_mapping;
224 };
225
226 /*
227 * Tx buffers have two mapping types. One, "inline", is pre-allocated and is
228 * used to hold the virtio_net_header. Small packets also get copied there, as
229 * it's faster then mapping them. Bigger packets get mapped using the "external"
230 * mapping array. An array is used, because a packet may consist of muptiple
231 * fragments, so each fragment gets bound to an entry. According to my
232 * observations, the number of fragments does not exceed 2, but just in case,
233 * a bigger, up to VIOIF_INDIRECT_MAX - 1 array is allocated. To save resources,
234 * the dma handles are allocated lazily in the tx path.
235 */
236 struct vioif_tx_buf {
237 mblk_t *tb_mp;
238
239 /* inline buffer */
240 struct vioif_buf_mapping tb_inline_mapping;
241
242 /* External buffers */
243 struct vioif_buf_mapping *tb_external_mapping;
244 unsigned int tb_external_num;
245 };
246
247 struct vioif_softc {
248 dev_info_t *sc_dev; /* mirrors virtio_softc->sc_dev */
249 struct virtio_softc sc_virtio;
250
251 mac_handle_t sc_mac_handle;
252 mac_register_t *sc_macp;
253
254 struct virtqueue *sc_rx_vq;
255 struct virtqueue *sc_tx_vq;
256 struct virtqueue *sc_ctrl_vq;
257
258 unsigned int sc_tx_stopped:1;
259
260 /* Feature bits. */
261 unsigned int sc_rx_csum:1;
262 unsigned int sc_tx_csum:1;
263 unsigned int sc_tx_tso4:1;
264
265 int sc_mtu;
266 uint8_t sc_mac[ETHERADDRL];
267 /*
268 * For rx buffers, we keep a pointer array, because the buffers
269 * can be loaned upstream, and we have to repopulate the array with
270 * new members.
271 */
272 struct vioif_rx_buf **sc_rxbufs;
273
274 /*
275 * For tx, we just allocate an array of buffers. The packet can
276 * either be copied into the inline buffer, or the external mapping
277 * could be used to map the packet
278 */
279 struct vioif_tx_buf *sc_txbufs;
280
281 kstat_t *sc_intrstat;
282 /*
283 * We "loan" rx buffers upstream and reuse them after they are
284 * freed. This lets us avoid allocations in the hot path.
285 */
286 kmem_cache_t *sc_rxbuf_cache;
287 ulong_t sc_rxloan;
288
289 /* Copying small packets turns out to be faster then mapping them. */
290 unsigned long sc_rxcopy_thresh;
291 unsigned long sc_txcopy_thresh;
292 /* Some statistic coming here */
293 uint64_t sc_ipackets;
294 uint64_t sc_opackets;
295 uint64_t sc_rbytes;
296 uint64_t sc_obytes;
297 uint64_t sc_brdcstxmt;
298 uint64_t sc_brdcstrcv;
299 uint64_t sc_multixmt;
300 uint64_t sc_multircv;
301 uint64_t sc_norecvbuf;
302 uint64_t sc_notxbuf;
303 uint64_t sc_toolongerr;
304 uint64_t sc_ierrors;
305 uint64_t sc_oerrors;
306 };
307
308 #define ETHER_HEADER_LEN sizeof (struct ether_header)
309
310 /* MTU + the ethernet header. */
311 #define MAX_PAYLOAD 65535
312 #define MAX_MTU (MAX_PAYLOAD - ETHER_HEADER_LEN)
313 #define DEFAULT_MTU ETHERMTU
314
315 /*
316 * Yeah, we spend 8M per device. Turns out, there is no point
317 * being smart and using merged rx buffers (VIRTIO_NET_F_MRG_RXBUF),
318 * because vhost does not support them, and we expect to be used with
319 * vhost in production environment.
320 */
321 /* The buffer keeps both the packet data and the virtio_net_header. */
322 #define VIOIF_RX_SIZE (MAX_PAYLOAD + sizeof (struct virtio_net_hdr))
323
324 /*
325 * We win a bit on header alignment, but the host wins a lot
326 * more on moving aligned buffers. Might need more thought.
327 */
328 #define VIOIF_IP_ALIGN 0
329
330 /* Maximum number of indirect descriptors, somewhat arbitrary. */
331 #define VIOIF_INDIRECT_MAX 128
332
333 /*
334 * We pre-allocate a reasonably large buffer to copy small packets
335 * there. Bigger packets are mapped, packets with multiple
336 * cookies are mapped as indirect buffers.
337 */
338 #define VIOIF_TX_INLINE_SIZE 2048
339
340 /* Native queue size for all queues */
341 #define VIOIF_RX_QLEN 0
342 #define VIOIF_TX_QLEN 0
343 #define VIOIF_CTRL_QLEN 0
344
345 static uchar_t vioif_broadcast[ETHERADDRL] = {
346 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
347 };
348
349 #define VIOIF_TX_THRESH_MAX 640
350 #define VIOIF_RX_THRESH_MAX 640
351
352 static char vioif_txcopy_thresh[] =
353 "vioif_txcopy_thresh";
354 static char vioif_rxcopy_thresh[] =
355 "vioif_rxcopy_thresh";
356
357 static char *vioif_priv_props[] = {
358 vioif_txcopy_thresh,
359 vioif_rxcopy_thresh,
360 NULL
361 };
362
363 /* Add up to ddi? */
364 static ddi_dma_cookie_t *
365 vioif_dma_curr_cookie(ddi_dma_handle_t dmah)
366 {
367 ddi_dma_impl_t *dmah_impl = (void *) dmah;
368 ASSERT(dmah_impl->dmai_cookie);
369 return (dmah_impl->dmai_cookie);
370 }
371
372 static void
373 vioif_dma_reset_cookie(ddi_dma_handle_t dmah, ddi_dma_cookie_t *dmac)
374 {
375 ddi_dma_impl_t *dmah_impl = (void *) dmah;
376 dmah_impl->dmai_cookie = dmac;
377 }
378
379 static link_state_t
380 vioif_link_state(struct vioif_softc *sc)
381 {
382 if (sc->sc_virtio.sc_features & VIRTIO_NET_F_STATUS) {
383 if (virtio_read_device_config_2(&sc->sc_virtio,
384 VIRTIO_NET_CONFIG_STATUS) & VIRTIO_NET_S_LINK_UP) {
385
386 dev_err(sc->sc_dev, CE_NOTE, "Link up\n");
387 return (LINK_STATE_UP);
388 } else {
389 dev_err(sc->sc_dev, CE_NOTE, "Link down\n");
390 return (LINK_STATE_DOWN);
391 }
392 }
393
394 dev_err(sc->sc_dev, CE_NOTE, "Link assumed up\n");
395
396 return (LINK_STATE_UP);
397 }
398
399 static ddi_dma_attr_t vioif_inline_buf_dma_attr = {
400 DMA_ATTR_V0, /* Version number */
401 0, /* low address */
402 0xFFFFFFFFFFFFFFFF, /* high address */
403 0xFFFFFFFF, /* counter register max */
404 1, /* page alignment */
405 1, /* burst sizes: 1 - 32 */
406 1, /* minimum transfer size */
407 0xFFFFFFFF, /* max transfer size */
408 0xFFFFFFFFFFFFFFF, /* address register max */
409 1, /* scatter-gather capacity */
410 1, /* device operates on bytes */
411 0, /* attr flag: set to 0 */
412 };
413
414 static ddi_dma_attr_t vioif_mapped_buf_dma_attr = {
415 DMA_ATTR_V0, /* Version number */
416 0, /* low address */
417 0xFFFFFFFFFFFFFFFF, /* high address */
418 0xFFFFFFFF, /* counter register max */
419 1, /* page alignment */
420 1, /* burst sizes: 1 - 32 */
421 1, /* minimum transfer size */
422 0xFFFFFFFF, /* max transfer size */
423 0xFFFFFFFFFFFFFFF, /* address register max */
424
425 /* One entry is used for the virtio_net_hdr on the tx path */
426 VIOIF_INDIRECT_MAX - 1, /* scatter-gather capacity */
427 1, /* device operates on bytes */
428 0, /* attr flag: set to 0 */
429 };
430
431 static ddi_device_acc_attr_t vioif_bufattr = {
432 DDI_DEVICE_ATTR_V0,
433 DDI_NEVERSWAP_ACC,
434 DDI_STORECACHING_OK_ACC,
435 DDI_DEFAULT_ACC
436 };
437
438 static void
439 vioif_rx_free(caddr_t free_arg)
440 {
441 struct vioif_rx_buf *buf = (void *) free_arg;
442 struct vioif_softc *sc = buf->rb_sc;
443
444 kmem_cache_free(sc->sc_rxbuf_cache, buf);
445 atomic_dec_ulong(&sc->sc_rxloan);
446 }
447
448 static int
449 vioif_rx_construct(void *buffer, void *user_arg, int kmflags)
450 {
451 _NOTE(ARGUNUSED(kmflags));
452 struct vioif_softc *sc = user_arg;
453 struct vioif_rx_buf *buf = buffer;
454 size_t len;
455
456 if (ddi_dma_alloc_handle(sc->sc_dev, &vioif_mapped_buf_dma_attr,
457 DDI_DMA_SLEEP, NULL, &buf->rb_mapping.vbm_dmah)) {
458 dev_err(sc->sc_dev, CE_WARN,
459 "Can't allocate dma handle for rx buffer");
460 goto exit_handle;
461 }
462
463 if (ddi_dma_mem_alloc(buf->rb_mapping.vbm_dmah,
464 VIOIF_RX_SIZE + sizeof (struct virtio_net_hdr),
465 &vioif_bufattr, DDI_DMA_STREAMING, DDI_DMA_SLEEP,
466 NULL, &buf->rb_mapping.vbm_buf, &len, &buf->rb_mapping.vbm_acch)) {
467 dev_err(sc->sc_dev, CE_WARN,
468 "Can't allocate rx buffer");
469 goto exit_alloc;
470 }
471 ASSERT(len >= VIOIF_RX_SIZE);
472
473 if (ddi_dma_addr_bind_handle(buf->rb_mapping.vbm_dmah, NULL,
474 buf->rb_mapping.vbm_buf, len, DDI_DMA_READ | DDI_DMA_STREAMING,
475 DDI_DMA_SLEEP, NULL, &buf->rb_mapping.vbm_dmac,
476 &buf->rb_mapping.vbm_ncookies)) {
477 dev_err(sc->sc_dev, CE_WARN, "Can't bind tx buffer");
478
479 goto exit_bind;
480 }
481
482 ASSERT(buf->rb_mapping.vbm_ncookies <= VIOIF_INDIRECT_MAX);
483
484 buf->rb_sc = sc;
485 buf->rb_frtn.free_arg = (void *) buf;
486 buf->rb_frtn.free_func = vioif_rx_free;
487
488 return (0);
489 exit_bind:
490 ddi_dma_mem_free(&buf->rb_mapping.vbm_acch);
491 exit_alloc:
492 ddi_dma_free_handle(&buf->rb_mapping.vbm_dmah);
493 exit_handle:
494
495 return (ENOMEM);
496 }
497
498 static void
499 vioif_rx_destruct(void *buffer, void *user_arg)
500 {
501 _NOTE(ARGUNUSED(user_arg));
502 struct vioif_rx_buf *buf = buffer;
503
504 ASSERT(buf->rb_mapping.vbm_acch);
505 ASSERT(buf->rb_mapping.vbm_acch);
506
507 (void) ddi_dma_unbind_handle(buf->rb_mapping.vbm_dmah);
508 ddi_dma_mem_free(&buf->rb_mapping.vbm_acch);
509 ddi_dma_free_handle(&buf->rb_mapping.vbm_dmah);
510 }
511
512 static void
513 vioif_free_mems(struct vioif_softc *sc)
514 {
515 int i;
516
517 for (i = 0; i < sc->sc_tx_vq->vq_num; i++) {
518 struct vioif_tx_buf *buf = &sc->sc_txbufs[i];
519 int j;
520
521 /* Tear down the internal mapping. */
522
523 ASSERT(buf->tb_inline_mapping.vbm_acch);
524 ASSERT(buf->tb_inline_mapping.vbm_dmah);
525
526 (void) ddi_dma_unbind_handle(buf->tb_inline_mapping.vbm_dmah);
527 ddi_dma_mem_free(&buf->tb_inline_mapping.vbm_acch);
528 ddi_dma_free_handle(&buf->tb_inline_mapping.vbm_dmah);
529
530 /* We should not see any in-flight buffers at this point. */
531 ASSERT(!buf->tb_mp);
532
533 /* Free all the dma hdnales we allocated lazily. */
534 for (j = 0; buf->tb_external_mapping[j].vbm_dmah; j++)
535 ddi_dma_free_handle(
536 &buf->tb_external_mapping[j].vbm_dmah);
537 /* Free the external mapping array. */
538 kmem_free(buf->tb_external_mapping,
539 sizeof (struct vioif_tx_buf) * VIOIF_INDIRECT_MAX - 1);
540 }
541
542 kmem_free(sc->sc_txbufs, sizeof (struct vioif_tx_buf) *
543 sc->sc_tx_vq->vq_num);
544
545 for (i = 0; i < sc->sc_rx_vq->vq_num; i++) {
546 struct vioif_rx_buf *buf = sc->sc_rxbufs[i];
547
548 if (buf)
549 kmem_cache_free(sc->sc_rxbuf_cache, buf);
550 }
551 kmem_free(sc->sc_rxbufs, sizeof (struct vioif_rx_buf *) *
552 sc->sc_rx_vq->vq_num);
553 }
554
555 static int
556 vioif_alloc_mems(struct vioif_softc *sc)
557 {
558 int i, txqsize, rxqsize;
559 size_t len;
560 unsigned int nsegments;
561
562 txqsize = sc->sc_tx_vq->vq_num;
563 rxqsize = sc->sc_rx_vq->vq_num;
564
565 sc->sc_txbufs = kmem_zalloc(sizeof (struct vioif_tx_buf) * txqsize,
566 KM_SLEEP);
567 if (sc->sc_txbufs == NULL) {
568 dev_err(sc->sc_dev, CE_WARN,
569 "Failed to allocate the tx buffers array");
570 goto exit_txalloc;
571 }
572
573 /*
574 * We don't allocate the rx vioif_buffs, just the pointers, as
575 * rx vioif_bufs can be loaned upstream, and we don't know the
576 * total number we need.
577 */
578 sc->sc_rxbufs = kmem_zalloc(sizeof (struct vioif_rx_buf *) * rxqsize,
579 KM_SLEEP);
580 if (sc->sc_rxbufs == NULL) {
581 dev_err(sc->sc_dev, CE_WARN,
582 "Failed to allocate the rx buffers pointer array");
583 goto exit_rxalloc;
584 }
585
586 for (i = 0; i < txqsize; i++) {
587 struct vioif_tx_buf *buf = &sc->sc_txbufs[i];
588
589 /* Allocate and bind an inline mapping. */
590
591 if (ddi_dma_alloc_handle(sc->sc_dev,
592 &vioif_inline_buf_dma_attr,
593 DDI_DMA_SLEEP, NULL, &buf->tb_inline_mapping.vbm_dmah)) {
594
595 dev_err(sc->sc_dev, CE_WARN,
596 "Can't allocate dma handle for tx buffer %d", i);
597 goto exit_tx;
598 }
599
600 if (ddi_dma_mem_alloc(buf->tb_inline_mapping.vbm_dmah,
601 VIOIF_TX_INLINE_SIZE, &vioif_bufattr, DDI_DMA_STREAMING,
602 DDI_DMA_SLEEP, NULL, &buf->tb_inline_mapping.vbm_buf,
603 &len, &buf->tb_inline_mapping.vbm_acch)) {
604
605 dev_err(sc->sc_dev, CE_WARN,
606 "Can't allocate tx buffer %d", i);
607 goto exit_tx;
608 }
609 ASSERT(len >= VIOIF_TX_INLINE_SIZE);
610
611 if (ddi_dma_addr_bind_handle(buf->tb_inline_mapping.vbm_dmah,
612 NULL, buf->tb_inline_mapping.vbm_buf, len,
613 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL,
614 &buf->tb_inline_mapping.vbm_dmac, &nsegments)) {
615
616 dev_err(sc->sc_dev, CE_WARN,
617 "Can't bind tx buffer %d", i);
618 goto exit_tx;
619 }
620
621 /* We asked for a single segment */
622 ASSERT(nsegments == 1);
623
624 /*
625 * We allow up to VIOIF_INDIRECT_MAX - 1 external mappings.
626 * In reality, I don't expect more then 2-3 used, but who
627 * knows.
628 */
629 buf->tb_external_mapping = kmem_zalloc(
630 sizeof (struct vioif_tx_buf) * VIOIF_INDIRECT_MAX - 1,
631 KM_SLEEP);
632
633 /*
634 * The external mapping's dma handles are allocate lazily,
635 * as we don't expect most of them to be used..
636 */
637 }
638
639 return (0);
640
641 exit_tx:
642 for (i = 0; i < txqsize; i++) {
643 struct vioif_tx_buf *buf = &sc->sc_txbufs[i];
644
645 if (buf->tb_inline_mapping.vbm_dmah)
646 (void) ddi_dma_unbind_handle(
647 buf->tb_inline_mapping.vbm_dmah);
648
649 if (buf->tb_inline_mapping.vbm_acch)
650 ddi_dma_mem_free(
651 &buf->tb_inline_mapping.vbm_acch);
652
653 if (buf->tb_inline_mapping.vbm_dmah)
654 ddi_dma_free_handle(
655 &buf->tb_inline_mapping.vbm_dmah);
656
657 if (buf->tb_external_mapping)
658 kmem_free(buf->tb_external_mapping,
659 sizeof (struct vioif_tx_buf) *
660 VIOIF_INDIRECT_MAX - 1);
661 }
662
663 kmem_free(sc->sc_rxbufs, sizeof (struct vioif_rx_buf) * rxqsize);
664
665 exit_rxalloc:
666 kmem_free(sc->sc_txbufs, sizeof (struct vioif_tx_buf) * txqsize);
667 exit_txalloc:
668 return (ENOMEM);
669 }
670
671 /* ARGSUSED */
672 int
673 vioif_multicst(void *arg, boolean_t add, const uint8_t *macaddr)
674 {
675 return (DDI_SUCCESS);
676 }
677
678 /* ARGSUSED */
679 int
680 vioif_promisc(void *arg, boolean_t on)
681 {
682 return (DDI_SUCCESS);
683 }
684
685 /* ARGSUSED */
686 int
687 vioif_unicst(void *arg, const uint8_t *macaddr)
688 {
689 return (DDI_FAILURE);
690 }
691
692
693 static int
694 vioif_add_rx(struct vioif_softc *sc, int kmflag)
695 {
696 struct vq_entry *ve;
697 struct vioif_rx_buf *buf;
698
699 ve = vq_alloc_entry(sc->sc_rx_vq);
700 if (!ve) {
701 /* Out of free descriptors - ring already full. */
702 /* would be better to update sc_norxdescavail */
703 /* but MAC does not ask for this info */
704 /* hence update sc_norecvbuf */
705 sc->sc_norecvbuf ++;
706 goto exit_vq;
707 }
708 buf = sc->sc_rxbufs[ve->qe_index];
709
710 if (!buf) {
711 /* First run, allocate the buffer. */
712 buf = kmem_cache_alloc(sc->sc_rxbuf_cache, kmflag);
713 sc->sc_rxbufs[ve->qe_index] = buf;
714 }
715
716 /* Still nothing? Bye. */
717 if (!buf) {
718 dev_err(sc->sc_dev, CE_WARN, "Can't allocate rx buffer");
719 sc->sc_norecvbuf ++;
720 goto exit_buf;
721 }
722
723 ASSERT(buf->rb_mapping.vbm_ncookies >= 1);
724
725 /*
726 * For an unknown reason, the virtio_net_hdr must be placed
727 * as a separate virtio queue entry.
728 */
729 virtio_ve_add_indirect_buf(ve, buf->rb_mapping.vbm_dmac.dmac_laddress,
730 sizeof (struct virtio_net_hdr), B_FALSE);
731
732 /* Add the rest of the first cookie. */
733 virtio_ve_add_indirect_buf(ve,
734 buf->rb_mapping.vbm_dmac.dmac_laddress +
735 sizeof (struct virtio_net_hdr),
736 buf->rb_mapping.vbm_dmac.dmac_size -
737 sizeof (struct virtio_net_hdr), B_FALSE);
738
739 /*
740 * If the buffer consists of a single cookie (unlikely for a
741 * 64-k buffer), we are done. Otherwise, add the rest of the cookies
742 * using indirect entries.
743 */
744 if (buf->rb_mapping.vbm_ncookies > 1) {
745 ddi_dma_cookie_t *first_extra_dmac;
746 ddi_dma_cookie_t dmac;
747 first_extra_dmac =
748 vioif_dma_curr_cookie(buf->rb_mapping.vbm_dmah);
749
750 ddi_dma_nextcookie(buf->rb_mapping.vbm_dmah, &dmac);
751 virtio_ve_add_cookie(ve, buf->rb_mapping.vbm_dmah,
752 dmac, buf->rb_mapping.vbm_ncookies - 1, B_FALSE);
753 vioif_dma_reset_cookie(buf->rb_mapping.vbm_dmah,
754 first_extra_dmac);
755 }
756
757 virtio_push_chain(ve, B_FALSE);
758
759 return (DDI_SUCCESS);
760
761 exit_buf:
762 vq_free_entry(sc->sc_rx_vq, ve);
763 exit_vq:
764 return (DDI_FAILURE);
765 }
766
767 static int
768 vioif_populate_rx(struct vioif_softc *sc, int kmflag)
769 {
770 int i = 0;
771 int ret;
772
773 for (;;) {
774 ret = vioif_add_rx(sc, kmflag);
775 if (ret)
776 /*
777 * We could not allocate some memory. Try to work with
778 * what we've got.
779 */
780 break;
781 i++;
782 }
783
784 if (i)
785 virtio_sync_vq(sc->sc_rx_vq);
786
787 return (i);
788 }
789
790 static int
791 vioif_process_rx(struct vioif_softc *sc)
792 {
793 struct vq_entry *ve;
794 struct vioif_rx_buf *buf;
795 mblk_t *mp;
796 uint32_t len;
797 int i = 0;
798
799 while ((ve = virtio_pull_chain(sc->sc_rx_vq, &len))) {
800
801 buf = sc->sc_rxbufs[ve->qe_index];
802 ASSERT(buf);
803
804 if (len < sizeof (struct virtio_net_hdr)) {
805 dev_err(sc->sc_dev, CE_WARN, "RX: Cnain too small: %u",
806 len - (uint32_t)sizeof (struct virtio_net_hdr));
807 sc->sc_ierrors ++;
808 virtio_free_chain(ve);
809 continue;
810 }
811
812 len -= sizeof (struct virtio_net_hdr);
813 /*
814 * We copy small packets that happenned to fit into a single
815 * cookie and reuse the buffers. For bigger ones, we loan
816 * the buffers upstream.
817 */
818 if (len < sc->sc_rxcopy_thresh) {
819 mp = allocb(len, 0);
820 if (!mp) {
821 cmn_err(CE_WARN, "Failed to allocale mblock!");
822 sc->sc_norecvbuf ++;
823 sc->sc_ierrors ++;
824
825 virtio_free_chain(ve);
826 break;
827 }
828
829 bcopy((char *)buf->rb_mapping.vbm_buf +
830 sizeof (struct virtio_net_hdr), mp->b_rptr, len);
831 mp->b_wptr = mp->b_rptr + len;
832
833 } else {
834 mp = desballoc((unsigned char *)
835 buf->rb_mapping.vbm_buf +
836 sizeof (struct virtio_net_hdr) +
837 VIOIF_IP_ALIGN, len, 0, &buf->rb_frtn);
838 if (!mp) {
839 cmn_err(CE_WARN, "Failed to allocale mblock!");
840 sc->sc_norecvbuf ++;
841 sc->sc_ierrors ++;
842
843 virtio_free_chain(ve);
844 break;
845 }
846 mp->b_wptr = mp->b_rptr + len;
847
848 atomic_inc_ulong(&sc->sc_rxloan);
849 /*
850 * Buffer loanded, we will have to allocte a new one
851 * for this slot.
852 */
853 sc->sc_rxbufs[ve->qe_index] = NULL;
854 }
855 /* virtio-net does not provide the info if this packet */
856 /* is multicast or broadcast. So we have to check it */
857 if (mp->b_rptr[0] & 0x1) {
858 if (bcmp(mp->b_rptr, vioif_broadcast, ETHERADDRL) != 0)
859 sc->sc_multircv ++;
860 else
861 sc->sc_brdcstrcv ++;
862 }
863
864 sc->sc_rbytes += len;
865 sc->sc_ipackets ++;
866
867 virtio_free_chain(ve);
868 mac_rx(sc->sc_mac_handle, NULL, mp);
869 i++;
870 }
871
872 return (i);
873 }
874
875 static void
876 vioif_reclaim_used_tx(struct vioif_softc *sc)
877 {
878 struct vq_entry *ve;
879 struct vioif_tx_buf *buf;
880 uint32_t len;
881 mblk_t *mp;
882 int i = 0;
883
884 while ((ve = virtio_pull_chain(sc->sc_tx_vq, &len))) {
885 /* We don't chain descriptors for tx, so don't expect any. */
886 ASSERT(!ve->qe_next);
887
888 buf = &sc->sc_txbufs[ve->qe_index];
889 mp = buf->tb_mp;
890 buf->tb_mp = NULL;
891
892 if (mp) {
893 for (i = 0; i < buf->tb_external_num; i++)
894 (void) ddi_dma_unbind_handle(
895 buf->tb_external_mapping[i].vbm_dmah);
896 }
897
898 virtio_free_chain(ve);
899
900 /* External mapping used, mp was not freed in vioif_send() */
901 if (mp)
902 freemsg(mp);
903 i++;
904 }
905
906 if (sc->sc_tx_stopped && i) {
907 sc->sc_tx_stopped = 0;
908 mac_tx_update(sc->sc_mac_handle);
909 }
910 }
911
912 /* sc will be used to update stat counters. */
913 /* ARGSUSED */
914 static inline void
915 vioif_tx_inline(struct vioif_softc *sc, struct vq_entry *ve, mblk_t *mp,
916 size_t msg_size)
917 {
918 struct vioif_tx_buf *buf;
919 buf = &sc->sc_txbufs[ve->qe_index];
920
921 ASSERT(buf);
922
923 /* Frees mp */
924 mcopymsg(mp, buf->tb_inline_mapping.vbm_buf +
925 sizeof (struct virtio_net_hdr));
926
927 virtio_ve_add_indirect_buf(ve,
928 buf->tb_inline_mapping.vbm_dmac.dmac_laddress +
929 sizeof (struct virtio_net_hdr), msg_size, B_TRUE);
930 }
931
932 static inline int
933 vioif_tx_lazy_handle_alloc(struct vioif_softc *sc, struct vioif_tx_buf *buf,
934 int i)
935 {
936 int ret = DDI_SUCCESS;
937
938 if (!buf->tb_external_mapping[i].vbm_dmah) {
939 ret = ddi_dma_alloc_handle(sc->sc_dev,
940 &vioif_mapped_buf_dma_attr, DDI_DMA_SLEEP, NULL,
941 &buf->tb_external_mapping[i].vbm_dmah);
942 if (ret != DDI_SUCCESS) {
943 dev_err(sc->sc_dev, CE_WARN,
944 "Can't allocate dma handle for external tx buffer");
945 }
946 }
947
948 return (ret);
949 }
950
951 static inline int
952 vioif_tx_external(struct vioif_softc *sc, struct vq_entry *ve, mblk_t *mp,
953 size_t msg_size)
954 {
955 _NOTE(ARGUNUSED(msg_size));
956
957 struct vioif_tx_buf *buf;
958 mblk_t *nmp;
959 int i, j;
960 int ret = DDI_SUCCESS;
961
962 buf = &sc->sc_txbufs[ve->qe_index];
963
964 ASSERT(buf);
965
966 buf->tb_external_num = 0;
967 i = 0;
968 nmp = mp;
969
970 while (nmp) {
971 size_t len;
972 ddi_dma_cookie_t dmac;
973 unsigned int ncookies;
974
975 len = MBLKL(nmp);
976 /*
977 * For some reason, the network stack can
978 * actually send us zero-length fragments.
979 */
980 if (len == 0) {
981 nmp = nmp->b_cont;
982 continue;
983 }
984
985 ret = vioif_tx_lazy_handle_alloc(sc, buf, i);
986 if (ret != DDI_SUCCESS) {
987 sc->sc_notxbuf ++;
988 sc->sc_oerrors ++;
989 goto exit_lazy_alloc;
990 }
991 ret = ddi_dma_addr_bind_handle(
992 buf->tb_external_mapping[i].vbm_dmah, NULL,
993 (caddr_t)nmp->b_rptr, len,
994 DDI_DMA_WRITE | DDI_DMA_STREAMING,
995 DDI_DMA_SLEEP, NULL, &dmac, &ncookies);
996
997 if (ret != DDI_SUCCESS) {
998 sc->sc_oerrors ++;
999 dev_err(sc->sc_dev, CE_NOTE,
1000 "TX: Failed to bind external handle");
1001 goto exit_bind;
1002 }
1003
1004 /* Check if we still fit into the indirect table. */
1005 if (virtio_ve_indirect_available(ve) < ncookies) {
1006 dev_err(sc->sc_dev, CE_NOTE,
1007 "TX: Indirect descriptor table limit reached."
1008 " It took %d fragments.", i);
1009 sc->sc_notxbuf ++;
1010 sc->sc_oerrors ++;
1011
1012 ret = DDI_FAILURE;
1013 goto exit_limit;
1014 }
1015
1016 virtio_ve_add_cookie(ve, buf->tb_external_mapping[i].vbm_dmah,
1017 dmac, ncookies, B_TRUE);
1018
1019 nmp = nmp->b_cont;
1020 i++;
1021 }
1022
1023 buf->tb_external_num = i;
1024 /* Save the mp to free it when the packet is sent. */
1025 buf->tb_mp = mp;
1026
1027 return (DDI_SUCCESS);
1028
1029 exit_limit:
1030 exit_bind:
1031 exit_lazy_alloc:
1032
1033 for (j = 0; j < i; j++) {
1034 (void) ddi_dma_unbind_handle(
1035 buf->tb_external_mapping[j].vbm_dmah);
1036 }
1037
1038 return (ret);
1039 }
1040
1041 static boolean_t
1042 vioif_send(struct vioif_softc *sc, mblk_t *mp)
1043 {
1044 struct vq_entry *ve;
1045 struct vioif_tx_buf *buf;
1046 struct virtio_net_hdr *net_header = NULL;
1047 size_t msg_size = 0;
1048 uint32_t csum_start;
1049 uint32_t csum_stuff;
1050 uint32_t csum_flags;
1051 uint32_t lso_flags;
1052 uint32_t lso_mss;
1053 mblk_t *nmp;
1054 int ret;
1055 boolean_t lso_required = B_FALSE;
1056
1057 for (nmp = mp; nmp; nmp = nmp->b_cont)
1058 msg_size += MBLKL(nmp);
1059
1060 if (sc->sc_tx_tso4) {
1061 mac_lso_get(mp, &lso_mss, &lso_flags);
1062 lso_required = (lso_flags & HW_LSO);
1063 }
1064 if (!lso_required && msg_size > sc->sc_mtu) {
1065 dev_err(sc->sc_dev, CE_WARN, "Message too big");
1066 freemsg(mp);
1067
1068 sc->sc_toolongerr ++;
1069 sc->sc_oerrors ++;
1070 return (B_FALSE);
1071 }
1072 ve = vq_alloc_entry(sc->sc_tx_vq);
1073
1074 if (!ve) {
1075 sc->sc_notxbuf ++;
1076 /* Out of free descriptors - try later. */
1077 return (B_FALSE);
1078 }
1079 buf = &sc->sc_txbufs[ve->qe_index];
1080
1081 /* Use the inline buffer of the first entry for the virtio_net_hdr. */
1082 (void) memset(buf->tb_inline_mapping.vbm_buf, 0,
1083 sizeof (struct virtio_net_hdr));
1084
1085 /* LINTED E_BAD_PTR_CAST_ALIGN */
1086 net_header = (struct virtio_net_hdr *)
1087 buf->tb_inline_mapping.vbm_buf;
1088
1089 mac_hcksum_get(mp, &csum_start, &csum_stuff, NULL,
1090 NULL, &csum_flags);
1091
1092 /* They want us to do the TCP/UDP csum calculation. */
1093 if (csum_flags & HCK_PARTIALCKSUM) {
1094 struct ether_header *eth_header;
1095 int eth_hsize;
1096
1097 /* Did we ask for it? */
1098 ASSERT(sc->sc_tx_csum);
1099
1100 /* We only asked for partial csum packets. */
1101 ASSERT(!(csum_flags & HCK_IPV4_HDRCKSUM));
1102 ASSERT(!(csum_flags & HCK_FULLCKSUM));
1103
1104 eth_header = (void *) mp->b_rptr;
1105 if (eth_header->ether_type == htons(ETHERTYPE_VLAN)) {
1106 eth_hsize = sizeof (struct ether_vlan_header);
1107 } else {
1108 eth_hsize = sizeof (struct ether_header);
1109 }
1110 net_header->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
1111 net_header->csum_start = eth_hsize + csum_start;
1112 net_header->csum_offset = csum_stuff - csum_start;
1113 }
1114
1115 /* setup LSO fields if required */
1116 if (lso_required) {
1117 net_header->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1118 net_header->gso_size = (uint16_t)lso_mss;
1119 }
1120
1121 virtio_ve_add_indirect_buf(ve,
1122 buf->tb_inline_mapping.vbm_dmac.dmac_laddress,
1123 sizeof (struct virtio_net_hdr), B_TRUE);
1124
1125 /*
1126 * We copy small packets into the inline buffer. The bigger ones
1127 * get mapped using the mapped buffer.
1128 */
1129 if (msg_size < sc->sc_txcopy_thresh) {
1130 vioif_tx_inline(sc, ve, mp, msg_size);
1131 } else {
1132 /* statistic gets updated by vioif_tx_external when fail */
1133 ret = vioif_tx_external(sc, ve, mp, msg_size);
1134 if (ret != DDI_SUCCESS)
1135 goto exit_tx_external;
1136 }
1137
1138 virtio_push_chain(ve, B_TRUE);
1139 /* meanwhile update the statistic */
1140 if (mp->b_rptr[0] & 0x1) {
1141 if (bcmp(mp->b_rptr, vioif_broadcast, ETHERADDRL) != 0)
1142 sc->sc_multixmt ++;
1143 else
1144 sc->sc_brdcstxmt ++;
1145 }
1146 sc->sc_opackets ++;
1147 sc->sc_obytes += msg_size;
1148
1149 return (B_TRUE);
1150
1151 exit_tx_external:
1152
1153 vq_free_entry(sc->sc_tx_vq, ve);
1154 /*
1155 * vioif_tx_external can fail when the buffer does not fit into the
1156 * indirect descriptor table. Free the mp. I don't expect this ever
1157 * to happen.
1158 */
1159 freemsg(mp);
1160
1161 return (B_TRUE);
1162 }
1163
1164 mblk_t *
1165 vioif_tx(void *arg, mblk_t *mp)
1166 {
1167 struct vioif_softc *sc = arg;
1168 mblk_t *nmp;
1169
1170 while (mp != NULL) {
1171 nmp = mp->b_next;
1172 mp->b_next = NULL;
1173
1174 if (!vioif_send(sc, mp)) {
1175 sc->sc_tx_stopped = 1;
1176 mp->b_next = nmp;
1177 break;
1178 }
1179 mp = nmp;
1180 }
1181
1182 return (mp);
1183 }
1184
1185 int
1186 vioif_start(void *arg)
1187 {
1188 struct vioif_softc *sc = arg;
1189
1190 mac_link_update(sc->sc_mac_handle,
1191 vioif_link_state(sc));
1192
1193 virtio_start_vq_intr(sc->sc_rx_vq);
1194
1195 return (DDI_SUCCESS);
1196 }
1197
1198 void
1199 vioif_stop(void *arg)
1200 {
1201 struct vioif_softc *sc = arg;
1202
1203 virtio_stop_vq_intr(sc->sc_rx_vq);
1204 }
1205
1206 /* ARGSUSED */
1207 static int
1208 vioif_stat(void *arg, uint_t stat, uint64_t *val)
1209 {
1210 struct vioif_softc *sc = arg;
1211
1212 switch (stat) {
1213 case MAC_STAT_IERRORS:
1214 *val = sc->sc_ierrors;
1215 break;
1216 case MAC_STAT_OERRORS:
1217 *val = sc->sc_oerrors;
1218 break;
1219 case MAC_STAT_MULTIRCV:
1220 *val = sc->sc_multircv;
1221 break;
1222 case MAC_STAT_BRDCSTRCV:
1223 *val = sc->sc_brdcstrcv;
1224 break;
1225 case MAC_STAT_MULTIXMT:
1226 *val = sc->sc_multixmt;
1227 break;
1228 case MAC_STAT_BRDCSTXMT:
1229 *val = sc->sc_brdcstxmt;
1230 break;
1231 case MAC_STAT_IPACKETS:
1232 *val = sc->sc_ipackets;
1233 break;
1234 case MAC_STAT_RBYTES:
1235 *val = sc->sc_rbytes;
1236 break;
1237 case MAC_STAT_OPACKETS:
1238 *val = sc->sc_opackets;
1239 break;
1240 case MAC_STAT_OBYTES:
1241 *val = sc->sc_obytes;
1242 break;
1243 case MAC_STAT_NORCVBUF:
1244 *val = sc->sc_norecvbuf;
1245 break;
1246 case MAC_STAT_NOXMTBUF:
1247 *val = sc->sc_notxbuf;
1248 break;
1249 case MAC_STAT_IFSPEED:
1250 /* always 1 Gbit */
1251 *val = 1000000000ULL;
1252 break;
1253 case ETHER_STAT_LINK_DUPLEX:
1254 /* virtual device, always full-duplex */
1255 *val = LINK_DUPLEX_FULL;
1256 break;
1257
1258 default:
1259 return (ENOTSUP);
1260 }
1261
1262 return (DDI_SUCCESS);
1263 }
1264
1265 static int
1266 vioif_set_prop_private(struct vioif_softc *sc, const char *pr_name,
1267 uint_t pr_valsize, const void *pr_val)
1268 {
1269 _NOTE(ARGUNUSED(pr_valsize));
1270
1271 long result;
1272
1273 if (strcmp(pr_name, vioif_txcopy_thresh) == 0) {
1274
1275 if (pr_val == NULL)
1276 return (EINVAL);
1277
1278 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
1279
1280 if (result < 0 || result > VIOIF_TX_THRESH_MAX)
1281 return (EINVAL);
1282 sc->sc_txcopy_thresh = result;
1283 }
1284 if (strcmp(pr_name, vioif_rxcopy_thresh) == 0) {
1285
1286 if (pr_val == NULL)
1287 return (EINVAL);
1288
1289 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
1290
1291 if (result < 0 || result > VIOIF_RX_THRESH_MAX)
1292 return (EINVAL);
1293 sc->sc_rxcopy_thresh = result;
1294 }
1295 return (0);
1296 }
1297
1298 static int
1299 vioif_setprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
1300 uint_t pr_valsize, const void *pr_val)
1301 {
1302 struct vioif_softc *sc = arg;
1303 const uint32_t *new_mtu;
1304 int err;
1305
1306 switch (pr_num) {
1307 case MAC_PROP_MTU:
1308 new_mtu = pr_val;
1309
1310 if (*new_mtu > MAX_MTU) {
1311 dev_err(sc->sc_dev, CE_WARN,
1312 "Requested mtu (%d) out of range",
1313 *new_mtu);
1314 return (EINVAL);
1315 }
1316
1317 err = mac_maxsdu_update(sc->sc_mac_handle, *new_mtu);
1318 if (err) {
1319 dev_err(sc->sc_dev, CE_WARN,
1320 "Failed to set the requested mtu (%d)",
1321 *new_mtu);
1322 return (err);
1323 }
1324 break;
1325 case MAC_PROP_PRIVATE:
1326 err = vioif_set_prop_private(sc, pr_name,
1327 pr_valsize, pr_val);
1328 if (err)
1329 return (err);
1330 default:
1331 return (ENOTSUP);
1332 }
1333
1334 return (0);
1335 }
1336
1337 static int
1338 vioif_get_prop_private(struct vioif_softc *sc, const char *pr_name,
1339 uint_t pr_valsize, void *pr_val)
1340 {
1341 int err = ENOTSUP;
1342 int value;
1343
1344 if (strcmp(pr_name, vioif_txcopy_thresh) == 0) {
1345
1346 value = sc->sc_txcopy_thresh;
1347 err = 0;
1348 goto done;
1349 }
1350 if (strcmp(pr_name, vioif_rxcopy_thresh) == 0) {
1351
1352 value = sc->sc_rxcopy_thresh;
1353 err = 0;
1354 goto done;
1355 }
1356 done:
1357 if (err == 0) {
1358 (void) snprintf(pr_val, pr_valsize, "%d", value);
1359 }
1360 return (err);
1361 }
1362
1363 static int
1364 vioif_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
1365 uint_t pr_valsize, void *pr_val)
1366 {
1367 struct vioif_softc *sc = arg;
1368 int err = ENOTSUP;
1369
1370 switch (pr_num) {
1371 case MAC_PROP_PRIVATE:
1372 err = vioif_get_prop_private(sc, pr_name,
1373 pr_valsize, pr_val);
1374 break;
1375 default:
1376 break;
1377 }
1378 return (err);
1379 }
1380
1381 static void
1382 vioif_propinfo(void *arg, const char *pr_name, mac_prop_id_t pr_num,
1383 mac_prop_info_handle_t prh)
1384 {
1385 struct vioif_softc *sc = arg;
1386
1387 switch (pr_num) {
1388
1389 case MAC_PROP_MTU:
1390 mac_prop_info_set_range_uint32(prh, ETHERMIN, MAX_MTU);
1391 break;
1392
1393 case MAC_PROP_PRIVATE: {
1394 char valstr[64];
1395 int value;
1396
1397 bzero(valstr, sizeof (valstr));
1398 if (strcmp(pr_name, vioif_txcopy_thresh) == 0) {
1399
1400 value = sc->sc_txcopy_thresh;
1401 } else if (strcmp(pr_name,
1402 vioif_rxcopy_thresh) == 0) {
1403 value = sc->sc_rxcopy_thresh;
1404 } else {
1405 return;
1406 }
1407 (void) snprintf(valstr, sizeof (valstr), "%d", value);
1408 }
1409 default:
1410 break;
1411 }
1412 }
1413
1414 static boolean_t
1415 vioif_getcapab(void *arg, mac_capab_t cap, void *cap_data)
1416 {
1417 struct vioif_softc *sc = arg;
1418
1419 switch (cap) {
1420 case MAC_CAPAB_HCKSUM:
1421 if (sc->sc_tx_csum) {
1422 uint32_t *txflags = cap_data;
1423
1424 *txflags = HCKSUM_INET_PARTIAL;
1425 return (B_TRUE);
1426 }
1427 return (B_FALSE);
1428 case MAC_CAPAB_LSO:
1429 if (sc->sc_tx_tso4) {
1430 mac_capab_lso_t *cap_lso = cap_data;
1431
1432 cap_lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
1433 cap_lso->lso_basic_tcp_ipv4.lso_max = MAX_MTU;
1434 return (B_TRUE);
1435 }
1436 return (B_FALSE);
1437 default:
1438 break;
1439 }
1440 return (B_FALSE);
1441 }
1442
1443 static mac_callbacks_t vioif_m_callbacks = {
1444 .mc_callbacks = MC_GETCAPAB,
1445 .mc_getstat = vioif_stat,
1446 .mc_start = vioif_start,
1447 .mc_stop = vioif_stop,
1448 .mc_setpromisc = vioif_promisc,
1449 .mc_multicst = vioif_multicst,
1450 .mc_unicst = vioif_unicst,
1451 .mc_tx = vioif_tx,
1452 /* Optional callbacks */
1453 .mc_reserved = NULL, /* reserved */
1454 .mc_ioctl = NULL, /* mc_ioctl */
1455 .mc_getcapab = vioif_getcapab, /* mc_getcapab */
1456 .mc_open = NULL, /* mc_open */
1457 .mc_close = NULL, /* mc_close */
1458 .mc_setprop = vioif_setprop,
1459 .mc_getprop = vioif_getprop,
1460 .mc_propinfo = vioif_propinfo,
1461 };
1462
1463 static void
1464 vioif_show_features(struct vioif_softc *sc, const char *prefix,
1465 uint32_t features)
1466 {
1467 char buf[512];
1468 char *bufp = buf;
1469 char *bufend = buf + sizeof (buf);
1470
1471 /* LINTED E_PTRDIFF_OVERFLOW */
1472 bufp += snprintf(bufp, bufend - bufp, prefix);
1473
1474 /* LINTED E_PTRDIFF_OVERFLOW */
1475 bufp += virtio_show_features(features, bufp, bufend - bufp);
1476
1477 /* LINTED E_PTRDIFF_OVERFLOW */
1478 bufp += snprintf(bufp, bufend - bufp, "Vioif ( ");
1479
1480 if (features & VIRTIO_NET_F_CSUM)
1481 /* LINTED E_PTRDIFF_OVERFLOW */
1482 bufp += snprintf(bufp, bufend - bufp, "CSUM ");
1483 if (features & VIRTIO_NET_F_GUEST_CSUM)
1484 /* LINTED E_PTRDIFF_OVERFLOW */
1485 bufp += snprintf(bufp, bufend - bufp, "GUEST_CSUM ");
1486 if (features & VIRTIO_NET_F_MAC)
1487 /* LINTED E_PTRDIFF_OVERFLOW */
1488 bufp += snprintf(bufp, bufend - bufp, "MAC ");
1489 if (features & VIRTIO_NET_F_GSO)
1490 /* LINTED E_PTRDIFF_OVERFLOW */
1491 bufp += snprintf(bufp, bufend - bufp, "GSO ");
1492 if (features & VIRTIO_NET_F_GUEST_TSO4)
1493 /* LINTED E_PTRDIFF_OVERFLOW */
1494 bufp += snprintf(bufp, bufend - bufp, "GUEST_TSO4 ");
1495 if (features & VIRTIO_NET_F_GUEST_TSO6)
1496 /* LINTED E_PTRDIFF_OVERFLOW */
1497 bufp += snprintf(bufp, bufend - bufp, "GUEST_TSO6 ");
1498 if (features & VIRTIO_NET_F_GUEST_ECN)
1499 /* LINTED E_PTRDIFF_OVERFLOW */
1500 bufp += snprintf(bufp, bufend - bufp, "GUEST_ECN ");
1501 if (features & VIRTIO_NET_F_GUEST_UFO)
1502 /* LINTED E_PTRDIFF_OVERFLOW */
1503 bufp += snprintf(bufp, bufend - bufp, "GUEST_UFO ");
1504 if (features & VIRTIO_NET_F_HOST_TSO4)
1505 /* LINTED E_PTRDIFF_OVERFLOW */
1506 bufp += snprintf(bufp, bufend - bufp, "HOST_TSO4 ");
1507 if (features & VIRTIO_NET_F_HOST_TSO6)
1508 /* LINTED E_PTRDIFF_OVERFLOW */
1509 bufp += snprintf(bufp, bufend - bufp, "HOST_TSO6 ");
1510 if (features & VIRTIO_NET_F_HOST_ECN)
1511 /* LINTED E_PTRDIFF_OVERFLOW */
1512 bufp += snprintf(bufp, bufend - bufp, "HOST_ECN ");
1513 if (features & VIRTIO_NET_F_HOST_UFO)
1514 /* LINTED E_PTRDIFF_OVERFLOW */
1515 bufp += snprintf(bufp, bufend - bufp, "HOST_UFO ");
1516 if (features & VIRTIO_NET_F_MRG_RXBUF)
1517 /* LINTED E_PTRDIFF_OVERFLOW */
1518 bufp += snprintf(bufp, bufend - bufp, "MRG_RXBUF ");
1519 if (features & VIRTIO_NET_F_STATUS)
1520 /* LINTED E_PTRDIFF_OVERFLOW */
1521 bufp += snprintf(bufp, bufend - bufp, "STATUS ");
1522 if (features & VIRTIO_NET_F_CTRL_VQ)
1523 /* LINTED E_PTRDIFF_OVERFLOW */
1524 bufp += snprintf(bufp, bufend - bufp, "CTRL_VQ ");
1525 if (features & VIRTIO_NET_F_CTRL_RX)
1526 /* LINTED E_PTRDIFF_OVERFLOW */
1527 bufp += snprintf(bufp, bufend - bufp, "CTRL_RX ");
1528 if (features & VIRTIO_NET_F_CTRL_VLAN)
1529 /* LINTED E_PTRDIFF_OVERFLOW */
1530 bufp += snprintf(bufp, bufend - bufp, "CTRL_VLAN ");
1531 if (features & VIRTIO_NET_F_CTRL_RX_EXTRA)
1532 /* LINTED E_PTRDIFF_OVERFLOW */
1533 bufp += snprintf(bufp, bufend - bufp, "CTRL_RX_EXTRA ");
1534
1535 /* LINTED E_PTRDIFF_OVERFLOW */
1536 bufp += snprintf(bufp, bufend - bufp, ")");
1537 *bufp = '\0';
1538
1539 dev_err(sc->sc_dev, CE_NOTE, "%s", buf);
1540 }
1541
1542 /*
1543 * Find out which features are supported by the device and
1544 * choose which ones we wish to use.
1545 */
1546 static int
1547 vioif_dev_features(struct vioif_softc *sc)
1548 {
1549 uint32_t host_features;
1550
1551 host_features = virtio_negotiate_features(&sc->sc_virtio,
1552 VIRTIO_NET_F_CSUM |
1553 VIRTIO_NET_F_HOST_TSO4 |
1554 VIRTIO_NET_F_HOST_ECN |
1555 VIRTIO_NET_F_MAC |
1556 VIRTIO_NET_F_STATUS |
1557 VIRTIO_F_RING_INDIRECT_DESC |
1558 VIRTIO_F_NOTIFY_ON_EMPTY);
1559
1560 vioif_show_features(sc, "Host features: ", host_features);
1561 vioif_show_features(sc, "Negotiated features: ",
1562 sc->sc_virtio.sc_features);
1563
1564 if (!(sc->sc_virtio.sc_features & VIRTIO_F_RING_INDIRECT_DESC)) {
1565 dev_err(sc->sc_dev, CE_NOTE,
1566 "Host does not support RING_INDIRECT_DESC, bye.");
1567 return (DDI_FAILURE);
1568 }
1569
1570 return (DDI_SUCCESS);
1571 }
1572
1573 static int
1574 vioif_has_feature(struct vioif_softc *sc, uint32_t feature)
1575 {
1576 return (virtio_has_feature(&sc->sc_virtio, feature));
1577 }
1578
1579 static void
1580 vioif_set_mac(struct vioif_softc *sc)
1581 {
1582 int i;
1583
1584 for (i = 0; i < ETHERADDRL; i++) {
1585 virtio_write_device_config_1(&sc->sc_virtio,
1586 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
1587 }
1588 }
1589
1590 /* Get the mac address out of the hardware, or make up one. */
1591 static void
1592 vioif_get_mac(struct vioif_softc *sc)
1593 {
1594 int i;
1595 if (sc->sc_virtio.sc_features & VIRTIO_NET_F_MAC) {
1596 for (i = 0; i < ETHERADDRL; i++) {
1597 sc->sc_mac[i] = virtio_read_device_config_1(
1598 &sc->sc_virtio,
1599 VIRTIO_NET_CONFIG_MAC + i);
1600 }
1601 dev_err(sc->sc_dev, CE_NOTE, "Got MAC address from host: %s",
1602 ether_sprintf((struct ether_addr *)sc->sc_mac));
1603 } else {
1604 /* Get a few random bytes */
1605 (void) random_get_pseudo_bytes(sc->sc_mac, ETHERADDRL);
1606 /* Make sure it's a unicast MAC */
1607 sc->sc_mac[0] &= ~1;
1608 /* Set the "locally administered" bit */
1609 sc->sc_mac[1] |= 2;
1610
1611 vioif_set_mac(sc);
1612
1613 dev_err(sc->sc_dev, CE_NOTE,
1614 "Generated a random MAC address: %s",
1615 ether_sprintf((struct ether_addr *)sc->sc_mac));
1616 }
1617 }
1618
1619 /*
1620 * Virtqueue interrupt handlers
1621 */
1622 /* ARGSUSED */
1623 uint_t
1624 vioif_rx_handler(caddr_t arg1, caddr_t arg2)
1625 {
1626 struct virtio_softc *vsc = (void *) arg1;
1627 struct vioif_softc *sc = container_of(vsc,
1628 struct vioif_softc, sc_virtio);
1629
1630 (void) vioif_process_rx(sc);
1631
1632 (void) vioif_populate_rx(sc, KM_NOSLEEP);
1633
1634 return (DDI_INTR_CLAIMED);
1635 }
1636
1637 /* ARGSUSED */
1638 uint_t
1639 vioif_tx_handler(caddr_t arg1, caddr_t arg2)
1640 {
1641 struct virtio_softc *vsc = (void *)arg1;
1642 struct vioif_softc *sc = container_of(vsc,
1643 struct vioif_softc, sc_virtio);
1644
1645 vioif_reclaim_used_tx(sc);
1646 return (DDI_INTR_CLAIMED);
1647 }
1648
1649 static int
1650 vioif_register_ints(struct vioif_softc *sc)
1651 {
1652 int ret;
1653
1654 struct virtio_int_handler vioif_vq_h[] = {
1655 { vioif_rx_handler },
1656 { vioif_tx_handler },
1657 { NULL }
1658 };
1659
1660 ret = virtio_register_ints(&sc->sc_virtio, NULL, vioif_vq_h);
1661
1662 return (ret);
1663 }
1664
1665
1666 static void
1667 vioif_check_features(struct vioif_softc *sc)
1668 {
1669 if (vioif_has_feature(sc, VIRTIO_NET_F_CSUM)) {
1670 /* The GSO/GRO featured depend on CSUM, check them here. */
1671 sc->sc_tx_csum = 1;
1672 sc->sc_rx_csum = 1;
1673
1674 if (!vioif_has_feature(sc, VIRTIO_NET_F_GUEST_CSUM)) {
1675 sc->sc_rx_csum = 0;
1676 }
1677 cmn_err(CE_NOTE, "Csum enabled.");
1678
1679 if (vioif_has_feature(sc, VIRTIO_NET_F_HOST_TSO4)) {
1680
1681 sc->sc_tx_tso4 = 1;
1682 /*
1683 * We don't seem to have a way to ask the system
1684 * not to send us LSO packets with Explicit
1685 * Congestion Notification bit set, so we require
1686 * the device to support it in order to do
1687 * LSO.
1688 */
1689 if (!vioif_has_feature(sc, VIRTIO_NET_F_HOST_ECN)) {
1690 dev_err(sc->sc_dev, CE_NOTE,
1691 "TSO4 supported, but not ECN. "
1692 "Not using LSO.");
1693 sc->sc_tx_tso4 = 0;
1694 } else {
1695 cmn_err(CE_NOTE, "LSO enabled");
1696 }
1697 }
1698 }
1699 }
1700
1701 static int
1702 vioif_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
1703 {
1704 int ret, instance;
1705 struct vioif_softc *sc;
1706 struct virtio_softc *vsc;
1707 mac_register_t *macp;
1708
1709 instance = ddi_get_instance(devinfo);
1710
1711 switch (cmd) {
1712 case DDI_ATTACH:
1713 break;
1714
1715 case DDI_RESUME:
1716 case DDI_PM_RESUME:
1717 dev_err(devinfo, CE_WARN, "resume not supported yet");
1718 goto exit;
1719
1720 default:
1721 dev_err(devinfo, CE_WARN, "cmd 0x%x unrecognized", cmd);
1722 goto exit;
1723 }
1724
1725 sc = kmem_zalloc(sizeof (struct vioif_softc), KM_SLEEP);
1726 ddi_set_driver_private(devinfo, sc);
1727
1728 vsc = &sc->sc_virtio;
1729
1730 /* Duplicate for less typing */
1731 sc->sc_dev = devinfo;
1732 vsc->sc_dev = devinfo;
1733
1734 /*
1735 * Initialize interrupt kstat.
1736 */
1737 sc->sc_intrstat = kstat_create("vioif", instance, "intr", "controller",
1738 KSTAT_TYPE_INTR, 1, 0);
1739 if (sc->sc_intrstat == NULL) {
1740 dev_err(devinfo, CE_WARN, "kstat_create failed");
1741 goto exit_intrstat;
1742 }
1743 kstat_install(sc->sc_intrstat);
1744
1745 /* map BAR 0 */
1746 ret = ddi_regs_map_setup(devinfo, 1,
1747 (caddr_t *)&sc->sc_virtio.sc_io_addr,
1748 0, 0, &vioif_attr, &sc->sc_virtio.sc_ioh);
1749 if (ret != DDI_SUCCESS) {
1750 dev_err(devinfo, CE_WARN, "unable to map bar 0: %d", ret);
1751 goto exit_map;
1752 }
1753
1754 virtio_device_reset(&sc->sc_virtio);
1755 virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
1756 virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
1757
1758 ret = vioif_dev_features(sc);
1759 if (ret)
1760 goto exit_features;
1761
1762 vsc->sc_nvqs = vioif_has_feature(sc, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;
1763
1764 sc->sc_rxbuf_cache = kmem_cache_create("vioif_rx",
1765 sizeof (struct vioif_rx_buf), 0, vioif_rx_construct,
1766 vioif_rx_destruct, NULL, sc, NULL, KM_SLEEP);
1767 if (sc->sc_rxbuf_cache == NULL) {
1768 cmn_err(CE_NOTE, "Can't allocate the buffer cache");
1769 goto exit_cache;
1770 }
1771
1772 ret = vioif_register_ints(sc);
1773 if (ret) {
1774 dev_err(sc->sc_dev, CE_WARN,
1775 "Failed to allocate interrupt(s)!");
1776 goto exit_ints;
1777 }
1778
1779 /*
1780 * Register layout determined, can now access the
1781 * device-speciffic bits
1782 */
1783 vioif_get_mac(sc);
1784
1785 sc->sc_rx_vq = virtio_alloc_vq(&sc->sc_virtio, 0,
1786 VIOIF_RX_QLEN, VIOIF_INDIRECT_MAX, "rx");
1787 if (!sc->sc_rx_vq)
1788 goto exit_alloc1;
1789 virtio_stop_vq_intr(sc->sc_rx_vq);
1790
1791 sc->sc_tx_vq = virtio_alloc_vq(&sc->sc_virtio, 1,
1792 VIOIF_TX_QLEN, VIOIF_INDIRECT_MAX, "tx");
1793 if (!sc->sc_rx_vq)
1794 goto exit_alloc2;
1795 virtio_stop_vq_intr(sc->sc_tx_vq);
1796
1797 if (vioif_has_feature(sc, VIRTIO_NET_F_CTRL_VQ)) {
1798 sc->sc_ctrl_vq = virtio_alloc_vq(&sc->sc_virtio, 2,
1799 VIOIF_CTRL_QLEN, 0, "ctrl");
1800 if (!sc->sc_ctrl_vq) {
1801 goto exit_alloc3;
1802 }
1803 virtio_stop_vq_intr(sc->sc_ctrl_vq);
1804 }
1805
1806 virtio_set_status(&sc->sc_virtio,
1807 VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
1808
1809 sc->sc_rxloan = 0;
1810
1811 /* set some reasonable-small default values */
1812 sc->sc_rxcopy_thresh = 300;
1813 sc->sc_txcopy_thresh = 300;
1814 sc->sc_mtu = ETHERMTU;
1815
1816 vioif_check_features(sc);
1817
1818 if (vioif_alloc_mems(sc))
1819 goto exit_alloc_mems;
1820
1821 if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
1822 dev_err(devinfo, CE_WARN, "Failed to alocate a mac_register");
1823 goto exit_macalloc;
1824 }
1825
1826 macp->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
1827 macp->m_driver = sc;
1828 macp->m_dip = devinfo;
1829 macp->m_src_addr = sc->sc_mac;
1830 macp->m_callbacks = &vioif_m_callbacks;
1831 macp->m_min_sdu = 0;
1832 macp->m_max_sdu = sc->sc_mtu;
1833 macp->m_margin = VLAN_TAGSZ;
1834 macp->m_priv_props = vioif_priv_props;
1835
1836 sc->sc_macp = macp;
1837
1838 /* Pre-fill the rx ring. */
1839 (void) vioif_populate_rx(sc, KM_SLEEP);
1840
1841 ret = mac_register(macp, &sc->sc_mac_handle);
1842 if (ret != 0) {
1843 dev_err(devinfo, CE_WARN, "vioif_attach: "
1844 "mac_register() failed, ret=%d", ret);
1845 goto exit_register;
1846 }
1847
1848 ret = virtio_enable_ints(&sc->sc_virtio);
1849 if (ret) {
1850 dev_err(devinfo, CE_WARN, "Failed to enable interrupts");
1851 goto exit_enable_ints;
1852 }
1853
1854 mac_link_update(sc->sc_mac_handle, LINK_STATE_UP);
1855 return (DDI_SUCCESS);
1856
1857 exit_enable_ints:
1858 (void) mac_unregister(sc->sc_mac_handle);
1859 exit_register:
1860 mac_free(macp);
1861 exit_macalloc:
1862 vioif_free_mems(sc);
1863 exit_alloc_mems:
1864 virtio_release_ints(&sc->sc_virtio);
1865 if (sc->sc_ctrl_vq)
1866 virtio_free_vq(sc->sc_ctrl_vq);
1867 exit_alloc3:
1868 virtio_free_vq(sc->sc_tx_vq);
1869 exit_alloc2:
1870 virtio_free_vq(sc->sc_rx_vq);
1871 exit_alloc1:
1872 exit_ints:
1873 kmem_cache_destroy(sc->sc_rxbuf_cache);
1874 exit_cache:
1875 exit_features:
1876 virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
1877 ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
1878 exit_intrstat:
1879 exit_map:
1880 kstat_delete(sc->sc_intrstat);
1881 kmem_free(sc, sizeof (struct vioif_softc));
1882 exit:
1883 return (DDI_FAILURE);
1884 }
1885
1886 static int
1887 vioif_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
1888 {
1889 struct vioif_softc *sc;
1890
1891 if ((sc = ddi_get_driver_private(devinfo)) == NULL)
1892 return (DDI_FAILURE);
1893
1894 switch (cmd) {
1895 case DDI_DETACH:
1896 break;
1897
1898 case DDI_PM_SUSPEND:
1899 cmn_err(CE_WARN, "suspend not supported yet");
1900 return (DDI_FAILURE);
1901
1902 default:
1903 cmn_err(CE_WARN, "cmd 0x%x unrecognized", cmd);
1904 return (DDI_FAILURE);
1905 }
1906
1907 if (sc->sc_rxloan) {
1908 cmn_err(CE_NOTE, "Some rx buffers are still upstream, "
1909 "Not detaching");
1910 return (DDI_FAILURE);
1911 }
1912
1913 virtio_stop_vq_intr(sc->sc_rx_vq);
1914 virtio_stop_vq_intr(sc->sc_tx_vq);
1915
1916 virtio_release_ints(&sc->sc_virtio);
1917
1918 if (mac_unregister(sc->sc_mac_handle)) {
1919 return (DDI_FAILURE);
1920 }
1921
1922 mac_free(sc->sc_macp);
1923
1924 vioif_free_mems(sc);
1925 virtio_free_vq(sc->sc_rx_vq);
1926 virtio_free_vq(sc->sc_tx_vq);
1927
1928 virtio_device_reset(&sc->sc_virtio);
1929
1930 ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
1931
1932 kmem_cache_destroy(sc->sc_rxbuf_cache);
1933 kstat_delete(sc->sc_intrstat);
1934 kmem_free(sc, sizeof (struct vioif_softc));
1935
1936 return (DDI_SUCCESS);
1937 }
1938
1939 static int
1940 vioif_quiesce(dev_info_t *devinfo)
1941 {
1942 struct vioif_softc *sc;
1943
1944 if ((sc = ddi_get_driver_private(devinfo)) == NULL)
1945 return (DDI_FAILURE);
1946
1947 virtio_stop_vq_intr(sc->sc_rx_vq);
1948 virtio_stop_vq_intr(sc->sc_tx_vq);
1949 virtio_device_reset(&sc->sc_virtio);
1950
1951 return (DDI_SUCCESS);
1952 }
1953
1954 int
1955 _init(void)
1956 {
1957 int ret = 0;
1958
1959 mac_init_ops(&vioif_ops, "vioif");
1960
1961 ret = mod_install(&modlinkage);
1962 if (ret != DDI_SUCCESS) {
1963 mac_fini_ops(&vioif_ops);
1964 cmn_err(CE_WARN, "Unable to install the driver");
1965 return (ret);
1966 }
1967
1968 return (0);
1969 }
1970
1971 int
1972 _fini(void)
1973 {
1974 int ret;
1975
1976 ret = mod_remove(&modlinkage);
1977 if (ret == DDI_SUCCESS) {
1978 mac_fini_ops(&vioif_ops);
1979 }
1980
1981 return (ret);
1982 }
1983
1984 int
1985 _info(struct modinfo *pModinfo)
1986 {
1987 return (mod_info(&modlinkage, pModinfo));
1988 }