Print this page
7351 NVMe driver sporadically lost track of completed I/O request, which
leads to zpool hanging and machine panic.
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/nvme/nvme.c
+++ new/usr/src/uts/common/io/nvme/nvme.c
1 1 /*
2 2 * This file and its contents are supplied under the terms of the
3 3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 4 * You may only use this file in accordance with the terms of version
5 5 * 1.0 of the CDDL.
6 6 *
7 7 * A full copy of the text of the CDDL should have accompanied this
8 8 * source. A copy of the CDDL is also available via the Internet at
9 9 * http://www.illumos.org/license/CDDL.
10 10 */
11 11
12 12 /*
13 13 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
14 14 * Copyright 2016 Tegile Systems, Inc. All rights reserved.
15 15 * Copyright (c) 2016 The MathWorks, Inc. All rights reserved.
16 16 */
17 17
18 18 /*
19 19 * blkdev driver for NVMe compliant storage devices
20 20 *
21 21 * This driver was written to conform to version 1.0e of the NVMe specification.
22 22 * It may work with newer versions, but that is completely untested and disabled
23 23 * by default.
24 24 *
25 25 * The driver has only been tested on x86 systems and will not work on big-
26 26 * endian systems without changes to the code accessing registers and data
27 27 * structures used by the hardware.
28 28 *
29 29 *
30 30 * Interrupt Usage:
31 31 *
32 32 * The driver will use a FIXED interrupt while configuring the device as the
33 33 * specification requires. Later in the attach process it will switch to MSI-X
34 34 * or MSI if supported. The driver wants to have one interrupt vector per CPU,
35 35 * but it will work correctly if less are available. Interrupts can be shared
36 36 * by queues, the interrupt handler will iterate through the I/O queue array by
37 37 * steps of n_intr_cnt. Usually only the admin queue will share an interrupt
38 38 * with one I/O queue. The interrupt handler will retrieve completed commands
39 39 * from all queues sharing an interrupt vector and will post them to a taskq
40 40 * for completion processing.
41 41 *
42 42 *
43 43 * Command Processing:
44 44 *
45 45 * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up
46 46 * to 65536 I/O commands. The driver will configure one I/O queue pair per
47 47 * available interrupt vector, with the queue length usually much smaller than
48 48 * the maximum of 65536. If the hardware doesn't provide enough queues, fewer
49 49 * interrupt vectors will be used.
50 50 *
51 51 * Additionally the hardware provides a single special admin queue pair that can
52 52 * hold up to 4096 admin commands.
53 53 *
54 54 * From the hardware perspective both queues of a queue pair are independent,
55 55 * but they share some driver state: the command array (holding pointers to
56 56 * commands currently being processed by the hardware) and the active command
57 57 * counter. Access to the submission side of a queue pair and the shared state
58 58 * is protected by nq_mutex. The completion side of a queue pair does not need
59 59 * that protection apart from its access to the shared state; it is called only
60 60 * in the interrupt handler which does not run concurrently for the same
61 61 * interrupt vector.
62 62 *
63 63 * When a command is submitted to a queue pair the active command counter is
64 64 * incremented and a pointer to the command is stored in the command array. The
65 65 * array index is used as command identifier (CID) in the submission queue
66 66 * entry. Some commands may take a very long time to complete, and if the queue
67 67 * wraps around in that time a submission may find the next array slot to still
68 68 * be used by a long-running command. In this case the array is sequentially
69 69 * searched for the next free slot. The length of the command array is the same
70 70 * as the configured queue length.
71 71 *
72 72 *
73 73 * Namespace Support:
74 74 *
75 75 * NVMe devices can have multiple namespaces, each being a independent data
76 76 * store. The driver supports multiple namespaces and creates a blkdev interface
77 77 * for each namespace found. Namespaces can have various attributes to support
78 78 * thin provisioning and protection information. This driver does not support
79 79 * any of this and ignores namespaces that have these attributes.
80 80 *
81 81 *
82 82 * Blkdev Interface:
83 83 *
84 84 * This driver uses blkdev to do all the heavy lifting involved with presenting
85 85 * a disk device to the system. As a result, the processing of I/O requests is
86 86 * relatively simple as blkdev takes care of partitioning, boundary checks, DMA
87 87 * setup, and splitting of transfers into manageable chunks.
88 88 *
89 89 * I/O requests coming in from blkdev are turned into NVM commands and posted to
90 90 * an I/O queue. The queue is selected by taking the CPU id modulo the number of
91 91 * queues. There is currently no timeout handling of I/O commands.
92 92 *
93 93 * Blkdev also supports querying device/media information and generating a
94 94 * devid. The driver reports the best block size as determined by the namespace
95 95 * format back to blkdev as physical block size to support partition and block
96 96 * alignment. The devid is composed using the device vendor ID, model number,
97 97 * serial number, and the namespace ID.
98 98 *
99 99 *
100 100 * Error Handling:
101 101 *
102 102 * Error handling is currently limited to detecting fatal hardware errors,
103 103 * either by asynchronous events, or synchronously through command status or
104 104 * admin command timeouts. In case of severe errors the device is fenced off,
105 105 * all further requests will return EIO. FMA is then called to fault the device.
106 106 *
107 107 * The hardware has a limit for outstanding asynchronous event requests. Before
108 108 * this limit is known the driver assumes it is at least 1 and posts a single
109 109 * asynchronous request. Later when the limit is known more asynchronous event
110 110 * requests are posted to allow quicker reception of error information. When an
111 111 * asynchronous event is posted by the hardware the driver will parse the error
112 112 * status fields and log information or fault the device, depending on the
113 113 * severity of the asynchronous event. The asynchronous event request is then
114 114 * reused and posted to the admin queue again.
115 115 *
116 116 * On command completion the command status is checked for errors. In case of
117 117 * errors indicating a driver bug the driver panics. Almost all other error
118 118 * status values just cause EIO to be returned.
119 119 *
120 120 * Command timeouts are currently detected for all admin commands except
121 121 * asynchronous event requests. If a command times out and the hardware appears
122 122 * to be healthy the driver attempts to abort the command. If this fails the
123 123 * driver assumes the device to be dead, fences it off, and calls FMA to retire
124 124 * it. In general admin commands are issued at attach time only. No timeout
125 125 * handling of normal I/O commands is presently done.
126 126 *
127 127 * In some cases it may be possible that the ABORT command times out, too. In
128 128 * that case the device is also declared dead and fenced off.
129 129 *
130 130 *
131 131 * Quiesce / Fast Reboot:
132 132 *
133 133 * The driver currently does not support fast reboot. A quiesce(9E) entry point
134 134 * is still provided which is used to send a shutdown notification to the
135 135 * device.
136 136 *
137 137 *
138 138 * Driver Configuration:
139 139 *
140 140 * The following driver properties can be changed to control some aspects of the
141 141 * drivers operation:
142 142 * - strict-version: can be set to 0 to allow devices conforming to newer
143 143 * versions to be used
144 144 * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor
145 145 * specific command status as a fatal error leading device faulting
146 146 * - admin-queue-len: the maximum length of the admin queue (16-4096)
147 147 * - io-queue-len: the maximum length of the I/O queues (16-65536)
148 148 * - async-event-limit: the maximum number of asynchronous event requests to be
149 149 * posted by the driver
150 150 *
151 151 *
152 152 * TODO:
153 153 * - figure out sane default for I/O queue depth reported to blkdev
154 154 * - polled I/O support to support kernel core dumping
155 155 * - FMA handling of media errors
156 156 * - support for the Volatile Write Cache
157 157 * - support for devices supporting very large I/O requests using chained PRPs
158 158 * - support for querying log pages from user space
159 159 * - support for configuring hardware parameters like interrupt coalescing
160 160 * - support for media formatting and hard partitioning into namespaces
161 161 * - support for big-endian systems
162 162 * - support for fast reboot
163 163 */
164 164
165 165 #include <sys/byteorder.h>
166 166 #ifdef _BIG_ENDIAN
167 167 #error nvme driver needs porting for big-endian platforms
168 168 #endif
169 169
170 170 #include <sys/modctl.h>
171 171 #include <sys/conf.h>
172 172 #include <sys/devops.h>
173 173 #include <sys/ddi.h>
174 174 #include <sys/sunddi.h>
|
↓ open down ↓ |
174 lines elided |
↑ open up ↑ |
175 175 #include <sys/bitmap.h>
176 176 #include <sys/sysmacros.h>
177 177 #include <sys/param.h>
178 178 #include <sys/varargs.h>
179 179 #include <sys/cpuvar.h>
180 180 #include <sys/disp.h>
181 181 #include <sys/blkdev.h>
182 182 #include <sys/atomic.h>
183 183 #include <sys/archsystm.h>
184 184 #include <sys/sata/sata_hba.h>
185 +#include <sys/time.h>
185 186
186 187 #include "nvme_reg.h"
187 188 #include "nvme_var.h"
188 189
189 190
190 191 /* NVMe spec version supported */
191 192 static const int nvme_version_major = 1;
192 193 static const int nvme_version_minor = 0;
193 194
194 195 /* tunable for admin command timeout in seconds, default is 1s */
195 196 static volatile int nvme_admin_cmd_timeout = 1;
196 197
197 198 static int nvme_attach(dev_info_t *, ddi_attach_cmd_t);
198 199 static int nvme_detach(dev_info_t *, ddi_detach_cmd_t);
199 200 static int nvme_quiesce(dev_info_t *);
200 201 static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *);
201 202 static int nvme_setup_interrupts(nvme_t *, int, int);
202 203 static void nvme_release_interrupts(nvme_t *);
203 204 static uint_t nvme_intr(caddr_t, caddr_t);
|
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
204 205
205 206 static void nvme_shutdown(nvme_t *, int, boolean_t);
206 207 static boolean_t nvme_reset(nvme_t *, boolean_t);
207 208 static int nvme_init(nvme_t *);
208 209 static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int);
209 210 static void nvme_free_cmd(nvme_cmd_t *);
210 211 static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t,
211 212 bd_xfer_t *);
212 213 static int nvme_admin_cmd(nvme_cmd_t *, int);
213 214 static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *);
214 -static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *);
215 +static int nvme_process_cq_cmds(nvme_t *, nvme_qpair_t *);
215 216 static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t);
216 217 static void nvme_wakeup_cmd(void *);
217 218 static void nvme_async_event_task(void *);
218 219
219 220 static int nvme_check_unknown_cmd_status(nvme_cmd_t *);
220 221 static int nvme_check_vendor_cmd_status(nvme_cmd_t *);
221 222 static int nvme_check_integrity_cmd_status(nvme_cmd_t *);
222 223 static int nvme_check_specific_cmd_status(nvme_cmd_t *);
223 224 static int nvme_check_generic_cmd_status(nvme_cmd_t *);
224 225 static inline int nvme_check_cmd_status(nvme_cmd_t *);
225 226
226 227 static void nvme_abort_cmd(nvme_cmd_t *);
227 228 static int nvme_async_event(nvme_t *);
228 229 static void *nvme_get_logpage(nvme_t *, uint8_t, ...);
229 230 static void *nvme_identify(nvme_t *, uint32_t);
230 231 static int nvme_set_nqueues(nvme_t *, uint16_t);
231 232
232 233 static void nvme_free_dma(nvme_dma_t *);
233 234 static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *,
234 235 nvme_dma_t **);
235 236 static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t,
236 237 nvme_dma_t **);
237 238 static void nvme_free_qpair(nvme_qpair_t *);
238 239 static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int);
239 240 static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t);
240 241
241 242 static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t);
242 243 static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t);
243 244 static inline uint64_t nvme_get64(nvme_t *, uintptr_t);
244 245 static inline uint32_t nvme_get32(nvme_t *, uintptr_t);
245 246
246 247 static boolean_t nvme_check_regs_hdl(nvme_t *);
247 248 static boolean_t nvme_check_dma_hdl(nvme_dma_t *);
248 249
249 250 static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *);
250 251
251 252 static void nvme_bd_xfer_done(void *);
|
↓ open down ↓ |
27 lines elided |
↑ open up ↑ |
252 253 static void nvme_bd_driveinfo(void *, bd_drive_t *);
253 254 static int nvme_bd_mediainfo(void *, bd_media_t *);
254 255 static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t);
255 256 static int nvme_bd_read(void *, bd_xfer_t *);
256 257 static int nvme_bd_write(void *, bd_xfer_t *);
257 258 static int nvme_bd_sync(void *, bd_xfer_t *);
258 259 static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *);
259 260
260 261 static void nvme_prepare_devid(nvme_t *, uint32_t);
261 262
263 +static void nvme_intr_monitor(void *arg);
264 +
262 265 static void *nvme_state;
263 266 static kmem_cache_t *nvme_cmd_cache;
264 267
268 +static list_t nvme_qp_list;
269 +static kmutex_t nvme_global_mutex;
270 +static ddi_periodic_t nvme_cyclic;
271 +int nvme_cyclic_seconds = 5;
272 +hrtime_t nvme_intr_timeout_ns = 3 * NANOSEC;
273 +uint64_t nvme_intr_timeouts = 0;
274 +boolean_t nvme_enable_intr_monitoring = B_TRUE;
275 +
265 276 /*
266 277 * DMA attributes for queue DMA memory
267 278 *
268 279 * Queue DMA memory must be page aligned. The maximum length of a queue is
269 280 * 65536 entries, and an entry can be 64 bytes long.
270 281 */
271 282 static ddi_dma_attr_t nvme_queue_dma_attr = {
272 283 .dma_attr_version = DMA_ATTR_V0,
273 284 .dma_attr_addr_lo = 0,
274 285 .dma_attr_addr_hi = 0xffffffffffffffffULL,
275 286 .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1,
276 287 .dma_attr_align = 0x1000,
277 288 .dma_attr_burstsizes = 0x7ff,
278 289 .dma_attr_minxfer = 0x1000,
279 290 .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t),
280 291 .dma_attr_seg = 0xffffffffffffffffULL,
281 292 .dma_attr_sgllen = 1,
282 293 .dma_attr_granular = 1,
283 294 .dma_attr_flags = 0,
284 295 };
285 296
286 297 /*
287 298 * DMA attributes for transfers using Physical Region Page (PRP) entries
288 299 *
289 300 * A PRP entry describes one page of DMA memory using the page size specified
290 301 * in the controller configuration's memory page size register (CC.MPS). It uses
291 302 * a 64bit base address aligned to this page size. There is no limitation on
292 303 * chaining PRPs together for arbitrarily large DMA transfers.
293 304 */
294 305 static ddi_dma_attr_t nvme_prp_dma_attr = {
295 306 .dma_attr_version = DMA_ATTR_V0,
296 307 .dma_attr_addr_lo = 0,
297 308 .dma_attr_addr_hi = 0xffffffffffffffffULL,
298 309 .dma_attr_count_max = 0xfff,
299 310 .dma_attr_align = 0x1000,
300 311 .dma_attr_burstsizes = 0x7ff,
301 312 .dma_attr_minxfer = 0x1000,
302 313 .dma_attr_maxxfer = 0x1000,
303 314 .dma_attr_seg = 0xfff,
304 315 .dma_attr_sgllen = -1,
305 316 .dma_attr_granular = 1,
306 317 .dma_attr_flags = 0,
307 318 };
308 319
309 320 /*
310 321 * DMA attributes for transfers using scatter/gather lists
311 322 *
312 323 * A SGL entry describes a chunk of DMA memory using a 64bit base address and a
313 324 * 32bit length field. SGL Segment and SGL Last Segment entries require the
314 325 * length to be a multiple of 16 bytes.
315 326 */
316 327 static ddi_dma_attr_t nvme_sgl_dma_attr = {
317 328 .dma_attr_version = DMA_ATTR_V0,
318 329 .dma_attr_addr_lo = 0,
319 330 .dma_attr_addr_hi = 0xffffffffffffffffULL,
320 331 .dma_attr_count_max = 0xffffffffUL,
321 332 .dma_attr_align = 1,
322 333 .dma_attr_burstsizes = 0x7ff,
323 334 .dma_attr_minxfer = 0x10,
324 335 .dma_attr_maxxfer = 0xfffffffffULL,
325 336 .dma_attr_seg = 0xffffffffffffffffULL,
326 337 .dma_attr_sgllen = -1,
327 338 .dma_attr_granular = 0x10,
328 339 .dma_attr_flags = 0
329 340 };
330 341
331 342 static ddi_device_acc_attr_t nvme_reg_acc_attr = {
332 343 .devacc_attr_version = DDI_DEVICE_ATTR_V0,
333 344 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
334 345 .devacc_attr_dataorder = DDI_STRICTORDER_ACC
335 346 };
336 347
337 348 static struct dev_ops nvme_dev_ops = {
338 349 .devo_rev = DEVO_REV,
339 350 .devo_refcnt = 0,
340 351 .devo_getinfo = ddi_no_info,
341 352 .devo_identify = nulldev,
342 353 .devo_probe = nulldev,
343 354 .devo_attach = nvme_attach,
344 355 .devo_detach = nvme_detach,
345 356 .devo_reset = nodev,
346 357 .devo_cb_ops = NULL,
347 358 .devo_bus_ops = NULL,
348 359 .devo_power = NULL,
349 360 .devo_quiesce = nvme_quiesce,
350 361 };
351 362
352 363 static struct modldrv nvme_modldrv = {
353 364 .drv_modops = &mod_driverops,
354 365 .drv_linkinfo = "NVMe v1.0e",
355 366 .drv_dev_ops = &nvme_dev_ops
356 367 };
357 368
358 369 static struct modlinkage nvme_modlinkage = {
359 370 .ml_rev = MODREV_1,
360 371 .ml_linkage = { &nvme_modldrv, NULL }
361 372 };
362 373
363 374 static bd_ops_t nvme_bd_ops = {
364 375 .o_version = BD_OPS_VERSION_0,
365 376 .o_drive_info = nvme_bd_driveinfo,
366 377 .o_media_info = nvme_bd_mediainfo,
367 378 .o_devid_init = nvme_bd_devid,
368 379 .o_sync_cache = nvme_bd_sync,
369 380 .o_read = nvme_bd_read,
370 381 .o_write = nvme_bd_write,
371 382 };
372 383
373 384 int
374 385 _init(void)
|
↓ open down ↓ |
100 lines elided |
↑ open up ↑ |
375 386 {
376 387 int error;
377 388
378 389 error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1);
379 390 if (error != DDI_SUCCESS)
380 391 return (error);
381 392
382 393 nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache",
383 394 sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
384 395
396 + mutex_init(&nvme_global_mutex, NULL, MUTEX_DRIVER, 0);
397 +
398 + list_create(&nvme_qp_list, sizeof (nvme_qpair_t),
399 + offsetof(nvme_qpair_t, nq_list_node));
400 +
401 + nvme_cyclic = ddi_periodic_add(nvme_intr_monitor, NULL,
402 + NANOSEC * nvme_cyclic_seconds, DDI_IPL_0);
403 +
385 404 bd_mod_init(&nvme_dev_ops);
386 405
387 406 error = mod_install(&nvme_modlinkage);
388 407 if (error != DDI_SUCCESS) {
389 408 ddi_soft_state_fini(&nvme_state);
390 409 bd_mod_fini(&nvme_dev_ops);
391 410 }
392 411
393 412 return (error);
394 413 }
395 414
396 415 int
397 416 _fini(void)
398 417 {
399 418 int error;
400 419
401 420 error = mod_remove(&nvme_modlinkage);
402 421 if (error == DDI_SUCCESS) {
403 422 ddi_soft_state_fini(&nvme_state);
404 423 kmem_cache_destroy(nvme_cmd_cache);
424 + if (nvme_cyclic != NULL) {
425 + ddi_periodic_delete(nvme_cyclic);
426 + nvme_cyclic = NULL;
427 + }
428 + mutex_destroy(&nvme_global_mutex);
405 429 bd_mod_fini(&nvme_dev_ops);
406 430 }
407 431
408 432 return (error);
409 433 }
410 434
411 435 int
412 436 _info(struct modinfo *modinfop)
413 437 {
414 438 return (mod_info(&nvme_modlinkage, modinfop));
415 439 }
416 440
417 441 static inline void
418 442 nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val)
419 443 {
420 444 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0);
421 445
422 446 /*LINTED: E_BAD_PTR_CAST_ALIGN*/
423 447 ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val);
424 448 }
425 449
426 450 static inline void
427 451 nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val)
428 452 {
429 453 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0);
430 454
431 455 /*LINTED: E_BAD_PTR_CAST_ALIGN*/
432 456 ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val);
433 457 }
434 458
435 459 static inline uint64_t
436 460 nvme_get64(nvme_t *nvme, uintptr_t reg)
437 461 {
438 462 uint64_t val;
439 463
440 464 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0);
441 465
442 466 /*LINTED: E_BAD_PTR_CAST_ALIGN*/
443 467 val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg));
444 468
445 469 return (val);
446 470 }
447 471
448 472 static inline uint32_t
449 473 nvme_get32(nvme_t *nvme, uintptr_t reg)
450 474 {
451 475 uint32_t val;
452 476
453 477 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0);
454 478
455 479 /*LINTED: E_BAD_PTR_CAST_ALIGN*/
456 480 val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg));
457 481
458 482 return (val);
459 483 }
460 484
461 485 static boolean_t
462 486 nvme_check_regs_hdl(nvme_t *nvme)
463 487 {
464 488 ddi_fm_error_t error;
465 489
466 490 ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION);
467 491
468 492 if (error.fme_status != DDI_FM_OK)
469 493 return (B_TRUE);
470 494
471 495 return (B_FALSE);
472 496 }
473 497
474 498 static boolean_t
475 499 nvme_check_dma_hdl(nvme_dma_t *dma)
476 500 {
477 501 ddi_fm_error_t error;
478 502
479 503 if (dma == NULL)
480 504 return (B_FALSE);
481 505
482 506 ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION);
483 507
484 508 if (error.fme_status != DDI_FM_OK)
485 509 return (B_TRUE);
486 510
487 511 return (B_FALSE);
488 512 }
489 513
490 514 static void
491 515 nvme_free_dma(nvme_dma_t *dma)
492 516 {
493 517 if (dma->nd_dmah != NULL)
494 518 (void) ddi_dma_unbind_handle(dma->nd_dmah);
495 519 if (dma->nd_acch != NULL)
496 520 ddi_dma_mem_free(&dma->nd_acch);
497 521 if (dma->nd_dmah != NULL)
498 522 ddi_dma_free_handle(&dma->nd_dmah);
499 523 kmem_free(dma, sizeof (nvme_dma_t));
500 524 }
501 525
502 526 static int
503 527 nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
504 528 ddi_dma_attr_t *dma_attr, nvme_dma_t **ret)
505 529 {
506 530 nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP);
507 531
508 532 if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL,
509 533 &dma->nd_dmah) != DDI_SUCCESS) {
510 534 /*
511 535 * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and
512 536 * the only other possible error is DDI_DMA_BADATTR which
513 537 * indicates a driver bug which should cause a panic.
514 538 */
515 539 dev_err(nvme->n_dip, CE_PANIC,
516 540 "!failed to get DMA handle, check DMA attributes");
517 541 return (DDI_FAILURE);
518 542 }
519 543
520 544 /*
521 545 * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified
522 546 * or the flags are conflicting, which isn't the case here.
523 547 */
524 548 (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr,
525 549 DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp,
526 550 &dma->nd_len, &dma->nd_acch);
527 551
528 552 if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp,
529 553 dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
530 554 &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) {
531 555 dev_err(nvme->n_dip, CE_WARN,
532 556 "!failed to bind DMA memory");
533 557 atomic_inc_32(&nvme->n_dma_bind_err);
534 558 *ret = NULL;
535 559 nvme_free_dma(dma);
536 560 return (DDI_FAILURE);
537 561 }
538 562
539 563 bzero(dma->nd_memp, dma->nd_len);
540 564
541 565 *ret = dma;
542 566 return (DDI_SUCCESS);
543 567 }
544 568
545 569 static int
546 570 nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len,
547 571 uint_t flags, nvme_dma_t **dma)
548 572 {
549 573 uint32_t len = nentry * qe_len;
550 574 ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr;
551 575
552 576 len = roundup(len, nvme->n_pagesize);
553 577
554 578 q_dma_attr.dma_attr_minxfer = len;
555 579
556 580 if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma)
557 581 != DDI_SUCCESS) {
558 582 dev_err(nvme->n_dip, CE_WARN,
559 583 "!failed to get DMA memory for queue");
560 584 goto fail;
561 585 }
562 586
563 587 if ((*dma)->nd_ncookie != 1) {
564 588 dev_err(nvme->n_dip, CE_WARN,
565 589 "!got too many cookies for queue DMA");
566 590 goto fail;
567 591 }
568 592
569 593 return (DDI_SUCCESS);
570 594
571 595 fail:
572 596 if (*dma) {
573 597 nvme_free_dma(*dma);
574 598 *dma = NULL;
575 599 }
576 600
577 601 return (DDI_FAILURE);
578 602 }
579 603
580 604 static void
581 605 nvme_free_qpair(nvme_qpair_t *qp)
582 606 {
583 607 int i;
584 608
585 609 mutex_destroy(&qp->nq_mutex);
586 610
587 611 if (qp->nq_sqdma != NULL)
588 612 nvme_free_dma(qp->nq_sqdma);
589 613 if (qp->nq_cqdma != NULL)
590 614 nvme_free_dma(qp->nq_cqdma);
591 615
592 616 if (qp->nq_active_cmds > 0)
593 617 for (i = 0; i != qp->nq_nentry; i++)
594 618 if (qp->nq_cmd[i] != NULL)
595 619 nvme_free_cmd(qp->nq_cmd[i]);
596 620
597 621 if (qp->nq_cmd != NULL)
598 622 kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry);
599 623
600 624 kmem_free(qp, sizeof (nvme_qpair_t));
601 625 }
602 626
603 627 static int
604 628 nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp,
605 629 int idx)
606 630 {
607 631 nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP);
608 632
609 633 mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER,
610 634 DDI_INTR_PRI(nvme->n_intr_pri));
611 635
612 636 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t),
613 637 DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS)
614 638 goto fail;
615 639
616 640 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t),
617 641 DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS)
618 642 goto fail;
619 643
620 644 qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp;
621 645 qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp;
622 646 qp->nq_nentry = nentry;
623 647
624 648 qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx);
625 649 qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx);
626 650
627 651 qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP);
628 652 qp->nq_next_cmd = 0;
629 653
630 654 *nqp = qp;
631 655 return (DDI_SUCCESS);
632 656
633 657 fail:
634 658 nvme_free_qpair(qp);
635 659 *nqp = NULL;
636 660
637 661 return (DDI_FAILURE);
638 662 }
639 663
640 664 static nvme_cmd_t *
641 665 nvme_alloc_cmd(nvme_t *nvme, int kmflag)
642 666 {
643 667 nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag);
644 668
645 669 if (cmd == NULL)
646 670 return (cmd);
647 671
648 672 bzero(cmd, sizeof (nvme_cmd_t));
649 673
650 674 cmd->nc_nvme = nvme;
651 675
652 676 mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER,
653 677 DDI_INTR_PRI(nvme->n_intr_pri));
654 678 cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL);
655 679
656 680 return (cmd);
657 681 }
658 682
659 683 static void
660 684 nvme_free_cmd(nvme_cmd_t *cmd)
661 685 {
662 686 if (cmd->nc_dma) {
663 687 nvme_free_dma(cmd->nc_dma);
664 688 cmd->nc_dma = NULL;
665 689 }
666 690
667 691 cv_destroy(&cmd->nc_cv);
668 692 mutex_destroy(&cmd->nc_mutex);
669 693
670 694 kmem_cache_free(nvme_cmd_cache, cmd);
671 695 }
672 696
673 697 static int
674 698 nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
675 699 {
676 700 nvme_reg_sqtdbl_t tail = { 0 };
677 701
678 702 mutex_enter(&qp->nq_mutex);
679 703
680 704 if (qp->nq_active_cmds == qp->nq_nentry) {
681 705 mutex_exit(&qp->nq_mutex);
682 706 return (DDI_FAILURE);
683 707 }
684 708
685 709 cmd->nc_completed = B_FALSE;
686 710
687 711 /*
688 712 * Try to insert the cmd into the active cmd array at the nq_next_cmd
689 713 * slot. If the slot is already occupied advance to the next slot and
690 714 * try again. This can happen for long running commands like async event
691 715 * requests.
692 716 */
693 717 while (qp->nq_cmd[qp->nq_next_cmd] != NULL)
694 718 qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry;
695 719 qp->nq_cmd[qp->nq_next_cmd] = cmd;
696 720
697 721 qp->nq_active_cmds++;
698 722
|
↓ open down ↓ |
284 lines elided |
↑ open up ↑ |
699 723 cmd->nc_sqe.sqe_cid = qp->nq_next_cmd;
700 724 bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t));
701 725 (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah,
702 726 sizeof (nvme_sqe_t) * qp->nq_sqtail,
703 727 sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV);
704 728 qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry;
705 729
706 730 tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry;
707 731 nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r);
708 732
733 + if (nvme_enable_intr_monitoring)
734 + qp->nq_ts = gethrtime();
709 735 mutex_exit(&qp->nq_mutex);
736 +
710 737 return (DDI_SUCCESS);
711 738 }
712 739
713 -static nvme_cmd_t *
714 -nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp)
740 +static int
741 +nvme_process_cq_cmds(nvme_t *nvme, nvme_qpair_t *qp)
715 742 {
716 743 nvme_reg_cqhdbl_t head = { 0 };
717 744
718 745 nvme_cqe_t *cqe;
719 746 nvme_cmd_t *cmd;
747 + int cnt_cmds = 0;
720 748
721 749 (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0,
722 750 sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL);
723 751
724 752 cqe = &qp->nq_cq[qp->nq_cqhead];
725 -
726 753 /* Check phase tag of CQE. Hardware inverts it for new entries. */
727 754 if (cqe->cqe_sf.sf_p == qp->nq_phase)
728 - return (NULL);
729 -
730 - ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp);
731 - ASSERT(cqe->cqe_cid < qp->nq_nentry);
755 + return (cnt_cmds);
732 756
733 757 mutex_enter(&qp->nq_mutex);
734 - cmd = qp->nq_cmd[cqe->cqe_cid];
735 - qp->nq_cmd[cqe->cqe_cid] = NULL;
736 - qp->nq_active_cmds--;
737 - mutex_exit(&qp->nq_mutex);
738 -
739 - ASSERT(cmd != NULL);
740 - ASSERT(cmd->nc_nvme == nvme);
741 - ASSERT(cmd->nc_sqid == cqe->cqe_sqid);
742 - ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid);
743 - bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t));
744 -
745 - qp->nq_sqhead = cqe->cqe_sqhd;
746 -
747 - head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry;
748 -
749 - /* Toggle phase on wrap-around. */
750 - if (qp->nq_cqhead == 0)
751 - qp->nq_phase = qp->nq_phase ? 0 : 1;
758 + while (cqe->cqe_sf.sf_p != qp->nq_phase) {
759 + ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp);
760 + ASSERT(cqe->cqe_cid < qp->nq_nentry);
761 +
762 + cmd = qp->nq_cmd[cqe->cqe_cid];
763 + qp->nq_cmd[cqe->cqe_cid] = NULL;
764 + qp->nq_active_cmds--;
765 +
766 + ASSERT(cmd != NULL);
767 + ASSERT(cmd->nc_nvme == nvme);
768 + ASSERT(cmd->nc_sqid == cqe->cqe_sqid);
769 + ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid);
770 + bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t));
771 +
772 + qp->nq_sqhead = cqe->cqe_sqhd;
773 +
774 + qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry;
775 +
776 + /* Toggle phase on wrap-around. */
777 + if (qp->nq_cqhead == 0)
778 + qp->nq_phase = qp->nq_phase ? 0 : 1;
779 + taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq,
780 + cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent);
781 + cnt_cmds++;
782 + cqe = &qp->nq_cq[qp->nq_cqhead];
783 + }
784 +
785 + if (cnt_cmds != 0) {
786 + head.b.cqhdbl_cqh = qp->nq_cqhead;
787 + nvme_put32(nvme, qp->nq_cqhdbl, head.r);
788 + if (nvme_enable_intr_monitoring)
789 + qp->nq_ts = gethrtime();
790 + }
752 791
753 - nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r);
792 + mutex_exit(&qp->nq_mutex);
754 793
755 - return (cmd);
794 + return (cnt_cmds);
756 795 }
757 796
758 797 static int
759 798 nvme_check_unknown_cmd_status(nvme_cmd_t *cmd)
760 799 {
761 800 nvme_cqe_t *cqe = &cmd->nc_cqe;
762 801
763 802 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
764 803 "!unknown command status received: opc = %x, sqid = %d, cid = %d, "
765 804 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc,
766 805 cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct,
767 806 cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m);
768 807
769 808 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
770 809
771 810 if (cmd->nc_nvme->n_strict_version) {
772 811 cmd->nc_nvme->n_dead = B_TRUE;
773 812 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
774 813 }
775 814
776 815 return (EIO);
777 816 }
778 817
779 818 static int
780 819 nvme_check_vendor_cmd_status(nvme_cmd_t *cmd)
781 820 {
782 821 nvme_cqe_t *cqe = &cmd->nc_cqe;
783 822
784 823 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
785 824 "!unknown command status received: opc = %x, sqid = %d, cid = %d, "
786 825 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc,
787 826 cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct,
788 827 cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m);
789 828 if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) {
790 829 cmd->nc_nvme->n_dead = B_TRUE;
791 830 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
792 831 }
793 832
794 833 return (EIO);
795 834 }
796 835
797 836 static int
798 837 nvme_check_integrity_cmd_status(nvme_cmd_t *cmd)
799 838 {
800 839 nvme_cqe_t *cqe = &cmd->nc_cqe;
801 840
802 841 switch (cqe->cqe_sf.sf_sc) {
803 842 case NVME_CQE_SC_INT_NVM_WRITE:
804 843 /* write fail */
805 844 /* TODO: post ereport */
806 845 bd_error(cmd->nc_xfer, BD_ERR_MEDIA);
807 846 return (EIO);
808 847
809 848 case NVME_CQE_SC_INT_NVM_READ:
810 849 /* read fail */
811 850 /* TODO: post ereport */
812 851 bd_error(cmd->nc_xfer, BD_ERR_MEDIA);
813 852 return (EIO);
814 853
815 854 default:
816 855 return (nvme_check_unknown_cmd_status(cmd));
817 856 }
818 857 }
819 858
820 859 static int
821 860 nvme_check_generic_cmd_status(nvme_cmd_t *cmd)
822 861 {
823 862 nvme_cqe_t *cqe = &cmd->nc_cqe;
824 863
825 864 switch (cqe->cqe_sf.sf_sc) {
826 865 case NVME_CQE_SC_GEN_SUCCESS:
827 866 return (0);
828 867
829 868 /*
830 869 * Errors indicating a bug in the driver should cause a panic.
831 870 */
832 871 case NVME_CQE_SC_GEN_INV_OPC:
833 872 /* Invalid Command Opcode */
834 873 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
835 874 "invalid opcode in cmd %p", (void *)cmd);
836 875 return (0);
837 876
838 877 case NVME_CQE_SC_GEN_INV_FLD:
839 878 /* Invalid Field in Command */
840 879 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
841 880 "invalid field in cmd %p", (void *)cmd);
842 881 return (0);
843 882
844 883 case NVME_CQE_SC_GEN_ID_CNFL:
845 884 /* Command ID Conflict */
846 885 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
847 886 "cmd ID conflict in cmd %p", (void *)cmd);
848 887 return (0);
849 888
850 889 case NVME_CQE_SC_GEN_INV_NS:
851 890 /* Invalid Namespace or Format */
852 891 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
853 892 "invalid NS/format in cmd %p", (void *)cmd);
854 893 return (0);
855 894
856 895 case NVME_CQE_SC_GEN_NVM_LBA_RANGE:
857 896 /* LBA Out Of Range */
858 897 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
859 898 "LBA out of range in cmd %p", (void *)cmd);
860 899 return (0);
861 900
862 901 /*
863 902 * Non-fatal errors, handle gracefully.
864 903 */
865 904 case NVME_CQE_SC_GEN_DATA_XFR_ERR:
866 905 /* Data Transfer Error (DMA) */
867 906 /* TODO: post ereport */
868 907 atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err);
869 908 bd_error(cmd->nc_xfer, BD_ERR_NTRDY);
870 909 return (EIO);
871 910
872 911 case NVME_CQE_SC_GEN_INTERNAL_ERR:
873 912 /*
874 913 * Internal Error. The spec (v1.0, section 4.5.1.2) says
875 914 * detailed error information is returned as async event,
876 915 * so we pretty much ignore the error here and handle it
877 916 * in the async event handler.
878 917 */
879 918 atomic_inc_32(&cmd->nc_nvme->n_internal_err);
880 919 bd_error(cmd->nc_xfer, BD_ERR_NTRDY);
881 920 return (EIO);
882 921
883 922 case NVME_CQE_SC_GEN_ABORT_REQUEST:
884 923 /*
885 924 * Command Abort Requested. This normally happens only when a
886 925 * command times out.
887 926 */
888 927 /* TODO: post ereport or change blkdev to handle this? */
889 928 atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err);
890 929 return (ECANCELED);
891 930
892 931 case NVME_CQE_SC_GEN_ABORT_PWRLOSS:
893 932 /* Command Aborted due to Power Loss Notification */
894 933 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
895 934 cmd->nc_nvme->n_dead = B_TRUE;
896 935 return (EIO);
897 936
898 937 case NVME_CQE_SC_GEN_ABORT_SQ_DEL:
899 938 /* Command Aborted due to SQ Deletion */
900 939 atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del);
901 940 return (EIO);
902 941
903 942 case NVME_CQE_SC_GEN_NVM_CAP_EXC:
904 943 /* Capacity Exceeded */
905 944 atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc);
906 945 bd_error(cmd->nc_xfer, BD_ERR_MEDIA);
907 946 return (EIO);
908 947
909 948 case NVME_CQE_SC_GEN_NVM_NS_NOTRDY:
910 949 /* Namespace Not Ready */
911 950 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy);
912 951 bd_error(cmd->nc_xfer, BD_ERR_NTRDY);
913 952 return (EIO);
914 953
915 954 default:
916 955 return (nvme_check_unknown_cmd_status(cmd));
917 956 }
918 957 }
919 958
920 959 static int
921 960 nvme_check_specific_cmd_status(nvme_cmd_t *cmd)
922 961 {
923 962 nvme_cqe_t *cqe = &cmd->nc_cqe;
924 963
925 964 switch (cqe->cqe_sf.sf_sc) {
926 965 case NVME_CQE_SC_SPC_INV_CQ:
927 966 /* Completion Queue Invalid */
928 967 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE);
929 968 atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err);
930 969 return (EINVAL);
931 970
932 971 case NVME_CQE_SC_SPC_INV_QID:
933 972 /* Invalid Queue Identifier */
934 973 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE ||
935 974 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE ||
936 975 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE ||
937 976 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE);
938 977 atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err);
939 978 return (EINVAL);
940 979
941 980 case NVME_CQE_SC_SPC_MAX_QSZ_EXC:
942 981 /* Max Queue Size Exceeded */
943 982 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE ||
944 983 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE);
945 984 atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc);
946 985 return (EINVAL);
947 986
948 987 case NVME_CQE_SC_SPC_ABRT_CMD_EXC:
949 988 /* Abort Command Limit Exceeded */
950 989 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT);
951 990 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
952 991 "abort command limit exceeded in cmd %p", (void *)cmd);
953 992 return (0);
954 993
955 994 case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC:
956 995 /* Async Event Request Limit Exceeded */
957 996 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT);
958 997 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
959 998 "async event request limit exceeded in cmd %p",
960 999 (void *)cmd);
961 1000 return (0);
962 1001
963 1002 case NVME_CQE_SC_SPC_INV_INT_VECT:
964 1003 /* Invalid Interrupt Vector */
965 1004 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE);
966 1005 atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect);
967 1006 return (EINVAL);
968 1007
969 1008 case NVME_CQE_SC_SPC_INV_LOG_PAGE:
970 1009 /* Invalid Log Page */
971 1010 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE);
972 1011 atomic_inc_32(&cmd->nc_nvme->n_inv_log_page);
973 1012 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
974 1013 return (EINVAL);
975 1014
976 1015 case NVME_CQE_SC_SPC_INV_FORMAT:
977 1016 /* Invalid Format */
978 1017 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT);
979 1018 atomic_inc_32(&cmd->nc_nvme->n_inv_format);
980 1019 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
981 1020 return (EINVAL);
982 1021
983 1022 case NVME_CQE_SC_SPC_INV_Q_DEL:
984 1023 /* Invalid Queue Deletion */
985 1024 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE);
986 1025 atomic_inc_32(&cmd->nc_nvme->n_inv_q_del);
987 1026 return (EINVAL);
988 1027
989 1028 case NVME_CQE_SC_SPC_NVM_CNFL_ATTR:
990 1029 /* Conflicting Attributes */
991 1030 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT ||
992 1031 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ ||
993 1032 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
994 1033 atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr);
995 1034 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
996 1035 return (EINVAL);
997 1036
998 1037 case NVME_CQE_SC_SPC_NVM_INV_PROT:
999 1038 /* Invalid Protection Information */
1000 1039 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE ||
1001 1040 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ ||
1002 1041 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
1003 1042 atomic_inc_32(&cmd->nc_nvme->n_inv_prot);
1004 1043 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1005 1044 return (EINVAL);
1006 1045
1007 1046 case NVME_CQE_SC_SPC_NVM_READONLY:
1008 1047 /* Write to Read Only Range */
1009 1048 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
1010 1049 atomic_inc_32(&cmd->nc_nvme->n_readonly);
1011 1050 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1012 1051 return (EROFS);
1013 1052
1014 1053 default:
1015 1054 return (nvme_check_unknown_cmd_status(cmd));
1016 1055 }
1017 1056 }
1018 1057
1019 1058 static inline int
1020 1059 nvme_check_cmd_status(nvme_cmd_t *cmd)
1021 1060 {
1022 1061 nvme_cqe_t *cqe = &cmd->nc_cqe;
1023 1062
1024 1063 /* take a shortcut if everything is alright */
1025 1064 if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC &&
1026 1065 cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS)
1027 1066 return (0);
1028 1067
1029 1068 if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC)
1030 1069 return (nvme_check_generic_cmd_status(cmd));
1031 1070 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC)
1032 1071 return (nvme_check_specific_cmd_status(cmd));
1033 1072 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY)
1034 1073 return (nvme_check_integrity_cmd_status(cmd));
1035 1074 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR)
1036 1075 return (nvme_check_vendor_cmd_status(cmd));
1037 1076
1038 1077 return (nvme_check_unknown_cmd_status(cmd));
1039 1078 }
1040 1079
1041 1080 /*
1042 1081 * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands
1043 1082 *
1044 1083 * This functions takes care of cleaning up aborted commands. The command
1045 1084 * status is checked to catch any fatal errors.
1046 1085 */
1047 1086 static void
1048 1087 nvme_abort_cmd_cb(void *arg)
1049 1088 {
1050 1089 nvme_cmd_t *cmd = arg;
1051 1090
1052 1091 /*
1053 1092 * Grab the command mutex. Once we have it we hold the last reference
1054 1093 * to the command and can safely free it.
1055 1094 */
1056 1095 mutex_enter(&cmd->nc_mutex);
1057 1096 (void) nvme_check_cmd_status(cmd);
1058 1097 mutex_exit(&cmd->nc_mutex);
1059 1098
1060 1099 nvme_free_cmd(cmd);
1061 1100 }
1062 1101
1063 1102 static void
1064 1103 nvme_abort_cmd(nvme_cmd_t *abort_cmd)
1065 1104 {
1066 1105 nvme_t *nvme = abort_cmd->nc_nvme;
1067 1106 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1068 1107 nvme_abort_cmd_t ac = { 0 };
1069 1108
1070 1109 sema_p(&nvme->n_abort_sema);
1071 1110
1072 1111 ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid;
1073 1112 ac.b.ac_sqid = abort_cmd->nc_sqid;
1074 1113
1075 1114 /*
1076 1115 * Drop the mutex of the aborted command. From this point on
1077 1116 * we must assume that the abort callback has freed the command.
1078 1117 */
1079 1118 mutex_exit(&abort_cmd->nc_mutex);
1080 1119
1081 1120 cmd->nc_sqid = 0;
1082 1121 cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT;
1083 1122 cmd->nc_callback = nvme_wakeup_cmd;
1084 1123 cmd->nc_sqe.sqe_cdw10 = ac.r;
1085 1124
1086 1125 /*
1087 1126 * Send the ABORT to the hardware. The ABORT command will return _after_
1088 1127 * the aborted command has completed (aborted or otherwise).
1089 1128 */
1090 1129 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
1091 1130 sema_v(&nvme->n_abort_sema);
1092 1131 dev_err(nvme->n_dip, CE_WARN,
1093 1132 "!nvme_admin_cmd failed for ABORT");
1094 1133 atomic_inc_32(&nvme->n_abort_failed);
1095 1134 return;
1096 1135 }
1097 1136 sema_v(&nvme->n_abort_sema);
1098 1137
1099 1138 if (nvme_check_cmd_status(cmd)) {
1100 1139 dev_err(nvme->n_dip, CE_WARN,
1101 1140 "!ABORT failed with sct = %x, sc = %x",
1102 1141 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1103 1142 atomic_inc_32(&nvme->n_abort_failed);
1104 1143 } else {
1105 1144 atomic_inc_32(&nvme->n_cmd_aborted);
1106 1145 }
1107 1146
1108 1147 nvme_free_cmd(cmd);
1109 1148 }
1110 1149
1111 1150 /*
1112 1151 * nvme_wait_cmd -- wait for command completion or timeout
1113 1152 *
1114 1153 * Returns B_TRUE if the command completed normally.
1115 1154 *
1116 1155 * Returns B_FALSE if the command timed out and an abort was attempted. The
1117 1156 * command mutex will be dropped and the command must be considered freed. The
1118 1157 * freeing of the command is normally done by the abort command callback.
1119 1158 *
1120 1159 * In case of a serious error or a timeout of the abort command the hardware
1121 1160 * will be declared dead and FMA will be notified.
1122 1161 */
1123 1162 static boolean_t
1124 1163 nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec)
1125 1164 {
1126 1165 clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC);
1127 1166 nvme_t *nvme = cmd->nc_nvme;
1128 1167 nvme_reg_csts_t csts;
1129 1168
1130 1169 ASSERT(mutex_owned(&cmd->nc_mutex));
1131 1170
1132 1171 while (!cmd->nc_completed) {
1133 1172 if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1)
1134 1173 break;
1135 1174 }
1136 1175
1137 1176 if (cmd->nc_completed)
1138 1177 return (B_TRUE);
1139 1178
1140 1179 /*
1141 1180 * The command timed out. Change the callback to the cleanup function.
1142 1181 */
1143 1182 cmd->nc_callback = nvme_abort_cmd_cb;
1144 1183
1145 1184 /*
1146 1185 * Check controller for fatal status, any errors associated with the
1147 1186 * register or DMA handle, or for a double timeout (abort command timed
1148 1187 * out). If necessary log a warning and call FMA.
1149 1188 */
1150 1189 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1151 1190 dev_err(nvme->n_dip, CE_WARN, "!command timeout, "
1152 1191 "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs);
1153 1192 atomic_inc_32(&nvme->n_cmd_timeout);
1154 1193
1155 1194 if (csts.b.csts_cfs ||
1156 1195 nvme_check_regs_hdl(nvme) ||
1157 1196 nvme_check_dma_hdl(cmd->nc_dma) ||
1158 1197 cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) {
1159 1198 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
1160 1199 nvme->n_dead = B_TRUE;
1161 1200 mutex_exit(&cmd->nc_mutex);
1162 1201 } else {
1163 1202 /*
1164 1203 * Try to abort the command. The command mutex is released by
1165 1204 * nvme_abort_cmd().
1166 1205 * If the abort succeeds it will have freed the aborted command.
1167 1206 * If the abort fails for other reasons we must assume that the
1168 1207 * command may complete at any time, and the callback will free
1169 1208 * it for us.
1170 1209 */
1171 1210 nvme_abort_cmd(cmd);
1172 1211 }
1173 1212
1174 1213 return (B_FALSE);
1175 1214 }
1176 1215
1177 1216 static void
1178 1217 nvme_wakeup_cmd(void *arg)
1179 1218 {
1180 1219 nvme_cmd_t *cmd = arg;
1181 1220
1182 1221 mutex_enter(&cmd->nc_mutex);
1183 1222 /*
1184 1223 * There is a slight chance that this command completed shortly after
1185 1224 * the timeout was hit in nvme_wait_cmd() but before the callback was
1186 1225 * changed. Catch that case here and clean up accordingly.
1187 1226 */
1188 1227 if (cmd->nc_callback == nvme_abort_cmd_cb) {
1189 1228 mutex_exit(&cmd->nc_mutex);
1190 1229 nvme_abort_cmd_cb(cmd);
1191 1230 return;
1192 1231 }
1193 1232
1194 1233 cmd->nc_completed = B_TRUE;
1195 1234 cv_signal(&cmd->nc_cv);
1196 1235 mutex_exit(&cmd->nc_mutex);
1197 1236 }
1198 1237
1199 1238 static void
1200 1239 nvme_async_event_task(void *arg)
1201 1240 {
1202 1241 nvme_cmd_t *cmd = arg;
1203 1242 nvme_t *nvme = cmd->nc_nvme;
1204 1243 nvme_error_log_entry_t *error_log = NULL;
1205 1244 nvme_health_log_t *health_log = NULL;
1206 1245 nvme_async_event_t event;
1207 1246 int ret;
1208 1247
1209 1248 /*
1210 1249 * Check for errors associated with the async request itself. The only
1211 1250 * command-specific error is "async event limit exceeded", which
1212 1251 * indicates a programming error in the driver and causes a panic in
1213 1252 * nvme_check_cmd_status().
1214 1253 *
1215 1254 * Other possible errors are various scenarios where the async request
1216 1255 * was aborted, or internal errors in the device. Internal errors are
1217 1256 * reported to FMA, the command aborts need no special handling here.
1218 1257 */
1219 1258 if (nvme_check_cmd_status(cmd)) {
1220 1259 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
1221 1260 "!async event request returned failure, sct = %x, "
1222 1261 "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct,
1223 1262 cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr,
1224 1263 cmd->nc_cqe.cqe_sf.sf_m);
1225 1264
1226 1265 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC &&
1227 1266 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) {
1228 1267 cmd->nc_nvme->n_dead = B_TRUE;
1229 1268 ddi_fm_service_impact(cmd->nc_nvme->n_dip,
1230 1269 DDI_SERVICE_LOST);
1231 1270 }
1232 1271 nvme_free_cmd(cmd);
1233 1272 return;
1234 1273 }
1235 1274
1236 1275
1237 1276 event.r = cmd->nc_cqe.cqe_dw0;
1238 1277
1239 1278 /* Clear CQE and re-submit the async request. */
1240 1279 bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t));
1241 1280 ret = nvme_submit_cmd(nvme->n_adminq, cmd);
1242 1281
1243 1282 if (ret != DDI_SUCCESS) {
1244 1283 dev_err(nvme->n_dip, CE_WARN,
1245 1284 "!failed to resubmit async event request");
1246 1285 atomic_inc_32(&nvme->n_async_resubmit_failed);
1247 1286 nvme_free_cmd(cmd);
1248 1287 }
1249 1288
1250 1289 switch (event.b.ae_type) {
1251 1290 case NVME_ASYNC_TYPE_ERROR:
1252 1291 if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) {
1253 1292 error_log = (nvme_error_log_entry_t *)
1254 1293 nvme_get_logpage(nvme, event.b.ae_logpage);
1255 1294 } else {
1256 1295 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in "
1257 1296 "async event reply: %d", event.b.ae_logpage);
1258 1297 atomic_inc_32(&nvme->n_wrong_logpage);
1259 1298 }
1260 1299
1261 1300 switch (event.b.ae_info) {
1262 1301 case NVME_ASYNC_ERROR_INV_SQ:
1263 1302 dev_err(nvme->n_dip, CE_PANIC, "programming error: "
1264 1303 "invalid submission queue");
1265 1304 return;
1266 1305
1267 1306 case NVME_ASYNC_ERROR_INV_DBL:
1268 1307 dev_err(nvme->n_dip, CE_PANIC, "programming error: "
1269 1308 "invalid doorbell write value");
1270 1309 return;
1271 1310
1272 1311 case NVME_ASYNC_ERROR_DIAGFAIL:
1273 1312 dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure");
1274 1313 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
1275 1314 nvme->n_dead = B_TRUE;
1276 1315 atomic_inc_32(&nvme->n_diagfail_event);
1277 1316 break;
1278 1317
1279 1318 case NVME_ASYNC_ERROR_PERSISTENT:
1280 1319 dev_err(nvme->n_dip, CE_WARN, "!persistent internal "
1281 1320 "device error");
1282 1321 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
1283 1322 nvme->n_dead = B_TRUE;
1284 1323 atomic_inc_32(&nvme->n_persistent_event);
1285 1324 break;
1286 1325
1287 1326 case NVME_ASYNC_ERROR_TRANSIENT:
1288 1327 dev_err(nvme->n_dip, CE_WARN, "!transient internal "
1289 1328 "device error");
1290 1329 /* TODO: send ereport */
1291 1330 atomic_inc_32(&nvme->n_transient_event);
1292 1331 break;
1293 1332
1294 1333 case NVME_ASYNC_ERROR_FW_LOAD:
1295 1334 dev_err(nvme->n_dip, CE_WARN,
1296 1335 "!firmware image load error");
1297 1336 atomic_inc_32(&nvme->n_fw_load_event);
1298 1337 break;
1299 1338 }
1300 1339 break;
1301 1340
1302 1341 case NVME_ASYNC_TYPE_HEALTH:
1303 1342 if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) {
1304 1343 health_log = (nvme_health_log_t *)
1305 1344 nvme_get_logpage(nvme, event.b.ae_logpage, -1);
1306 1345 } else {
1307 1346 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in "
1308 1347 "async event reply: %d", event.b.ae_logpage);
1309 1348 atomic_inc_32(&nvme->n_wrong_logpage);
1310 1349 }
1311 1350
1312 1351 switch (event.b.ae_info) {
1313 1352 case NVME_ASYNC_HEALTH_RELIABILITY:
1314 1353 dev_err(nvme->n_dip, CE_WARN,
1315 1354 "!device reliability compromised");
1316 1355 /* TODO: send ereport */
1317 1356 atomic_inc_32(&nvme->n_reliability_event);
1318 1357 break;
1319 1358
1320 1359 case NVME_ASYNC_HEALTH_TEMPERATURE:
1321 1360 dev_err(nvme->n_dip, CE_WARN,
1322 1361 "!temperature above threshold");
1323 1362 /* TODO: send ereport */
1324 1363 atomic_inc_32(&nvme->n_temperature_event);
1325 1364 break;
1326 1365
1327 1366 case NVME_ASYNC_HEALTH_SPARE:
1328 1367 dev_err(nvme->n_dip, CE_WARN,
1329 1368 "!spare space below threshold");
1330 1369 /* TODO: send ereport */
1331 1370 atomic_inc_32(&nvme->n_spare_event);
1332 1371 break;
1333 1372 }
1334 1373 break;
1335 1374
1336 1375 case NVME_ASYNC_TYPE_VENDOR:
1337 1376 dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event "
1338 1377 "received, info = %x, logpage = %x", event.b.ae_info,
1339 1378 event.b.ae_logpage);
1340 1379 atomic_inc_32(&nvme->n_vendor_event);
1341 1380 break;
1342 1381
1343 1382 default:
1344 1383 dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, "
1345 1384 "type = %x, info = %x, logpage = %x", event.b.ae_type,
1346 1385 event.b.ae_info, event.b.ae_logpage);
1347 1386 atomic_inc_32(&nvme->n_unknown_event);
1348 1387 break;
1349 1388 }
1350 1389
1351 1390 if (error_log)
1352 1391 kmem_free(error_log, sizeof (nvme_error_log_entry_t) *
1353 1392 nvme->n_error_log_len);
1354 1393
1355 1394 if (health_log)
1356 1395 kmem_free(health_log, sizeof (nvme_health_log_t));
1357 1396 }
1358 1397
1359 1398 static int
1360 1399 nvme_admin_cmd(nvme_cmd_t *cmd, int sec)
1361 1400 {
1362 1401 int ret;
1363 1402
1364 1403 mutex_enter(&cmd->nc_mutex);
1365 1404 ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd);
1366 1405
1367 1406 if (ret != DDI_SUCCESS) {
1368 1407 mutex_exit(&cmd->nc_mutex);
1369 1408 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
1370 1409 "!nvme_submit_cmd failed");
1371 1410 atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full);
1372 1411 nvme_free_cmd(cmd);
1373 1412 return (DDI_FAILURE);
1374 1413 }
1375 1414
1376 1415 if (nvme_wait_cmd(cmd, sec) == B_FALSE) {
1377 1416 /*
1378 1417 * The command timed out. An abort command was posted that
1379 1418 * will take care of the cleanup.
1380 1419 */
1381 1420 return (DDI_FAILURE);
1382 1421 }
1383 1422 mutex_exit(&cmd->nc_mutex);
1384 1423
1385 1424 return (DDI_SUCCESS);
1386 1425 }
1387 1426
1388 1427 static int
1389 1428 nvme_async_event(nvme_t *nvme)
1390 1429 {
1391 1430 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1392 1431 int ret;
1393 1432
1394 1433 cmd->nc_sqid = 0;
1395 1434 cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT;
1396 1435 cmd->nc_callback = nvme_async_event_task;
1397 1436
1398 1437 ret = nvme_submit_cmd(nvme->n_adminq, cmd);
1399 1438
1400 1439 if (ret != DDI_SUCCESS) {
1401 1440 dev_err(nvme->n_dip, CE_WARN,
1402 1441 "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT");
1403 1442 nvme_free_cmd(cmd);
1404 1443 return (DDI_FAILURE);
1405 1444 }
1406 1445
1407 1446 return (DDI_SUCCESS);
1408 1447 }
1409 1448
1410 1449 static void *
1411 1450 nvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...)
1412 1451 {
1413 1452 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1414 1453 void *buf = NULL;
1415 1454 nvme_getlogpage_t getlogpage = { 0 };
1416 1455 size_t bufsize;
1417 1456 va_list ap;
1418 1457
1419 1458 va_start(ap, logpage);
1420 1459
1421 1460 cmd->nc_sqid = 0;
1422 1461 cmd->nc_callback = nvme_wakeup_cmd;
1423 1462 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE;
1424 1463
1425 1464 getlogpage.b.lp_lid = logpage;
1426 1465
1427 1466 switch (logpage) {
1428 1467 case NVME_LOGPAGE_ERROR:
1429 1468 cmd->nc_sqe.sqe_nsid = (uint32_t)-1;
1430 1469 bufsize = nvme->n_error_log_len *
1431 1470 sizeof (nvme_error_log_entry_t);
1432 1471 break;
1433 1472
1434 1473 case NVME_LOGPAGE_HEALTH:
1435 1474 cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t);
1436 1475 bufsize = sizeof (nvme_health_log_t);
1437 1476 break;
1438 1477
1439 1478 case NVME_LOGPAGE_FWSLOT:
1440 1479 cmd->nc_sqe.sqe_nsid = (uint32_t)-1;
1441 1480 bufsize = sizeof (nvme_fwslot_log_t);
1442 1481 break;
1443 1482
1444 1483 default:
1445 1484 dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d",
1446 1485 logpage);
1447 1486 atomic_inc_32(&nvme->n_unknown_logpage);
1448 1487 goto fail;
1449 1488 }
1450 1489
1451 1490 va_end(ap);
1452 1491
1453 1492 getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1;
1454 1493
1455 1494 cmd->nc_sqe.sqe_cdw10 = getlogpage.r;
1456 1495
1457 1496 if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t),
1458 1497 DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
1459 1498 dev_err(nvme->n_dip, CE_WARN,
1460 1499 "!nvme_zalloc_dma failed for GET LOG PAGE");
1461 1500 goto fail;
1462 1501 }
1463 1502
1464 1503 if (cmd->nc_dma->nd_ncookie > 2) {
1465 1504 dev_err(nvme->n_dip, CE_WARN,
1466 1505 "!too many DMA cookies for GET LOG PAGE");
1467 1506 atomic_inc_32(&nvme->n_too_many_cookies);
1468 1507 goto fail;
1469 1508 }
1470 1509
1471 1510 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress;
1472 1511 if (cmd->nc_dma->nd_ncookie > 1) {
1473 1512 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah,
1474 1513 &cmd->nc_dma->nd_cookie);
1475 1514 cmd->nc_sqe.sqe_dptr.d_prp[1] =
1476 1515 cmd->nc_dma->nd_cookie.dmac_laddress;
1477 1516 }
1478 1517
1479 1518 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
1480 1519 dev_err(nvme->n_dip, CE_WARN,
1481 1520 "!nvme_admin_cmd failed for GET LOG PAGE");
1482 1521 return (NULL);
1483 1522 }
1484 1523
1485 1524 if (nvme_check_cmd_status(cmd)) {
1486 1525 dev_err(nvme->n_dip, CE_WARN,
1487 1526 "!GET LOG PAGE failed with sct = %x, sc = %x",
1488 1527 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1489 1528 goto fail;
1490 1529 }
1491 1530
1492 1531 buf = kmem_alloc(bufsize, KM_SLEEP);
1493 1532 bcopy(cmd->nc_dma->nd_memp, buf, bufsize);
1494 1533
1495 1534 fail:
1496 1535 nvme_free_cmd(cmd);
1497 1536
1498 1537 return (buf);
1499 1538 }
1500 1539
1501 1540 static void *
1502 1541 nvme_identify(nvme_t *nvme, uint32_t nsid)
1503 1542 {
1504 1543 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1505 1544 void *buf = NULL;
1506 1545
1507 1546 cmd->nc_sqid = 0;
1508 1547 cmd->nc_callback = nvme_wakeup_cmd;
1509 1548 cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY;
1510 1549 cmd->nc_sqe.sqe_nsid = nsid;
1511 1550 cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL;
1512 1551
1513 1552 if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ,
1514 1553 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
1515 1554 dev_err(nvme->n_dip, CE_WARN,
1516 1555 "!nvme_zalloc_dma failed for IDENTIFY");
1517 1556 goto fail;
1518 1557 }
1519 1558
1520 1559 if (cmd->nc_dma->nd_ncookie > 2) {
1521 1560 dev_err(nvme->n_dip, CE_WARN,
1522 1561 "!too many DMA cookies for IDENTIFY");
1523 1562 atomic_inc_32(&nvme->n_too_many_cookies);
1524 1563 goto fail;
1525 1564 }
1526 1565
1527 1566 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress;
1528 1567 if (cmd->nc_dma->nd_ncookie > 1) {
1529 1568 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah,
1530 1569 &cmd->nc_dma->nd_cookie);
1531 1570 cmd->nc_sqe.sqe_dptr.d_prp[1] =
1532 1571 cmd->nc_dma->nd_cookie.dmac_laddress;
1533 1572 }
1534 1573
1535 1574 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
1536 1575 dev_err(nvme->n_dip, CE_WARN,
1537 1576 "!nvme_admin_cmd failed for IDENTIFY");
1538 1577 return (NULL);
1539 1578 }
1540 1579
1541 1580 if (nvme_check_cmd_status(cmd)) {
1542 1581 dev_err(nvme->n_dip, CE_WARN,
1543 1582 "!IDENTIFY failed with sct = %x, sc = %x",
1544 1583 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1545 1584 goto fail;
1546 1585 }
1547 1586
1548 1587 buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP);
1549 1588 bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE);
1550 1589
1551 1590 fail:
1552 1591 nvme_free_cmd(cmd);
1553 1592
1554 1593 return (buf);
1555 1594 }
1556 1595
1557 1596 static int
1558 1597 nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues)
1559 1598 {
1560 1599 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1561 1600 nvme_nqueue_t nq = { 0 };
1562 1601
1563 1602 nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1;
1564 1603
1565 1604 cmd->nc_sqid = 0;
1566 1605 cmd->nc_callback = nvme_wakeup_cmd;
1567 1606 cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES;
1568 1607 cmd->nc_sqe.sqe_cdw10 = NVME_FEAT_NQUEUES;
1569 1608 cmd->nc_sqe.sqe_cdw11 = nq.r;
1570 1609
1571 1610 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
1572 1611 dev_err(nvme->n_dip, CE_WARN,
1573 1612 "!nvme_admin_cmd failed for SET FEATURES (NQUEUES)");
1574 1613 return (0);
1575 1614 }
1576 1615
1577 1616 if (nvme_check_cmd_status(cmd)) {
1578 1617 dev_err(nvme->n_dip, CE_WARN,
1579 1618 "!SET FEATURES (NQUEUES) failed with sct = %x, sc = %x",
1580 1619 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1581 1620 nvme_free_cmd(cmd);
1582 1621 return (0);
1583 1622 }
1584 1623
1585 1624 nq.r = cmd->nc_cqe.cqe_dw0;
1586 1625 nvme_free_cmd(cmd);
1587 1626
1588 1627 /*
1589 1628 * Always use the same number of submission and completion queues, and
1590 1629 * never use more than the requested number of queues.
1591 1630 */
1592 1631 return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1));
1593 1632 }
1594 1633
1595 1634 static int
1596 1635 nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx)
1597 1636 {
1598 1637 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1599 1638 nvme_create_queue_dw10_t dw10 = { 0 };
1600 1639 nvme_create_cq_dw11_t c_dw11 = { 0 };
1601 1640 nvme_create_sq_dw11_t s_dw11 = { 0 };
1602 1641
1603 1642 dw10.b.q_qid = idx;
1604 1643 dw10.b.q_qsize = qp->nq_nentry - 1;
1605 1644
1606 1645 c_dw11.b.cq_pc = 1;
1607 1646 c_dw11.b.cq_ien = 1;
1608 1647 c_dw11.b.cq_iv = idx % nvme->n_intr_cnt;
1609 1648
1610 1649 cmd->nc_sqid = 0;
1611 1650 cmd->nc_callback = nvme_wakeup_cmd;
1612 1651 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE;
1613 1652 cmd->nc_sqe.sqe_cdw10 = dw10.r;
1614 1653 cmd->nc_sqe.sqe_cdw11 = c_dw11.r;
1615 1654 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress;
1616 1655
1617 1656 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
1618 1657 dev_err(nvme->n_dip, CE_WARN,
1619 1658 "!nvme_admin_cmd failed for CREATE CQUEUE");
1620 1659 return (DDI_FAILURE);
1621 1660 }
1622 1661
1623 1662 if (nvme_check_cmd_status(cmd)) {
1624 1663 dev_err(nvme->n_dip, CE_WARN,
1625 1664 "!CREATE CQUEUE failed with sct = %x, sc = %x",
1626 1665 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1627 1666 nvme_free_cmd(cmd);
1628 1667 return (DDI_FAILURE);
1629 1668 }
1630 1669
1631 1670 nvme_free_cmd(cmd);
1632 1671
1633 1672 s_dw11.b.sq_pc = 1;
1634 1673 s_dw11.b.sq_cqid = idx;
1635 1674
1636 1675 cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1637 1676 cmd->nc_sqid = 0;
1638 1677 cmd->nc_callback = nvme_wakeup_cmd;
1639 1678 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE;
1640 1679 cmd->nc_sqe.sqe_cdw10 = dw10.r;
1641 1680 cmd->nc_sqe.sqe_cdw11 = s_dw11.r;
1642 1681 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress;
1643 1682
1644 1683 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
1645 1684 dev_err(nvme->n_dip, CE_WARN,
1646 1685 "!nvme_admin_cmd failed for CREATE SQUEUE");
1647 1686 return (DDI_FAILURE);
1648 1687 }
1649 1688
|
↓ open down ↓ |
884 lines elided |
↑ open up ↑ |
1650 1689 if (nvme_check_cmd_status(cmd)) {
1651 1690 dev_err(nvme->n_dip, CE_WARN,
1652 1691 "!CREATE SQUEUE failed with sct = %x, sc = %x",
1653 1692 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1654 1693 nvme_free_cmd(cmd);
1655 1694 return (DDI_FAILURE);
1656 1695 }
1657 1696
1658 1697 nvme_free_cmd(cmd);
1659 1698
1699 + mutex_enter(&nvme_global_mutex);
1700 + list_insert_head(&nvme_qp_list, qp);
1701 + qp->nq_nvme = nvme;
1702 + mutex_exit(&nvme_global_mutex);
1703 +
1660 1704 return (DDI_SUCCESS);
1661 1705 }
1662 1706
1663 1707 static boolean_t
1664 1708 nvme_reset(nvme_t *nvme, boolean_t quiesce)
1665 1709 {
1666 1710 nvme_reg_csts_t csts;
1667 1711 int i;
1668 1712
1669 1713 nvme_put32(nvme, NVME_REG_CC, 0);
1670 1714
1671 1715 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1672 1716 if (csts.b.csts_rdy == 1) {
1673 1717 nvme_put32(nvme, NVME_REG_CC, 0);
1674 1718 for (i = 0; i != nvme->n_timeout * 10; i++) {
1675 1719 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1676 1720 if (csts.b.csts_rdy == 0)
1677 1721 break;
1678 1722
1679 1723 if (quiesce)
1680 1724 drv_usecwait(50000);
1681 1725 else
1682 1726 delay(drv_usectohz(50000));
1683 1727 }
1684 1728 }
1685 1729
1686 1730 nvme_put32(nvme, NVME_REG_AQA, 0);
1687 1731 nvme_put32(nvme, NVME_REG_ASQ, 0);
1688 1732 nvme_put32(nvme, NVME_REG_ACQ, 0);
1689 1733
1690 1734 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1691 1735 return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE);
1692 1736 }
1693 1737
1694 1738 static void
1695 1739 nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce)
1696 1740 {
1697 1741 nvme_reg_cc_t cc;
1698 1742 nvme_reg_csts_t csts;
1699 1743 int i;
1700 1744
1701 1745 ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT);
1702 1746
1703 1747 cc.r = nvme_get32(nvme, NVME_REG_CC);
1704 1748 cc.b.cc_shn = mode & 0x3;
1705 1749 nvme_put32(nvme, NVME_REG_CC, cc.r);
1706 1750
1707 1751 for (i = 0; i != 10; i++) {
1708 1752 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1709 1753 if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE)
1710 1754 break;
1711 1755
1712 1756 if (quiesce)
1713 1757 drv_usecwait(100000);
1714 1758 else
1715 1759 delay(drv_usectohz(100000));
1716 1760 }
1717 1761 }
1718 1762
1719 1763
1720 1764 static void
1721 1765 nvme_prepare_devid(nvme_t *nvme, uint32_t nsid)
1722 1766 {
1723 1767 char model[sizeof (nvme->n_idctl->id_model) + 1];
1724 1768 char serial[sizeof (nvme->n_idctl->id_serial) + 1];
1725 1769
1726 1770 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model));
1727 1771 bcopy(nvme->n_idctl->id_serial, serial,
1728 1772 sizeof (nvme->n_idctl->id_serial));
1729 1773
1730 1774 model[sizeof (nvme->n_idctl->id_model)] = '\0';
1731 1775 serial[sizeof (nvme->n_idctl->id_serial)] = '\0';
1732 1776
1733 1777 (void) snprintf(nvme->n_ns[nsid - 1].ns_devid,
1734 1778 sizeof (nvme->n_ns[0].ns_devid), "%4X-%s-%s-%X",
1735 1779 nvme->n_idctl->id_vid, model, serial, nsid);
1736 1780 }
1737 1781
1738 1782 static int
1739 1783 nvme_init(nvme_t *nvme)
1740 1784 {
1741 1785 nvme_reg_cc_t cc = { 0 };
1742 1786 nvme_reg_aqa_t aqa = { 0 };
1743 1787 nvme_reg_asq_t asq = { 0 };
1744 1788 nvme_reg_acq_t acq = { 0 };
1745 1789 nvme_reg_cap_t cap;
1746 1790 nvme_reg_vs_t vs;
1747 1791 nvme_reg_csts_t csts;
1748 1792 int i = 0;
1749 1793 int nqueues;
1750 1794 char model[sizeof (nvme->n_idctl->id_model) + 1];
1751 1795 char *vendor, *product;
1752 1796
1753 1797 /* Check controller version */
1754 1798 vs.r = nvme_get32(nvme, NVME_REG_VS);
1755 1799 dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d",
1756 1800 vs.b.vs_mjr, vs.b.vs_mnr);
1757 1801
1758 1802 if (nvme_version_major < vs.b.vs_mjr ||
1759 1803 (nvme_version_major == vs.b.vs_mjr &&
1760 1804 nvme_version_minor < vs.b.vs_mnr)) {
1761 1805 dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d",
1762 1806 nvme_version_major, nvme_version_minor);
1763 1807 if (nvme->n_strict_version)
1764 1808 goto fail;
1765 1809 }
1766 1810
1767 1811 /* retrieve controller configuration */
1768 1812 cap.r = nvme_get64(nvme, NVME_REG_CAP);
1769 1813
1770 1814 if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) {
1771 1815 dev_err(nvme->n_dip, CE_WARN,
1772 1816 "!NVM command set not supported by hardware");
1773 1817 goto fail;
1774 1818 }
1775 1819
1776 1820 nvme->n_nssr_supported = cap.b.cap_nssrs;
1777 1821 nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd;
1778 1822 nvme->n_timeout = cap.b.cap_to;
1779 1823 nvme->n_arbitration_mechanisms = cap.b.cap_ams;
1780 1824 nvme->n_cont_queues_reqd = cap.b.cap_cqr;
1781 1825 nvme->n_max_queue_entries = cap.b.cap_mqes + 1;
1782 1826
1783 1827 /*
1784 1828 * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify
1785 1829 * the base page size of 4k (1<<12), so add 12 here to get the real
1786 1830 * page size value.
1787 1831 */
1788 1832 nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT),
1789 1833 cap.b.cap_mpsmax + 12);
1790 1834 nvme->n_pagesize = 1UL << (nvme->n_pageshift);
1791 1835
1792 1836 /*
1793 1837 * Set up Queue DMA to transfer at least 1 page-aligned page at a time.
1794 1838 */
1795 1839 nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize;
1796 1840 nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize;
1797 1841
1798 1842 /*
1799 1843 * Set up PRP DMA to transfer 1 page-aligned page at a time.
1800 1844 * Maxxfer may be increased after we identified the controller limits.
1801 1845 */
1802 1846 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize;
1803 1847 nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize;
1804 1848 nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize;
1805 1849 nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1;
1806 1850
1807 1851 /*
1808 1852 * Reset controller if it's still in ready state.
1809 1853 */
1810 1854 if (nvme_reset(nvme, B_FALSE) == B_FALSE) {
1811 1855 dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller");
1812 1856 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
1813 1857 nvme->n_dead = B_TRUE;
1814 1858 goto fail;
1815 1859 }
1816 1860
1817 1861 /*
1818 1862 * Create the admin queue pair.
1819 1863 */
1820 1864 if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0)
1821 1865 != DDI_SUCCESS) {
1822 1866 dev_err(nvme->n_dip, CE_WARN,
1823 1867 "!unable to allocate admin qpair");
1824 1868 goto fail;
1825 1869 }
1826 1870 nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP);
1827 1871 nvme->n_ioq[0] = nvme->n_adminq;
1828 1872
1829 1873 nvme->n_progress |= NVME_ADMIN_QUEUE;
1830 1874
1831 1875 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
1832 1876 "admin-queue-len", nvme->n_admin_queue_len);
1833 1877
1834 1878 aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1;
1835 1879 asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress;
1836 1880 acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress;
1837 1881
1838 1882 ASSERT((asq & (nvme->n_pagesize - 1)) == 0);
1839 1883 ASSERT((acq & (nvme->n_pagesize - 1)) == 0);
1840 1884
1841 1885 nvme_put32(nvme, NVME_REG_AQA, aqa.r);
1842 1886 nvme_put64(nvme, NVME_REG_ASQ, asq);
1843 1887 nvme_put64(nvme, NVME_REG_ACQ, acq);
1844 1888
1845 1889 cc.b.cc_ams = 0; /* use Round-Robin arbitration */
1846 1890 cc.b.cc_css = 0; /* use NVM command set */
1847 1891 cc.b.cc_mps = nvme->n_pageshift - 12;
1848 1892 cc.b.cc_shn = 0; /* no shutdown in progress */
1849 1893 cc.b.cc_en = 1; /* enable controller */
1850 1894 cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */
1851 1895 cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */
1852 1896
1853 1897 nvme_put32(nvme, NVME_REG_CC, cc.r);
1854 1898
1855 1899 /*
1856 1900 * Wait for the controller to become ready.
1857 1901 */
1858 1902 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1859 1903 if (csts.b.csts_rdy == 0) {
1860 1904 for (i = 0; i != nvme->n_timeout * 10; i++) {
1861 1905 delay(drv_usectohz(50000));
1862 1906 csts.r = nvme_get32(nvme, NVME_REG_CSTS);
1863 1907
1864 1908 if (csts.b.csts_cfs == 1) {
1865 1909 dev_err(nvme->n_dip, CE_WARN,
1866 1910 "!controller fatal status at init");
1867 1911 ddi_fm_service_impact(nvme->n_dip,
1868 1912 DDI_SERVICE_LOST);
1869 1913 nvme->n_dead = B_TRUE;
1870 1914 goto fail;
1871 1915 }
1872 1916
1873 1917 if (csts.b.csts_rdy == 1)
1874 1918 break;
1875 1919 }
1876 1920 }
1877 1921
1878 1922 if (csts.b.csts_rdy == 0) {
1879 1923 dev_err(nvme->n_dip, CE_WARN, "!controller not ready");
1880 1924 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
1881 1925 nvme->n_dead = B_TRUE;
1882 1926 goto fail;
1883 1927 }
1884 1928
1885 1929 /*
1886 1930 * Assume an abort command limit of 1. We'll destroy and re-init
1887 1931 * that later when we know the true abort command limit.
1888 1932 */
1889 1933 sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL);
1890 1934
1891 1935 /*
1892 1936 * Setup initial interrupt for admin queue.
1893 1937 */
1894 1938 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1)
1895 1939 != DDI_SUCCESS) &&
1896 1940 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1)
1897 1941 != DDI_SUCCESS) &&
1898 1942 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1)
1899 1943 != DDI_SUCCESS)) {
1900 1944 dev_err(nvme->n_dip, CE_WARN,
1901 1945 "!failed to setup initial interrupt");
1902 1946 goto fail;
1903 1947 }
1904 1948
1905 1949 /*
1906 1950 * Post an asynchronous event command to catch errors.
1907 1951 */
1908 1952 if (nvme_async_event(nvme) != DDI_SUCCESS) {
1909 1953 dev_err(nvme->n_dip, CE_WARN,
1910 1954 "!failed to post async event");
1911 1955 goto fail;
1912 1956 }
1913 1957
1914 1958 /*
1915 1959 * Identify Controller
1916 1960 */
1917 1961 nvme->n_idctl = nvme_identify(nvme, 0);
1918 1962 if (nvme->n_idctl == NULL) {
1919 1963 dev_err(nvme->n_dip, CE_WARN,
1920 1964 "!failed to identify controller");
1921 1965 goto fail;
1922 1966 }
1923 1967
1924 1968 /*
1925 1969 * Get Vendor & Product ID
1926 1970 */
1927 1971 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model));
1928 1972 model[sizeof (nvme->n_idctl->id_model)] = '\0';
1929 1973 sata_split_model(model, &vendor, &product);
1930 1974
1931 1975 if (vendor == NULL)
1932 1976 nvme->n_vendor = strdup("NVMe");
1933 1977 else
1934 1978 nvme->n_vendor = strdup(vendor);
1935 1979
1936 1980 nvme->n_product = strdup(product);
1937 1981
1938 1982 /*
1939 1983 * Get controller limits.
1940 1984 */
1941 1985 nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT,
1942 1986 MIN(nvme->n_admin_queue_len / 10,
1943 1987 MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit)));
1944 1988
1945 1989 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
1946 1990 "async-event-limit", nvme->n_async_event_limit);
1947 1991
1948 1992 nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1;
1949 1993
1950 1994 /*
1951 1995 * Reinitialize the semaphore with the true abort command limit
1952 1996 * supported by the hardware. It's not necessary to disable interrupts
1953 1997 * as only command aborts use the semaphore, and no commands are
1954 1998 * executed or aborted while we're here.
1955 1999 */
1956 2000 sema_destroy(&nvme->n_abort_sema);
1957 2001 sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL,
1958 2002 SEMA_DRIVER, NULL);
1959 2003
1960 2004 nvme->n_progress |= NVME_CTRL_LIMITS;
1961 2005
1962 2006 if (nvme->n_idctl->id_mdts == 0)
1963 2007 nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536;
1964 2008 else
1965 2009 nvme->n_max_data_transfer_size =
1966 2010 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts);
1967 2011
1968 2012 nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1;
1969 2013
1970 2014 /*
1971 2015 * Limit n_max_data_transfer_size to what we can handle in one PRP.
1972 2016 * Chained PRPs are currently unsupported.
1973 2017 *
1974 2018 * This is a no-op on hardware which doesn't support a transfer size
1975 2019 * big enough to require chained PRPs.
1976 2020 */
1977 2021 nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size,
1978 2022 (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize));
1979 2023
1980 2024 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size;
1981 2025
1982 2026 /*
1983 2027 * Make sure the minimum/maximum queue entry sizes are not
1984 2028 * larger/smaller than the default.
1985 2029 */
1986 2030
1987 2031 if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) ||
1988 2032 ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) ||
1989 2033 ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) ||
1990 2034 ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t)))
1991 2035 goto fail;
1992 2036
1993 2037 /*
1994 2038 * Check for the presence of a Volatile Write Cache. If present,
1995 2039 * enable it by default.
1996 2040 */
1997 2041 if (nvme->n_idctl->id_vwc.vwc_present == 0) {
1998 2042 nvme->n_volatile_write_cache_enabled = B_FALSE;
1999 2043 nvme_bd_ops.o_sync_cache = NULL;
2000 2044 } else {
2001 2045 /*
2002 2046 * TODO: send SET FEATURES to enable VWC
2003 2047 * (have no hardware to test this)
2004 2048 */
2005 2049 nvme->n_volatile_write_cache_enabled = B_FALSE;
2006 2050 nvme_bd_ops.o_sync_cache = NULL;
2007 2051 }
2008 2052
2009 2053 /*
2010 2054 * Grab a copy of all mandatory log pages.
2011 2055 *
2012 2056 * TODO: should go away once user space tool exists to print logs
2013 2057 */
2014 2058 nvme->n_error_log = (nvme_error_log_entry_t *)
2015 2059 nvme_get_logpage(nvme, NVME_LOGPAGE_ERROR);
2016 2060 nvme->n_health_log = (nvme_health_log_t *)
2017 2061 nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1);
2018 2062 nvme->n_fwslot_log = (nvme_fwslot_log_t *)
2019 2063 nvme_get_logpage(nvme, NVME_LOGPAGE_FWSLOT);
2020 2064
2021 2065 /*
2022 2066 * Identify Namespaces
2023 2067 */
2024 2068 nvme->n_namespace_count = nvme->n_idctl->id_nn;
2025 2069 nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) *
2026 2070 nvme->n_namespace_count, KM_SLEEP);
2027 2071
2028 2072 for (i = 0; i != nvme->n_namespace_count; i++) {
2029 2073 nvme_identify_nsid_t *idns;
2030 2074 int last_rp;
2031 2075
2032 2076 nvme->n_ns[i].ns_nvme = nvme;
2033 2077 nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1);
2034 2078
2035 2079 if (idns == NULL) {
2036 2080 dev_err(nvme->n_dip, CE_WARN,
2037 2081 "!failed to identify namespace %d", i + 1);
2038 2082 goto fail;
2039 2083 }
2040 2084
2041 2085 nvme->n_ns[i].ns_id = i + 1;
2042 2086 nvme->n_ns[i].ns_block_count = idns->id_nsize;
2043 2087 nvme->n_ns[i].ns_block_size =
2044 2088 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads;
2045 2089 nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size;
2046 2090
2047 2091 nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id);
2048 2092
2049 2093 /*
2050 2094 * Find the LBA format with no metadata and the best relative
2051 2095 * performance. A value of 3 means "degraded", 0 is best.
2052 2096 */
2053 2097 last_rp = 3;
2054 2098 for (int j = 0; j <= idns->id_nlbaf; j++) {
2055 2099 if (idns->id_lbaf[j].lbaf_lbads == 0)
2056 2100 break;
2057 2101 if (idns->id_lbaf[j].lbaf_ms != 0)
2058 2102 continue;
2059 2103 if (idns->id_lbaf[j].lbaf_rp >= last_rp)
2060 2104 continue;
2061 2105 last_rp = idns->id_lbaf[j].lbaf_rp;
2062 2106 nvme->n_ns[i].ns_best_block_size =
2063 2107 1 << idns->id_lbaf[j].lbaf_lbads;
2064 2108 }
2065 2109
2066 2110 /*
2067 2111 * We currently don't support namespaces that use either:
2068 2112 * - thin provisioning
2069 2113 * - protection information
2070 2114 */
2071 2115 if (idns->id_nsfeat.f_thin ||
2072 2116 idns->id_dps.dp_pinfo) {
2073 2117 dev_err(nvme->n_dip, CE_WARN,
2074 2118 "!ignoring namespace %d, unsupported features: "
2075 2119 "thin = %d, pinfo = %d", i + 1,
2076 2120 idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo);
2077 2121 nvme->n_ns[i].ns_ignore = B_TRUE;
2078 2122 }
2079 2123 }
2080 2124
2081 2125 /*
2082 2126 * Try to set up MSI/MSI-X interrupts.
2083 2127 */
2084 2128 if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX))
2085 2129 != 0) {
2086 2130 nvme_release_interrupts(nvme);
2087 2131
2088 2132 nqueues = MIN(UINT16_MAX, ncpus);
2089 2133
2090 2134 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX,
2091 2135 nqueues) != DDI_SUCCESS) &&
2092 2136 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI,
2093 2137 nqueues) != DDI_SUCCESS)) {
2094 2138 dev_err(nvme->n_dip, CE_WARN,
2095 2139 "!failed to setup MSI/MSI-X interrupts");
2096 2140 goto fail;
2097 2141 }
2098 2142 }
2099 2143
2100 2144 nqueues = nvme->n_intr_cnt;
2101 2145
2102 2146 /*
2103 2147 * Create I/O queue pairs.
2104 2148 */
2105 2149 nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues);
2106 2150 if (nvme->n_ioq_count == 0) {
2107 2151 dev_err(nvme->n_dip, CE_WARN,
2108 2152 "!failed to set number of I/O queues to %d", nqueues);
2109 2153 goto fail;
2110 2154 }
2111 2155
2112 2156 /*
2113 2157 * Reallocate I/O queue array
2114 2158 */
2115 2159 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *));
2116 2160 nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) *
2117 2161 (nvme->n_ioq_count + 1), KM_SLEEP);
2118 2162 nvme->n_ioq[0] = nvme->n_adminq;
2119 2163
2120 2164 /*
2121 2165 * If we got less queues than we asked for we might as well give
2122 2166 * some of the interrupt vectors back to the system.
2123 2167 */
2124 2168 if (nvme->n_ioq_count < nqueues) {
2125 2169 nvme_release_interrupts(nvme);
2126 2170
2127 2171 if (nvme_setup_interrupts(nvme, nvme->n_intr_type,
2128 2172 nvme->n_ioq_count) != DDI_SUCCESS) {
2129 2173 dev_err(nvme->n_dip, CE_WARN,
2130 2174 "!failed to reduce number of interrupts");
2131 2175 goto fail;
2132 2176 }
2133 2177 }
2134 2178
2135 2179 /*
2136 2180 * Alloc & register I/O queue pairs
2137 2181 */
2138 2182 nvme->n_io_queue_len =
2139 2183 MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries);
2140 2184 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len",
2141 2185 nvme->n_io_queue_len);
2142 2186
2143 2187 for (i = 1; i != nvme->n_ioq_count + 1; i++) {
2144 2188 if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len,
2145 2189 &nvme->n_ioq[i], i) != DDI_SUCCESS) {
2146 2190 dev_err(nvme->n_dip, CE_WARN,
2147 2191 "!unable to allocate I/O qpair %d", i);
2148 2192 goto fail;
2149 2193 }
2150 2194
2151 2195 if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i)
2152 2196 != DDI_SUCCESS) {
2153 2197 dev_err(nvme->n_dip, CE_WARN,
2154 2198 "!unable to create I/O qpair %d", i);
2155 2199 goto fail;
2156 2200 }
2157 2201 }
2158 2202
2159 2203 /*
2160 2204 * Post more asynchronous events commands to reduce event reporting
2161 2205 * latency as suggested by the spec.
2162 2206 */
2163 2207 for (i = 1; i != nvme->n_async_event_limit; i++) {
2164 2208 if (nvme_async_event(nvme) != DDI_SUCCESS) {
2165 2209 dev_err(nvme->n_dip, CE_WARN,
2166 2210 "!failed to post async event %d", i);
2167 2211 goto fail;
2168 2212 }
2169 2213 }
2170 2214
2171 2215 return (DDI_SUCCESS);
2172 2216
2173 2217 fail:
2174 2218 (void) nvme_reset(nvme, B_FALSE);
2175 2219 return (DDI_FAILURE);
2176 2220 }
|
↓ open down ↓ |
507 lines elided |
↑ open up ↑ |
2177 2221
2178 2222 static uint_t
2179 2223 nvme_intr(caddr_t arg1, caddr_t arg2)
2180 2224 {
2181 2225 /*LINTED: E_PTR_BAD_CAST_ALIGN*/
2182 2226 nvme_t *nvme = (nvme_t *)arg1;
2183 2227 int inum = (int)(uintptr_t)arg2;
2184 2228 int ccnt = 0;
2185 2229 int qnum;
2186 2230 nvme_cmd_t *cmd;
2231 + int cnt_cmds;
2187 2232
2188 2233 if (inum >= nvme->n_intr_cnt)
2189 2234 return (DDI_INTR_UNCLAIMED);
2190 2235
2191 2236 /*
2192 2237 * The interrupt vector a queue uses is calculated as queue_idx %
2193 2238 * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array
2194 2239 * in steps of n_intr_cnt to process all queues using this vector.
2195 2240 */
2196 2241 for (qnum = inum;
2197 2242 qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL;
2198 2243 qnum += nvme->n_intr_cnt) {
2199 - while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) {
2200 - taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq,
2201 - cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent);
2202 - ccnt++;
2203 - }
2244 + cnt_cmds = nvme_process_cq_cmds(nvme, nvme->n_ioq[qnum]);
2245 + ccnt += cnt_cmds;
2204 2246 }
2205 2247
2206 2248 return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
2207 2249 }
2208 2250
2209 2251 static void
2210 2252 nvme_release_interrupts(nvme_t *nvme)
2211 2253 {
2212 2254 int i;
2213 2255
2214 2256 for (i = 0; i < nvme->n_intr_cnt; i++) {
2215 2257 if (nvme->n_inth[i] == NULL)
2216 2258 break;
2217 2259
2218 2260 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK)
2219 2261 (void) ddi_intr_block_disable(&nvme->n_inth[i], 1);
2220 2262 else
2221 2263 (void) ddi_intr_disable(nvme->n_inth[i]);
2222 2264
2223 2265 (void) ddi_intr_remove_handler(nvme->n_inth[i]);
2224 2266 (void) ddi_intr_free(nvme->n_inth[i]);
2225 2267 }
2226 2268
2227 2269 kmem_free(nvme->n_inth, nvme->n_inth_sz);
2228 2270 nvme->n_inth = NULL;
2229 2271 nvme->n_inth_sz = 0;
2230 2272
2231 2273 nvme->n_progress &= ~NVME_INTERRUPTS;
2232 2274 }
2233 2275
2234 2276 static int
2235 2277 nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs)
2236 2278 {
2237 2279 int nintrs, navail, count;
2238 2280 int ret;
2239 2281 int i;
2240 2282
2241 2283 if (nvme->n_intr_types == 0) {
2242 2284 ret = ddi_intr_get_supported_types(nvme->n_dip,
2243 2285 &nvme->n_intr_types);
2244 2286 if (ret != DDI_SUCCESS) {
2245 2287 dev_err(nvme->n_dip, CE_WARN,
2246 2288 "!%s: ddi_intr_get_supported types failed",
2247 2289 __func__);
2248 2290 return (ret);
2249 2291 }
2250 2292 }
2251 2293
2252 2294 if ((nvme->n_intr_types & intr_type) == 0)
2253 2295 return (DDI_FAILURE);
2254 2296
2255 2297 ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs);
2256 2298 if (ret != DDI_SUCCESS) {
2257 2299 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed",
2258 2300 __func__);
2259 2301 return (ret);
2260 2302 }
2261 2303
2262 2304 ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail);
2263 2305 if (ret != DDI_SUCCESS) {
2264 2306 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed",
2265 2307 __func__);
2266 2308 return (ret);
2267 2309 }
2268 2310
2269 2311 /* We want at most one interrupt per queue pair. */
2270 2312 if (navail > nqpairs)
2271 2313 navail = nqpairs;
2272 2314
2273 2315 nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail;
2274 2316 nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP);
2275 2317
2276 2318 ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail,
2277 2319 &count, 0);
2278 2320 if (ret != DDI_SUCCESS) {
2279 2321 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed",
2280 2322 __func__);
2281 2323 goto fail;
2282 2324 }
2283 2325
2284 2326 nvme->n_intr_cnt = count;
2285 2327
2286 2328 ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri);
2287 2329 if (ret != DDI_SUCCESS) {
2288 2330 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed",
2289 2331 __func__);
2290 2332 goto fail;
2291 2333 }
2292 2334
2293 2335 for (i = 0; i < count; i++) {
2294 2336 ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr,
2295 2337 (void *)nvme, (void *)(uintptr_t)i);
2296 2338 if (ret != DDI_SUCCESS) {
2297 2339 dev_err(nvme->n_dip, CE_WARN,
2298 2340 "!%s: ddi_intr_add_handler failed", __func__);
2299 2341 goto fail;
2300 2342 }
2301 2343 }
2302 2344
2303 2345 (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap);
2304 2346
2305 2347 for (i = 0; i < count; i++) {
2306 2348 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK)
2307 2349 ret = ddi_intr_block_enable(&nvme->n_inth[i], 1);
2308 2350 else
2309 2351 ret = ddi_intr_enable(nvme->n_inth[i]);
2310 2352
2311 2353 if (ret != DDI_SUCCESS) {
2312 2354 dev_err(nvme->n_dip, CE_WARN,
2313 2355 "!%s: enabling interrupt %d failed", __func__, i);
2314 2356 goto fail;
2315 2357 }
2316 2358 }
2317 2359
2318 2360 nvme->n_intr_type = intr_type;
2319 2361
2320 2362 nvme->n_progress |= NVME_INTERRUPTS;
2321 2363
2322 2364 return (DDI_SUCCESS);
2323 2365
2324 2366 fail:
2325 2367 nvme_release_interrupts(nvme);
2326 2368
2327 2369 return (ret);
2328 2370 }
2329 2371
2330 2372 static int
2331 2373 nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg)
2332 2374 {
2333 2375 _NOTE(ARGUNUSED(arg));
2334 2376
2335 2377 pci_ereport_post(dip, fm_error, NULL);
2336 2378 return (fm_error->fme_status);
2337 2379 }
2338 2380
2339 2381 static int
2340 2382 nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2341 2383 {
2342 2384 nvme_t *nvme;
2343 2385 int instance;
2344 2386 int nregs;
2345 2387 off_t regsize;
2346 2388 int i;
2347 2389 char name[32];
2348 2390
2349 2391 if (cmd != DDI_ATTACH)
2350 2392 return (DDI_FAILURE);
2351 2393
2352 2394 instance = ddi_get_instance(dip);
2353 2395
2354 2396 if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS)
2355 2397 return (DDI_FAILURE);
2356 2398
2357 2399 nvme = ddi_get_soft_state(nvme_state, instance);
2358 2400 ddi_set_driver_private(dip, nvme);
2359 2401 nvme->n_dip = dip;
2360 2402
2361 2403 nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
2362 2404 DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE;
2363 2405 nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY,
2364 2406 dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ?
2365 2407 B_TRUE : B_FALSE;
2366 2408 nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
2367 2409 DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN);
2368 2410 nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
2369 2411 DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN);
2370 2412 nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
2371 2413 DDI_PROP_DONTPASS, "async-event-limit",
2372 2414 NVME_DEFAULT_ASYNC_EVENT_LIMIT);
2373 2415
2374 2416 if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN)
2375 2417 nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN;
2376 2418 else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN)
2377 2419 nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN;
2378 2420
2379 2421 if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN)
2380 2422 nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN;
2381 2423
2382 2424 if (nvme->n_async_event_limit < 1)
2383 2425 nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT;
2384 2426
2385 2427 nvme->n_reg_acc_attr = nvme_reg_acc_attr;
2386 2428 nvme->n_queue_dma_attr = nvme_queue_dma_attr;
2387 2429 nvme->n_prp_dma_attr = nvme_prp_dma_attr;
2388 2430 nvme->n_sgl_dma_attr = nvme_sgl_dma_attr;
2389 2431
2390 2432 /*
2391 2433 * Setup FMA support.
2392 2434 */
2393 2435 nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip,
2394 2436 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
2395 2437 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
2396 2438 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
2397 2439
2398 2440 ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc);
2399 2441
2400 2442 if (nvme->n_fm_cap) {
2401 2443 if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE)
2402 2444 nvme->n_reg_acc_attr.devacc_attr_access =
2403 2445 DDI_FLAGERR_ACC;
2404 2446
2405 2447 if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) {
2406 2448 nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
2407 2449 nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
2408 2450 }
2409 2451
2410 2452 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) ||
2411 2453 DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
2412 2454 pci_ereport_setup(dip);
2413 2455
2414 2456 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
2415 2457 ddi_fm_handler_register(dip, nvme_fm_errcb,
2416 2458 (void *)nvme);
2417 2459 }
2418 2460
2419 2461 nvme->n_progress |= NVME_FMA_INIT;
2420 2462
2421 2463 /*
2422 2464 * The spec defines several register sets. Only the controller
2423 2465 * registers (set 1) are currently used.
2424 2466 */
2425 2467 if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE ||
2426 2468 nregs < 2 ||
2427 2469 ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE)
2428 2470 goto fail;
2429 2471
2430 2472 if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize,
2431 2473 &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) {
2432 2474 dev_err(dip, CE_WARN, "!failed to map regset 1");
2433 2475 goto fail;
2434 2476 }
2435 2477
2436 2478 nvme->n_progress |= NVME_REGS_MAPPED;
2437 2479
2438 2480 /*
2439 2481 * Create taskq for command completion.
2440 2482 */
2441 2483 (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq",
2442 2484 ddi_driver_name(dip), ddi_get_instance(dip));
2443 2485 nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus),
2444 2486 TASKQ_DEFAULTPRI, 0);
2445 2487 if (nvme->n_cmd_taskq == NULL) {
2446 2488 dev_err(dip, CE_WARN, "!failed to create cmd taskq");
2447 2489 goto fail;
2448 2490 }
2449 2491
2450 2492
2451 2493 if (nvme_init(nvme) != DDI_SUCCESS)
2452 2494 goto fail;
2453 2495
2454 2496 /*
2455 2497 * Attach the blkdev driver for each namespace.
2456 2498 */
2457 2499 for (i = 0; i != nvme->n_namespace_count; i++) {
2458 2500 if (nvme->n_ns[i].ns_ignore)
2459 2501 continue;
2460 2502
2461 2503 nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i],
2462 2504 &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP);
2463 2505
2464 2506 if (nvme->n_ns[i].ns_bd_hdl == NULL) {
2465 2507 dev_err(dip, CE_WARN,
2466 2508 "!failed to get blkdev handle for namespace %d", i);
2467 2509 goto fail;
2468 2510 }
2469 2511
2470 2512 if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl)
2471 2513 != DDI_SUCCESS) {
2472 2514 dev_err(dip, CE_WARN,
2473 2515 "!failed to attach blkdev handle for namespace %d",
2474 2516 i);
2475 2517 goto fail;
2476 2518 }
2477 2519 }
2478 2520
2479 2521 return (DDI_SUCCESS);
2480 2522
2481 2523 fail:
2482 2524 /* attach successful anyway so that FMA can retire the device */
2483 2525 if (nvme->n_dead)
2484 2526 return (DDI_SUCCESS);
2485 2527
2486 2528 (void) nvme_detach(dip, DDI_DETACH);
2487 2529
2488 2530 return (DDI_FAILURE);
2489 2531 }
2490 2532
2491 2533 static int
2492 2534 nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2493 2535 {
2494 2536 int instance, i;
2495 2537 nvme_t *nvme;
2496 2538
2497 2539 if (cmd != DDI_DETACH)
2498 2540 return (DDI_FAILURE);
2499 2541
2500 2542 instance = ddi_get_instance(dip);
2501 2543
2502 2544 nvme = ddi_get_soft_state(nvme_state, instance);
2503 2545
2504 2546 if (nvme == NULL)
2505 2547 return (DDI_FAILURE);
2506 2548
2507 2549 if (nvme->n_ns) {
2508 2550 for (i = 0; i != nvme->n_namespace_count; i++) {
2509 2551 if (nvme->n_ns[i].ns_bd_hdl) {
2510 2552 (void) bd_detach_handle(
2511 2553 nvme->n_ns[i].ns_bd_hdl);
2512 2554 bd_free_handle(nvme->n_ns[i].ns_bd_hdl);
2513 2555 }
2514 2556
2515 2557 if (nvme->n_ns[i].ns_idns)
2516 2558 kmem_free(nvme->n_ns[i].ns_idns,
2517 2559 sizeof (nvme_identify_nsid_t));
2518 2560 }
2519 2561
2520 2562 kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) *
2521 2563 nvme->n_namespace_count);
2522 2564 }
2523 2565
|
↓ open down ↓ |
310 lines elided |
↑ open up ↑ |
2524 2566 if (nvme->n_progress & NVME_INTERRUPTS)
2525 2567 nvme_release_interrupts(nvme);
2526 2568
2527 2569 if (nvme->n_cmd_taskq)
2528 2570 ddi_taskq_wait(nvme->n_cmd_taskq);
2529 2571
2530 2572 if (nvme->n_ioq_count > 0) {
2531 2573 for (i = 1; i != nvme->n_ioq_count + 1; i++) {
2532 2574 if (nvme->n_ioq[i] != NULL) {
2533 2575 /* TODO: send destroy queue commands */
2576 + mutex_enter(&nvme_global_mutex);
2577 + list_remove(&nvme_qp_list, nvme->n_ioq[i]);
2578 + mutex_exit(&nvme_global_mutex);
2534 2579 nvme_free_qpair(nvme->n_ioq[i]);
2535 2580 }
2536 2581 }
2537 2582
2538 2583 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) *
2539 2584 (nvme->n_ioq_count + 1));
2540 2585 }
2541 2586
2542 2587 if (nvme->n_progress & NVME_REGS_MAPPED) {
2543 2588 nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE);
2544 2589 (void) nvme_reset(nvme, B_FALSE);
2545 2590 }
2546 2591
2547 2592 if (nvme->n_cmd_taskq)
2548 2593 ddi_taskq_destroy(nvme->n_cmd_taskq);
2549 2594
2550 2595 if (nvme->n_progress & NVME_CTRL_LIMITS)
2551 2596 sema_destroy(&nvme->n_abort_sema);
2552 2597
2553 2598 if (nvme->n_progress & NVME_ADMIN_QUEUE)
2554 2599 nvme_free_qpair(nvme->n_adminq);
2555 2600
2556 2601 if (nvme->n_idctl)
2557 2602 kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t));
2558 2603
2559 2604 if (nvme->n_progress & NVME_REGS_MAPPED)
2560 2605 ddi_regs_map_free(&nvme->n_regh);
2561 2606
2562 2607 if (nvme->n_progress & NVME_FMA_INIT) {
2563 2608 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
2564 2609 ddi_fm_handler_unregister(nvme->n_dip);
2565 2610
2566 2611 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) ||
2567 2612 DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
2568 2613 pci_ereport_teardown(nvme->n_dip);
2569 2614
2570 2615 ddi_fm_fini(nvme->n_dip);
2571 2616 }
2572 2617
2573 2618 if (nvme->n_vendor != NULL)
2574 2619 strfree(nvme->n_vendor);
2575 2620
2576 2621 if (nvme->n_product != NULL)
2577 2622 strfree(nvme->n_product);
2578 2623
2579 2624 ddi_soft_state_free(nvme_state, instance);
2580 2625
2581 2626 return (DDI_SUCCESS);
2582 2627 }
2583 2628
2584 2629 static int
2585 2630 nvme_quiesce(dev_info_t *dip)
2586 2631 {
2587 2632 int instance;
2588 2633 nvme_t *nvme;
2589 2634
2590 2635 instance = ddi_get_instance(dip);
2591 2636
2592 2637 nvme = ddi_get_soft_state(nvme_state, instance);
2593 2638
2594 2639 if (nvme == NULL)
2595 2640 return (DDI_FAILURE);
2596 2641
2597 2642 nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE);
2598 2643
2599 2644 (void) nvme_reset(nvme, B_TRUE);
2600 2645
2601 2646 return (DDI_FAILURE);
2602 2647 }
2603 2648
2604 2649 static int
2605 2650 nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer)
2606 2651 {
2607 2652 nvme_t *nvme = cmd->nc_nvme;
2608 2653 int nprp_page, nprp;
2609 2654 uint64_t *prp;
2610 2655
2611 2656 if (xfer->x_ndmac == 0)
2612 2657 return (DDI_FAILURE);
2613 2658
2614 2659 cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress;
2615 2660 ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac);
2616 2661
2617 2662 if (xfer->x_ndmac == 1) {
2618 2663 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0;
2619 2664 return (DDI_SUCCESS);
2620 2665 } else if (xfer->x_ndmac == 2) {
2621 2666 cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress;
2622 2667 return (DDI_SUCCESS);
2623 2668 }
2624 2669
2625 2670 xfer->x_ndmac--;
2626 2671
2627 2672 nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1;
2628 2673 ASSERT(nprp_page > 0);
2629 2674 nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page;
2630 2675
2631 2676 /*
2632 2677 * We currently don't support chained PRPs and set up our DMA
2633 2678 * attributes to reflect that. If we still get an I/O request
2634 2679 * that needs a chained PRP something is very wrong.
2635 2680 */
2636 2681 VERIFY(nprp == 1);
2637 2682
2638 2683 if (nvme_zalloc_dma(nvme, nvme->n_pagesize * nprp, DDI_DMA_READ,
2639 2684 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
2640 2685 dev_err(nvme->n_dip, CE_WARN, "!%s: nvme_zalloc_dma failed",
2641 2686 __func__);
2642 2687 return (DDI_FAILURE);
2643 2688 }
2644 2689
2645 2690 cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress;
2646 2691 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, &cmd->nc_dma->nd_cookie);
2647 2692
2648 2693 /*LINTED: E_PTR_BAD_CAST_ALIGN*/
2649 2694 for (prp = (uint64_t *)cmd->nc_dma->nd_memp;
2650 2695 xfer->x_ndmac > 0;
2651 2696 prp++, xfer->x_ndmac--) {
2652 2697 *prp = xfer->x_dmac.dmac_laddress;
2653 2698 ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac);
2654 2699 }
2655 2700
2656 2701 (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len,
2657 2702 DDI_DMA_SYNC_FORDEV);
2658 2703 return (DDI_SUCCESS);
2659 2704 }
2660 2705
2661 2706 static nvme_cmd_t *
2662 2707 nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer)
2663 2708 {
2664 2709 nvme_t *nvme = ns->ns_nvme;
2665 2710 nvme_cmd_t *cmd;
2666 2711
2667 2712 /*
2668 2713 * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep.
2669 2714 */
2670 2715 cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ?
2671 2716 KM_NOSLEEP : KM_SLEEP);
2672 2717
2673 2718 if (cmd == NULL)
2674 2719 return (NULL);
2675 2720
2676 2721 cmd->nc_sqe.sqe_opc = opc;
2677 2722 cmd->nc_callback = nvme_bd_xfer_done;
2678 2723 cmd->nc_xfer = xfer;
2679 2724
2680 2725 switch (opc) {
2681 2726 case NVME_OPC_NVM_WRITE:
2682 2727 case NVME_OPC_NVM_READ:
2683 2728 VERIFY(xfer->x_nblks <= 0x10000);
2684 2729
2685 2730 cmd->nc_sqe.sqe_nsid = ns->ns_id;
2686 2731
2687 2732 cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu;
2688 2733 cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32);
2689 2734 cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1);
2690 2735
2691 2736 if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS)
2692 2737 goto fail;
2693 2738 break;
2694 2739
2695 2740 case NVME_OPC_NVM_FLUSH:
2696 2741 cmd->nc_sqe.sqe_nsid = ns->ns_id;
2697 2742 break;
2698 2743
2699 2744 default:
2700 2745 goto fail;
2701 2746 }
2702 2747
2703 2748 return (cmd);
2704 2749
2705 2750 fail:
2706 2751 nvme_free_cmd(cmd);
2707 2752 return (NULL);
2708 2753 }
2709 2754
2710 2755 static void
2711 2756 nvme_bd_xfer_done(void *arg)
2712 2757 {
2713 2758 nvme_cmd_t *cmd = arg;
2714 2759 bd_xfer_t *xfer = cmd->nc_xfer;
2715 2760 int error = 0;
2716 2761
2717 2762 error = nvme_check_cmd_status(cmd);
2718 2763 nvme_free_cmd(cmd);
2719 2764
2720 2765 bd_xfer_done(xfer, error);
2721 2766 }
2722 2767
2723 2768 static void
2724 2769 nvme_bd_driveinfo(void *arg, bd_drive_t *drive)
2725 2770 {
2726 2771 nvme_namespace_t *ns = arg;
2727 2772 nvme_t *nvme = ns->ns_nvme;
2728 2773
2729 2774 /*
2730 2775 * blkdev maintains one queue size per instance (namespace),
2731 2776 * but all namespace share the I/O queues.
2732 2777 * TODO: need to figure out a sane default, or use per-NS I/O queues,
2733 2778 * or change blkdev to handle EAGAIN
2734 2779 */
2735 2780 drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len
2736 2781 / nvme->n_namespace_count;
2737 2782
2738 2783 /*
2739 2784 * d_maxxfer is not set, which means the value is taken from the DMA
2740 2785 * attributes specified to bd_alloc_handle.
2741 2786 */
2742 2787
2743 2788 drive->d_removable = B_FALSE;
2744 2789 drive->d_hotpluggable = B_FALSE;
2745 2790
2746 2791 drive->d_target = ns->ns_id;
2747 2792 drive->d_lun = 0;
2748 2793
2749 2794 drive->d_model = nvme->n_idctl->id_model;
2750 2795 drive->d_model_len = sizeof (nvme->n_idctl->id_model);
2751 2796 drive->d_vendor = nvme->n_vendor;
2752 2797 drive->d_vendor_len = strlen(nvme->n_vendor);
2753 2798 drive->d_product = nvme->n_product;
2754 2799 drive->d_product_len = strlen(nvme->n_product);
2755 2800 drive->d_serial = nvme->n_idctl->id_serial;
2756 2801 drive->d_serial_len = sizeof (nvme->n_idctl->id_serial);
2757 2802 drive->d_revision = nvme->n_idctl->id_fwrev;
2758 2803 drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev);
2759 2804 }
2760 2805
2761 2806 static int
2762 2807 nvme_bd_mediainfo(void *arg, bd_media_t *media)
2763 2808 {
2764 2809 nvme_namespace_t *ns = arg;
2765 2810
2766 2811 media->m_nblks = ns->ns_block_count;
2767 2812 media->m_blksize = ns->ns_block_size;
2768 2813 media->m_readonly = B_FALSE;
2769 2814 media->m_solidstate = B_TRUE;
2770 2815
2771 2816 media->m_pblksize = ns->ns_best_block_size;
2772 2817
2773 2818 return (0);
2774 2819 }
2775 2820
2776 2821 static int
2777 2822 nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc)
2778 2823 {
2779 2824 nvme_t *nvme = ns->ns_nvme;
2780 2825 nvme_cmd_t *cmd;
2781 2826
2782 2827 if (nvme->n_dead)
2783 2828 return (EIO);
2784 2829
2785 2830 /* No polling for now */
2786 2831 if (xfer->x_flags & BD_XFER_POLL)
2787 2832 return (EIO);
2788 2833
2789 2834 cmd = nvme_create_nvm_cmd(ns, opc, xfer);
2790 2835 if (cmd == NULL)
2791 2836 return (ENOMEM);
2792 2837
2793 2838 cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1;
2794 2839 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count);
2795 2840
2796 2841 if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd)
2797 2842 != DDI_SUCCESS)
2798 2843 return (EAGAIN);
2799 2844
2800 2845 return (0);
2801 2846 }
2802 2847
2803 2848 static int
2804 2849 nvme_bd_read(void *arg, bd_xfer_t *xfer)
2805 2850 {
2806 2851 nvme_namespace_t *ns = arg;
2807 2852
2808 2853 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ));
2809 2854 }
2810 2855
2811 2856 static int
2812 2857 nvme_bd_write(void *arg, bd_xfer_t *xfer)
2813 2858 {
2814 2859 nvme_namespace_t *ns = arg;
2815 2860
2816 2861 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE));
2817 2862 }
2818 2863
2819 2864 static int
2820 2865 nvme_bd_sync(void *arg, bd_xfer_t *xfer)
2821 2866 {
2822 2867 nvme_namespace_t *ns = arg;
2823 2868
2824 2869 if (ns->ns_nvme->n_dead)
2825 2870 return (EIO);
2826 2871
2827 2872 /*
2828 2873 * If the volatile write cache isn't enabled the FLUSH command is a
2829 2874 * no-op, so we can take a shortcut here.
2830 2875 */
2831 2876 if (ns->ns_nvme->n_volatile_write_cache_enabled == B_FALSE) {
2832 2877 bd_xfer_done(xfer, ENOTSUP);
2833 2878 return (0);
2834 2879 }
2835 2880
|
↓ open down ↓ |
292 lines elided |
↑ open up ↑ |
2836 2881 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH));
2837 2882 }
2838 2883
2839 2884 static int
2840 2885 nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid)
2841 2886 {
2842 2887 nvme_namespace_t *ns = arg;
2843 2888
2844 2889 return (ddi_devid_init(devinfo, DEVID_ENCAP, strlen(ns->ns_devid),
2845 2890 ns->ns_devid, devid));
2891 +}
2892 +
2893 +static void
2894 +nvme_intr_monitor(void *arg)
2895 +{
2896 + nvme_qpair_t *qp;
2897 + hrtime_t diff, now_ns;
2898 +
2899 + if (!nvme_enable_intr_monitoring)
2900 + return;
2901 + mutex_enter(&nvme_global_mutex);
2902 + now_ns = gethrtime();
2903 + for (qp = list_head(&nvme_qp_list); qp != NULL;
2904 + qp = list_next(&nvme_qp_list, qp)) {
2905 + diff = now_ns - qp->nq_ts;
2906 + if (diff >= nvme_intr_timeout_ns && qp->nq_active_cmds > 0) {
2907 + if (nvme_process_cq_cmds(qp->nq_nvme, qp)) {
2908 + nvme_intr_timeouts++;
2909 + qp->nq_nvme->n_intr_timeouts++;
2910 + }
2911 + }
2912 + }
2913 + mutex_exit(&nvme_global_mutex);
2846 2914 }
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX