1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27
28 /*
29 * Deimos - cryptographic acceleration based upon Broadcom 582x.
30 */
31
32 #include <sys/types.h>
33 #include <sys/modctl.h>
34 #include <sys/conf.h>
35 #include <sys/devops.h>
36 #include <sys/ddi.h>
37 #include <sys/sunddi.h>
38 #include <sys/cmn_err.h>
39 #include <sys/varargs.h>
40 #include <sys/file.h>
41 #include <sys/stat.h>
42 #include <sys/kmem.h>
43 #include <sys/ioccom.h>
44 #include <sys/open.h>
45 #include <sys/cred.h>
46 #include <sys/kstat.h>
47 #include <sys/strsun.h>
48 #include <sys/note.h>
49 #include <sys/crypto/common.h>
50 #include <sys/crypto/spi.h>
51 #include <sys/ddifm.h>
52 #include <sys/fm/protocol.h>
53 #include <sys/fm/util.h>
54 #include <sys/fm/io/ddi.h>
55 #include <sys/crypto/dca.h>
56
57 /*
58 * Core Deimos driver.
59 */
60
61 static void dca_enlist2(dca_listnode_t *, dca_listnode_t *,
62 kmutex_t *);
63 static void dca_rmlist2(dca_listnode_t *node, kmutex_t *);
64 static dca_listnode_t *dca_delist2(dca_listnode_t *q, kmutex_t *);
65 static void dca_free_context_list(dca_t *dca);
66 static int dca_free_context_low(crypto_ctx_t *ctx);
67 static int dca_attach(dev_info_t *, ddi_attach_cmd_t);
68 static int dca_detach(dev_info_t *, ddi_detach_cmd_t);
69 static int dca_suspend(dca_t *);
70 static int dca_resume(dca_t *);
71 static int dca_init(dca_t *);
72 static int dca_reset(dca_t *, int);
73 static int dca_initworklist(dca_t *, dca_worklist_t *);
74 static void dca_uninit(dca_t *);
75 static void dca_initq(dca_listnode_t *);
76 static void dca_enqueue(dca_listnode_t *, dca_listnode_t *);
77 static dca_listnode_t *dca_dequeue(dca_listnode_t *);
78 static dca_listnode_t *dca_unqueue(dca_listnode_t *);
79 static dca_request_t *dca_newreq(dca_t *);
80 static dca_work_t *dca_getwork(dca_t *, int);
81 static void dca_freework(dca_work_t *);
82 static dca_work_t *dca_newwork(dca_t *);
83 static void dca_destroywork(dca_work_t *);
84 static void dca_schedule(dca_t *, int);
85 static void dca_reclaim(dca_t *, int);
86 static uint_t dca_intr(char *);
87 static void dca_failure(dca_t *, ddi_fault_location_t,
88 dca_fma_eclass_t index, uint64_t, int, char *, ...);
89 static void dca_jobtimeout(void *);
90 static int dca_drain(dca_t *);
91 static void dca_undrain(dca_t *);
92 static void dca_rejectjobs(dca_t *);
93
94 #ifdef SCHEDDELAY
95 static void dca_schedtimeout(void *);
96 #endif
97
98 /*
99 * We want these inlined for performance.
100 */
101 #ifndef DEBUG
102 #pragma inline(dca_freereq, dca_getreq, dca_freework, dca_getwork)
103 #pragma inline(dca_enqueue, dca_dequeue, dca_rmqueue, dca_done)
104 #pragma inline(dca_reverse, dca_length)
105 #endif
106
107 /*
108 * Device operations.
109 */
110 static struct dev_ops devops = {
111 DEVO_REV, /* devo_rev */
112 0, /* devo_refcnt */
113 nodev, /* devo_getinfo */
114 nulldev, /* devo_identify */
115 nulldev, /* devo_probe */
116 dca_attach, /* devo_attach */
117 dca_detach, /* devo_detach */
118 nodev, /* devo_reset */
119 NULL, /* devo_cb_ops */
120 NULL, /* devo_bus_ops */
121 ddi_power, /* devo_power */
122 ddi_quiesce_not_supported, /* devo_quiesce */
123 };
124
125 #define IDENT "PCI Crypto Accelerator"
126 #define IDENT_SYM "Crypto Accel Sym 2.0"
127 #define IDENT_ASYM "Crypto Accel Asym 2.0"
128
129 /* Space-padded, will be filled in dynamically during registration */
130 #define IDENT3 "PCI Crypto Accelerator Mod 2.0"
131
132 #define VENDOR "Sun Microsystems, Inc."
133
134 #define STALETIME (30 * SECOND)
135
136 #define crypto_prov_notify crypto_provider_notification
137 /* A 28 char function name doesn't leave much line space */
138
139 /*
140 * Module linkage.
141 */
142 static struct modldrv modldrv = {
143 &mod_driverops, /* drv_modops */
144 IDENT, /* drv_linkinfo */
145 &devops, /* drv_dev_ops */
146 };
147
148 extern struct mod_ops mod_cryptoops;
149
150 static struct modlcrypto modlcrypto = {
151 &mod_cryptoops,
152 IDENT3
153 };
154
155 static struct modlinkage modlinkage = {
156 MODREV_1, /* ml_rev */
157 &modldrv, /* ml_linkage */
158 &modlcrypto,
159 NULL
160 };
161
162 /*
163 * CSPI information (entry points, provider info, etc.)
164 */
165
166 /* Mechanisms for the symmetric cipher provider */
167 static crypto_mech_info_t dca_mech_info_tab1[] = {
168 /* DES-CBC */
169 {SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE,
170 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
171 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
172 DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
173 /* 3DES-CBC */
174 {SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE,
175 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
176 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
177 DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}
178 };
179
180 /* Mechanisms for the asymmetric cipher provider */
181 static crypto_mech_info_t dca_mech_info_tab2[] = {
182 /* DSA */
183 {SUN_CKM_DSA, DSA_MECH_INFO_TYPE,
184 CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY |
185 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC,
186 CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN),
187 CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN),
188 CRYPTO_KEYSIZE_UNIT_IN_BITS},
189
190 /* RSA */
191 {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
192 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
193 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
194 CRYPTO_FG_VERIFY_RECOVER |
195 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
196 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
197 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
198 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
199 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
200 CRYPTO_KEYSIZE_UNIT_IN_BITS},
201 {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
202 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
203 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
204 CRYPTO_FG_VERIFY_RECOVER |
205 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
206 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
207 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
208 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
209 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
210 CRYPTO_KEYSIZE_UNIT_IN_BITS}
211 };
212
213 static void dca_provider_status(crypto_provider_handle_t, uint_t *);
214
215 static crypto_control_ops_t dca_control_ops = {
216 dca_provider_status
217 };
218
219 static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
220 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
221 static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
222 crypto_req_handle_t);
223 static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *,
224 crypto_data_t *, crypto_req_handle_t);
225 static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *,
226 crypto_req_handle_t);
227 static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
228 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
229 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
230
231 static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
232 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
233 static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
234 crypto_req_handle_t);
235 static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *,
236 crypto_data_t *, crypto_req_handle_t);
237 static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *,
238 crypto_req_handle_t);
239 static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
240 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
241 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
242
243 static crypto_cipher_ops_t dca_cipher_ops = {
244 dca_encrypt_init,
245 dca_encrypt,
246 dca_encrypt_update,
247 dca_encrypt_final,
248 dca_encrypt_atomic,
249 dca_decrypt_init,
250 dca_decrypt,
251 dca_decrypt_update,
252 dca_decrypt_final,
253 dca_decrypt_atomic
254 };
255
256 static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
257 crypto_spi_ctx_template_t, crypto_req_handle_t);
258 static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
259 crypto_req_handle_t);
260 static int dca_sign_update(crypto_ctx_t *, crypto_data_t *,
261 crypto_req_handle_t);
262 static int dca_sign_final(crypto_ctx_t *, crypto_data_t *,
263 crypto_req_handle_t);
264 static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
265 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
266 crypto_spi_ctx_template_t, crypto_req_handle_t);
267 static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
268 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
269 static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
270 crypto_req_handle_t);
271 static int dca_sign_recover_atomic(crypto_provider_handle_t,
272 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
273 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
274
275 static crypto_sign_ops_t dca_sign_ops = {
276 dca_sign_init,
277 dca_sign,
278 dca_sign_update,
279 dca_sign_final,
280 dca_sign_atomic,
281 dca_sign_recover_init,
282 dca_sign_recover,
283 dca_sign_recover_atomic
284 };
285
286 static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *,
287 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
288 static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
289 crypto_req_handle_t);
290 static int dca_verify_update(crypto_ctx_t *, crypto_data_t *,
291 crypto_req_handle_t);
292 static int dca_verify_final(crypto_ctx_t *, crypto_data_t *,
293 crypto_req_handle_t);
294 static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
295 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
296 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
297 static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
298 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
299 static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *,
300 crypto_data_t *, crypto_req_handle_t);
301 static int dca_verify_recover_atomic(crypto_provider_handle_t,
302 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
303 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
304
305 static crypto_verify_ops_t dca_verify_ops = {
306 dca_verify_init,
307 dca_verify,
308 dca_verify_update,
309 dca_verify_final,
310 dca_verify_atomic,
311 dca_verify_recover_init,
312 dca_verify_recover,
313 dca_verify_recover_atomic
314 };
315
316 static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t,
317 uchar_t *, size_t, crypto_req_handle_t);
318
319 static crypto_random_number_ops_t dca_random_number_ops = {
320 NULL,
321 dca_generate_random
322 };
323
324 static int ext_info_sym(crypto_provider_handle_t prov,
325 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
326 static int ext_info_asym(crypto_provider_handle_t prov,
327 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
328 static int ext_info_base(crypto_provider_handle_t prov,
329 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id);
330
331 static crypto_provider_management_ops_t dca_provmanage_ops_1 = {
332 ext_info_sym, /* ext_info */
333 NULL, /* init_token */
334 NULL, /* init_pin */
335 NULL /* set_pin */
336 };
337
338 static crypto_provider_management_ops_t dca_provmanage_ops_2 = {
339 ext_info_asym, /* ext_info */
340 NULL, /* init_token */
341 NULL, /* init_pin */
342 NULL /* set_pin */
343 };
344
345 int dca_free_context(crypto_ctx_t *);
346
347 static crypto_ctx_ops_t dca_ctx_ops = {
348 NULL,
349 dca_free_context
350 };
351
352 /* Operations for the symmetric cipher provider */
353 static crypto_ops_t dca_crypto_ops1 = {
354 &dca_control_ops,
355 NULL, /* digest_ops */
356 &dca_cipher_ops,
357 NULL, /* mac_ops */
358 NULL, /* sign_ops */
359 NULL, /* verify_ops */
360 NULL, /* dual_ops */
361 NULL, /* cipher_mac_ops */
362 NULL, /* random_number_ops */
363 NULL, /* session_ops */
364 NULL, /* object_ops */
365 NULL, /* key_ops */
366 &dca_provmanage_ops_1, /* management_ops */
367 &dca_ctx_ops
368 };
369
370 /* Operations for the asymmetric cipher provider */
371 static crypto_ops_t dca_crypto_ops2 = {
372 &dca_control_ops,
373 NULL, /* digest_ops */
374 &dca_cipher_ops,
375 NULL, /* mac_ops */
376 &dca_sign_ops,
377 &dca_verify_ops,
378 NULL, /* dual_ops */
379 NULL, /* cipher_mac_ops */
380 &dca_random_number_ops,
381 NULL, /* session_ops */
382 NULL, /* object_ops */
383 NULL, /* key_ops */
384 &dca_provmanage_ops_2, /* management_ops */
385 &dca_ctx_ops
386 };
387
388 /* Provider information for the symmetric cipher provider */
389 static crypto_provider_info_t dca_prov_info1 = {
390 CRYPTO_SPI_VERSION_1,
391 NULL, /* pi_provider_description */
392 CRYPTO_HW_PROVIDER,
393 NULL, /* pi_provider_dev */
394 NULL, /* pi_provider_handle */
395 &dca_crypto_ops1,
396 sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t),
397 dca_mech_info_tab1,
398 0, /* pi_logical_provider_count */
399 NULL /* pi_logical_providers */
400 };
401
402 /* Provider information for the asymmetric cipher provider */
403 static crypto_provider_info_t dca_prov_info2 = {
404 CRYPTO_SPI_VERSION_1,
405 NULL, /* pi_provider_description */
406 CRYPTO_HW_PROVIDER,
407 NULL, /* pi_provider_dev */
408 NULL, /* pi_provider_handle */
409 &dca_crypto_ops2,
410 sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t),
411 dca_mech_info_tab2,
412 0, /* pi_logical_provider_count */
413 NULL /* pi_logical_providers */
414 };
415
416 /* Convenience macros */
417 /* Retrieve the softc and instance number from a SPI crypto context */
418 #define DCA_SOFTC_FROM_CTX(ctx, softc, instance) { \
419 (softc) = (dca_t *)(ctx)->cc_provider; \
420 (instance) = ddi_get_instance((softc)->dca_dip); \
421 }
422
423 #define DCA_MECH_FROM_CTX(ctx) \
424 (((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type)
425
426 static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
427 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
428 dca_chain_t *head, int *n_chain);
429 static uint64_t dca_ena(uint64_t ena);
430 static caddr_t dca_bufdaddr_out(crypto_data_t *data);
431 static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index);
432 static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
433 dca_fma_eclass_t eclass_index);
434
435 static void dca_fma_init(dca_t *dca);
436 static void dca_fma_fini(dca_t *dca);
437 static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
438 const void *impl_data);
439
440
441 static dca_device_t dca_devices[] = {
442 /* Broadcom vanilla variants */
443 { 0x14e4, 0x5820, "Broadcom 5820" },
444 { 0x14e4, 0x5821, "Broadcom 5821" },
445 { 0x14e4, 0x5822, "Broadcom 5822" },
446 { 0x14e4, 0x5825, "Broadcom 5825" },
447 /* Sun specific OEMd variants */
448 { 0x108e, 0x5454, "SCA" },
449 { 0x108e, 0x5455, "SCA 1000" },
450 { 0x108e, 0x5457, "SCA 500" },
451 /* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */
452 { 0x108e, 0x1, "SCA 500" },
453 };
454
455 /*
456 * Device attributes.
457 */
458 static struct ddi_device_acc_attr dca_regsattr = {
459 DDI_DEVICE_ATTR_V1,
460 DDI_STRUCTURE_LE_ACC,
461 DDI_STRICTORDER_ACC,
462 DDI_FLAGERR_ACC
463 };
464
465 static struct ddi_device_acc_attr dca_devattr = {
466 DDI_DEVICE_ATTR_V0,
467 DDI_STRUCTURE_LE_ACC,
468 DDI_STRICTORDER_ACC
469 };
470
471 #if !defined(i386) && !defined(__i386)
472 static struct ddi_device_acc_attr dca_bufattr = {
473 DDI_DEVICE_ATTR_V0,
474 DDI_NEVERSWAP_ACC,
475 DDI_STRICTORDER_ACC
476 };
477 #endif
478
479 static struct ddi_dma_attr dca_dmaattr = {
480 DMA_ATTR_V0, /* dma_attr_version */
481 0x0, /* dma_attr_addr_lo */
482 0xffffffffUL, /* dma_attr_addr_hi */
483 0x00ffffffUL, /* dma_attr_count_max */
484 0x40, /* dma_attr_align */
485 0x40, /* dma_attr_burstsizes */
486 0x1, /* dma_attr_minxfer */
487 0x00ffffffUL, /* dma_attr_maxxfer */
488 0xffffffffUL, /* dma_attr_seg */
489 #if defined(i386) || defined(__i386) || defined(__amd64)
490 512, /* dma_attr_sgllen */
491 #else
492 1, /* dma_attr_sgllen */
493 #endif
494 1, /* dma_attr_granular */
495 DDI_DMA_FLAGERR /* dma_attr_flags */
496 };
497
498 static void *dca_state = NULL;
499 int dca_mindma = 2500;
500
501 /*
502 * FMA eclass string definitions. Note that these string arrays must be
503 * consistent with the dca_fma_eclass_t enum.
504 */
505 static char *dca_fma_eclass_sca1000[] = {
506 "sca1000.hw.device",
507 "sca1000.hw.timeout",
508 "sca1000.none"
509 };
510
511 static char *dca_fma_eclass_sca500[] = {
512 "sca500.hw.device",
513 "sca500.hw.timeout",
514 "sca500.none"
515 };
516
517 /*
518 * DDI entry points.
519 */
520 int
521 _init(void)
522 {
523 int rv;
524
525 DBG(NULL, DMOD, "dca: in _init");
526
527 if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) {
528 /* this should *never* happen! */
529 return (rv);
530 }
531
532 if ((rv = mod_install(&modlinkage)) != 0) {
533 /* cleanup here */
534 ddi_soft_state_fini(&dca_state);
535 return (rv);
536 }
537
538 return (0);
539 }
540
541 int
542 _fini(void)
543 {
544 int rv;
545
546 DBG(NULL, DMOD, "dca: in _fini");
547
548 if ((rv = mod_remove(&modlinkage)) == 0) {
549 /* cleanup here */
550 ddi_soft_state_fini(&dca_state);
551 }
552 return (rv);
553 }
554
555 int
556 _info(struct modinfo *modinfop)
557 {
558 DBG(NULL, DMOD, "dca: in _info");
559
560 return (mod_info(&modlinkage, modinfop));
561 }
562
563 int
564 dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
565 {
566 ddi_acc_handle_t pci;
567 int instance;
568 ddi_iblock_cookie_t ibc;
569 int intr_added = 0;
570 dca_t *dca;
571 ushort_t venid;
572 ushort_t devid;
573 ushort_t revid;
574 ushort_t subsysid;
575 ushort_t subvenid;
576 int i;
577 int ret;
578 char ID[64];
579 static char *unknowndev = "Unknown device";
580
581 #if DEBUG
582 /* these are only used for debugging */
583 ushort_t pcicomm;
584 ushort_t pcistat;
585 uchar_t cachelinesz;
586 uchar_t mingnt;
587 uchar_t maxlat;
588 uchar_t lattmr;
589 #endif
590
591 instance = ddi_get_instance(dip);
592
593 DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance);
594
595 switch (cmd) {
596 case DDI_RESUME:
597 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
598 dca_diperror(dip, "no soft state in detach");
599 return (DDI_FAILURE);
600 }
601 /* assumption: we won't be DDI_DETACHed until we return */
602 return (dca_resume(dca));
603 case DDI_ATTACH:
604 break;
605 default:
606 return (DDI_FAILURE);
607 }
608
609 if (ddi_slaveonly(dip) == DDI_SUCCESS) {
610 dca_diperror(dip, "slot does not support PCI bus-master");
611 return (DDI_FAILURE);
612 }
613
614 if (ddi_intr_hilevel(dip, 0) != 0) {
615 dca_diperror(dip, "hilevel interrupts not supported");
616 return (DDI_FAILURE);
617 }
618
619 if (pci_config_setup(dip, &pci) != DDI_SUCCESS) {
620 dca_diperror(dip, "unable to setup PCI config handle");
621 return (DDI_FAILURE);
622 }
623
624 /* common PCI attributes */
625 venid = pci_config_get16(pci, PCI_VENID);
626 devid = pci_config_get16(pci, PCI_DEVID);
627 revid = pci_config_get8(pci, PCI_REVID);
628 subvenid = pci_config_get16(pci, PCI_SUBVENID);
629 subsysid = pci_config_get16(pci, PCI_SUBSYSID);
630
631 /*
632 * Broadcom-specific timings.
633 * We disable these timers/counters since they can cause
634 * incorrect false failures when the bus is just a little
635 * bit slow, or busy.
636 */
637 pci_config_put8(pci, PCI_TRDYTO, 0);
638 pci_config_put8(pci, PCI_RETRIES, 0);
639
640 /* initialize PCI access settings */
641 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
642 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);
643
644 /* set up our PCI latency timer */
645 pci_config_put8(pci, PCI_LATTMR, 0x40);
646
647 #if DEBUG
648 /* read registers (for debugging) */
649 pcicomm = pci_config_get16(pci, PCI_COMM);
650 pcistat = pci_config_get16(pci, PCI_STATUS);
651 cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ);
652 mingnt = pci_config_get8(pci, PCI_MINGNT);
653 maxlat = pci_config_get8(pci, PCI_MAXLAT);
654 lattmr = pci_config_get8(pci, PCI_LATTMR);
655 #endif
656
657 pci_config_teardown(&pci);
658
659 if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) {
660 dca_diperror(dip, "unable to get iblock cookie");
661 return (DDI_FAILURE);
662 }
663
664 if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) {
665 dca_diperror(dip, "unable to allocate soft state");
666 return (DDI_FAILURE);
667 }
668
669 dca = ddi_get_soft_state(dca_state, instance);
670 ASSERT(dca != NULL);
671 dca->dca_dip = dip;
672 WORKLIST(dca, MCR1)->dwl_prov = NULL;
673 WORKLIST(dca, MCR2)->dwl_prov = NULL;
674 /* figure pagesize */
675 dca->dca_pagesize = ddi_ptob(dip, 1);
676
677 /*
678 * Search for the device in our supported devices table. This
679 * is here for two reasons. First, we want to ensure that
680 * only Sun-qualified (and presumably Sun-labeled) devices can
681 * be used with this driver. Second, some devices have
682 * specific differences. E.g. the 5821 has support for a
683 * special mode of RC4, deeper queues, power management, and
684 * other changes. Also, the export versions of some of these
685 * chips don't support RC4 or 3DES, so we catch that here.
686 *
687 * Note that we only look at the upper nibble of the device
688 * id, which is used to distinguish export vs. domestic
689 * versions of the chip. (The lower nibble is used for
690 * stepping information.)
691 */
692 for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) {
693 /*
694 * Try to match the subsystem information first.
695 */
696 if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) &&
697 subsysid && (subsysid == dca_devices[i].dd_device_id)) {
698 dca->dca_model = dca_devices[i].dd_model;
699 dca->dca_devid = dca_devices[i].dd_device_id;
700 break;
701 }
702 /*
703 * Failing that, try the generic vendor and device id.
704 * Even if we find a match, we keep searching anyway,
705 * since we would prefer to find a match based on the
706 * subsystem ids.
707 */
708 if ((venid == dca_devices[i].dd_vendor_id) &&
709 (devid == dca_devices[i].dd_device_id)) {
710 dca->dca_model = dca_devices[i].dd_model;
711 dca->dca_devid = dca_devices[i].dd_device_id;
712 }
713 }
714 /* try and handle an unrecognized device */
715 if (dca->dca_model == NULL) {
716 dca->dca_model = unknowndev;
717 dca_error(dca, "device not recognized, not supported");
718 DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d",
719 i, venid, devid, revid);
720 }
721
722 if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description",
723 dca->dca_model) != DDI_SUCCESS) {
724 dca_error(dca, "unable to create description property");
725 return (DDI_FAILURE);
726 }
727
728 DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x",
729 pcicomm, pcistat, cachelinesz);
730 DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x",
731 mingnt, maxlat, lattmr);
732
733 /*
734 * initialize locks, etc.
735 */
736 (void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc);
737
738 /* use RNGSHA1 by default */
739 if (ddi_getprop(DDI_DEV_T_ANY, dip,
740 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) {
741 dca->dca_flags |= DCA_RNGSHA1;
742 }
743
744 /* initialize FMA */
745 dca_fma_init(dca);
746
747 /* initialize some key data structures */
748 if (dca_init(dca) != DDI_SUCCESS) {
749 goto failed;
750 }
751
752 /* initialize kstats */
753 dca_ksinit(dca);
754
755 /* setup access to registers */
756 if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs,
757 0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) {
758 dca_error(dca, "unable to map registers");
759 goto failed;
760 }
761
762 DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1));
763 DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL));
764 DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT));
765 DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA));
766 DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2));
767
768 /* reset the chip */
769 if (dca_reset(dca, 0) < 0) {
770 goto failed;
771 }
772
773 /* initialize the chip */
774 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
775 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
776 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
777 goto failed;
778 }
779
780 /* add the interrupt */
781 if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr,
782 (void *)dca) != DDI_SUCCESS) {
783 DBG(dca, DWARN, "ddi_add_intr failed");
784 goto failed;
785 } else {
786 intr_added = 1;
787 }
788
789 /* enable interrupts on the device */
790 /*
791 * XXX: Note, 5820A1 errata indicates that this may clobber
792 * bits 24 and 23, which affect the speed of the RNG. Since
793 * we always want to run in full-speed mode, this should be
794 * harmless.
795 */
796 if (dca->dca_devid == 0x5825) {
797 /* for 5825 - increase the DMA read size */
798 SETBIT(dca, CSR_DMACTL,
799 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
800 } else {
801 SETBIT(dca, CSR_DMACTL,
802 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
803 }
804 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
805 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
806 goto failed;
807 }
808
809 /* register MCR1 with the crypto framework */
810 /* Be careful not to exceed 32 chars */
811 (void) sprintf(ID, "%s/%d %s",
812 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM);
813 dca_prov_info1.pi_provider_description = ID;
814 dca_prov_info1.pi_provider_dev.pd_hw = dip;
815 dca_prov_info1.pi_provider_handle = dca;
816 if ((ret = crypto_register_provider(&dca_prov_info1,
817 &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) {
818 cmn_err(CE_WARN,
819 "crypto_register_provider() failed (%d) for MCR1", ret);
820 goto failed;
821 }
822
823 /* register MCR2 with the crypto framework */
824 /* Be careful not to exceed 32 chars */
825 (void) sprintf(ID, "%s/%d %s",
826 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM);
827 dca_prov_info2.pi_provider_description = ID;
828 dca_prov_info2.pi_provider_dev.pd_hw = dip;
829 dca_prov_info2.pi_provider_handle = dca;
830 if ((ret = crypto_register_provider(&dca_prov_info2,
831 &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) {
832 cmn_err(CE_WARN,
833 "crypto_register_provider() failed (%d) for MCR2", ret);
834 goto failed;
835 }
836
837 crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov,
838 CRYPTO_PROVIDER_READY);
839 crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov,
840 CRYPTO_PROVIDER_READY);
841
842 /* Initialize the local random number pool for this instance */
843 if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) {
844 goto failed;
845 }
846
847 mutex_enter(&dca->dca_intrlock);
848 dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca,
849 drv_usectohz(SECOND));
850 mutex_exit(&dca->dca_intrlock);
851
852 ddi_set_driver_private(dip, (caddr_t)dca);
853
854 ddi_report_dev(dip);
855
856 if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) {
857 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED);
858 }
859
860 return (DDI_SUCCESS);
861
862 failed:
863 /* unregister from the crypto framework */
864 if (WORKLIST(dca, MCR1)->dwl_prov != NULL) {
865 (void) crypto_unregister_provider(
866 WORKLIST(dca, MCR1)->dwl_prov);
867 }
868 if (WORKLIST(dca, MCR2)->dwl_prov != NULL) {
869 (void) crypto_unregister_provider(
870 WORKLIST(dca, MCR2)->dwl_prov);
871 }
872 if (intr_added) {
873 CLRBIT(dca, CSR_DMACTL,
874 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
875 /* unregister intr handler */
876 ddi_remove_intr(dip, 0, dca->dca_icookie);
877 }
878 if (dca->dca_regs_handle) {
879 ddi_regs_map_free(&dca->dca_regs_handle);
880 }
881 if (dca->dca_intrstats) {
882 kstat_delete(dca->dca_intrstats);
883 }
884 if (dca->dca_ksp) {
885 kstat_delete(dca->dca_ksp);
886 }
887 dca_uninit(dca);
888
889 /* finalize FMA */
890 dca_fma_fini(dca);
891
892 mutex_destroy(&dca->dca_intrlock);
893 ddi_soft_state_free(dca_state, instance);
894 return (DDI_FAILURE);
895
896 }
897
898 int
899 dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
900 {
901 int instance;
902 dca_t *dca;
903 timeout_id_t tid;
904
905 instance = ddi_get_instance(dip);
906
907 DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance);
908
909 switch (cmd) {
910 case DDI_SUSPEND:
911 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
912 dca_diperror(dip, "no soft state in detach");
913 return (DDI_FAILURE);
914 }
915 /* assumption: we won't be DDI_DETACHed until we return */
916 return (dca_suspend(dca));
917
918 case DDI_DETACH:
919 break;
920 default:
921 return (DDI_FAILURE);
922 }
923
924 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
925 dca_diperror(dip, "no soft state in detach");
926 return (DDI_FAILURE);
927 }
928
929 /*
930 * Unregister from kCF.
931 * This needs to be done at the beginning of detach.
932 */
933 if (WORKLIST(dca, MCR1)->dwl_prov != NULL) {
934 if (crypto_unregister_provider(
935 WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) {
936 dca_error(dca, "unable to unregister MCR1 from kcf");
937 return (DDI_FAILURE);
938 }
939 }
940
941 if (WORKLIST(dca, MCR2)->dwl_prov != NULL) {
942 if (crypto_unregister_provider(
943 WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) {
944 dca_error(dca, "unable to unregister MCR2 from kcf");
945 return (DDI_FAILURE);
946 }
947 }
948
949 /*
950 * Cleanup the private context list. Once the
951 * crypto_unregister_provider returns, it is safe to do so.
952 */
953 dca_free_context_list(dca);
954
955 /* Cleanup the local random number pool */
956 dca_random_fini(dca);
957
958 /* send any jobs in the waitq back to kCF */
959 dca_rejectjobs(dca);
960
961 /* untimeout the timeouts */
962 mutex_enter(&dca->dca_intrlock);
963 tid = dca->dca_jobtid;
964 dca->dca_jobtid = 0;
965 mutex_exit(&dca->dca_intrlock);
966 if (tid) {
967 (void) untimeout(tid);
968 }
969
970 /* disable device interrupts */
971 CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
972
973 /* unregister interrupt handlers */
974 ddi_remove_intr(dip, 0, dca->dca_icookie);
975
976 /* release our regs handle */
977 ddi_regs_map_free(&dca->dca_regs_handle);
978
979 /* toss out kstats */
980 if (dca->dca_intrstats) {
981 kstat_delete(dca->dca_intrstats);
982 }
983 if (dca->dca_ksp) {
984 kstat_delete(dca->dca_ksp);
985 }
986
987 mutex_destroy(&dca->dca_intrlock);
988 dca_uninit(dca);
989
990 /* finalize FMA */
991 dca_fma_fini(dca);
992
993 ddi_soft_state_free(dca_state, instance);
994
995 return (DDI_SUCCESS);
996 }
997
998 int
999 dca_resume(dca_t *dca)
1000 {
1001 ddi_acc_handle_t pci;
1002
1003 if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) {
1004 dca_error(dca, "unable to setup PCI config handle");
1005 return (DDI_FAILURE);
1006 }
1007
1008 /*
1009 * Reprogram registers in PCI configuration space.
1010 */
1011
1012 /* Broadcom-specific timers -- we disable them. */
1013 pci_config_put8(pci, PCI_TRDYTO, 0);
1014 pci_config_put8(pci, PCI_RETRIES, 0);
1015
1016 /* initialize PCI access settings */
1017 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
1018 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);
1019
1020 /* set up our PCI latency timer */
1021 pci_config_put8(pci, PCI_LATTMR, 0x40);
1022
1023 pci_config_teardown(&pci);
1024
1025 if (dca_reset(dca, 0) < 0) {
1026 dca_error(dca, "unable to reset device during resume");
1027 return (DDI_FAILURE);
1028 }
1029
1030 /*
1031 * Now restore the card-specific CSRs.
1032 */
1033
1034 /* restore endianness settings */
1035 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
1036 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1037 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1038 return (DDI_FAILURE);
1039
1040 /* restore interrupt enables */
1041 if (dca->dca_devid == 0x5825) {
1042 /* for 5825 set 256 byte read size to improve performance */
1043 SETBIT(dca, CSR_DMACTL,
1044 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
1045 } else {
1046 SETBIT(dca, CSR_DMACTL,
1047 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
1048 }
1049 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1050 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1051 return (DDI_FAILURE);
1052
1053 /* resume scheduling jobs on the device */
1054 dca_undrain(dca);
1055
1056 return (DDI_SUCCESS);
1057 }
1058
1059 int
1060 dca_suspend(dca_t *dca)
1061 {
1062 if ((dca_drain(dca)) != 0) {
1063 return (DDI_FAILURE);
1064 }
1065 if (dca_reset(dca, 0) < 0) {
1066 dca_error(dca, "unable to reset device during suspend");
1067 return (DDI_FAILURE);
1068 }
1069 return (DDI_SUCCESS);
1070 }
1071
1072 /*
1073 * Hardware access stuff.
1074 */
1075 int
1076 dca_reset(dca_t *dca, int failreset)
1077 {
1078 int i;
1079
1080 if (dca->dca_regs_handle == NULL) {
1081 return (-1);
1082 }
1083
1084 PUTCSR(dca, CSR_DMACTL, DMACTL_RESET);
1085 if (!failreset) {
1086 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1087 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1088 return (-1);
1089 }
1090
1091 /* now wait for a reset */
1092 for (i = 1; i < 100; i++) {
1093 uint32_t dmactl;
1094 drv_usecwait(100);
1095 dmactl = GETCSR(dca, CSR_DMACTL);
1096 if (!failreset) {
1097 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1098 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1099 return (-1);
1100 }
1101 if ((dmactl & DMACTL_RESET) == 0) {
1102 DBG(dca, DCHATTY, "reset in %d usec", i * 100);
1103 return (0);
1104 }
1105 }
1106 if (!failreset) {
1107 dca_failure(dca, DDI_DEVICE_FAULT,
1108 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
1109 "timeout waiting for reset after %d usec", i * 100);
1110 }
1111 return (-1);
1112 }
1113
1114 int
1115 dca_initworklist(dca_t *dca, dca_worklist_t *wlp)
1116 {
1117 int i;
1118 int reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR);
1119
1120 /*
1121 * Set up work queue.
1122 */
1123 mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie);
1124 mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER,
1125 dca->dca_icookie);
1126 mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie);
1127 cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL);
1128
1129 mutex_enter(&wlp->dwl_lock);
1130
1131 dca_initq(&wlp->dwl_freereqs);
1132 dca_initq(&wlp->dwl_waitq);
1133 dca_initq(&wlp->dwl_freework);
1134 dca_initq(&wlp->dwl_runq);
1135
1136 for (i = 0; i < MAXWORK; i++) {
1137 dca_work_t *workp;
1138
1139 if ((workp = dca_newwork(dca)) == NULL) {
1140 dca_error(dca, "unable to allocate work");
1141 mutex_exit(&wlp->dwl_lock);
1142 return (DDI_FAILURE);
1143 }
1144 workp->dw_wlp = wlp;
1145 dca_freework(workp);
1146 }
1147 mutex_exit(&wlp->dwl_lock);
1148
1149 for (i = 0; i < reqprealloc; i++) {
1150 dca_request_t *reqp;
1151
1152 if ((reqp = dca_newreq(dca)) == NULL) {
1153 dca_error(dca, "unable to allocate request");
1154 return (DDI_FAILURE);
1155 }
1156 reqp->dr_dca = dca;
1157 reqp->dr_wlp = wlp;
1158 dca_freereq(reqp);
1159 }
1160 return (DDI_SUCCESS);
1161 }
1162
1163 int
1164 dca_init(dca_t *dca)
1165 {
1166 dca_worklist_t *wlp;
1167
1168 /* Initialize the private context list and the corresponding lock. */
1169 mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL);
1170 dca_initq(&dca->dca_ctx_list);
1171
1172 /*
1173 * MCR1 algorithms.
1174 */
1175 wlp = WORKLIST(dca, MCR1);
1176 (void) sprintf(wlp->dwl_name, "dca%d:mcr1",
1177 ddi_get_instance(dca->dca_dip));
1178 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
1179 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1180 "mcr1_lowater", MCR1LOWATER);
1181 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
1182 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1183 "mcr1_hiwater", MCR1HIWATER);
1184 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
1185 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1186 "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR);
1187 wlp->dwl_dca = dca;
1188 wlp->dwl_mcr = MCR1;
1189 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
1190 return (DDI_FAILURE);
1191 }
1192
1193 /*
1194 * MCR2 algorithms.
1195 */
1196 wlp = WORKLIST(dca, MCR2);
1197 (void) sprintf(wlp->dwl_name, "dca%d:mcr2",
1198 ddi_get_instance(dca->dca_dip));
1199 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
1200 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1201 "mcr2_lowater", MCR2LOWATER);
1202 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
1203 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1204 "mcr2_hiwater", MCR2HIWATER);
1205 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
1206 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1207 "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR);
1208 wlp->dwl_dca = dca;
1209 wlp->dwl_mcr = MCR2;
1210 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
1211 return (DDI_FAILURE);
1212 }
1213 return (DDI_SUCCESS);
1214 }
1215
1216 /*
1217 * Uninitialize worklists. This routine should only be called when no
1218 * active jobs (hence DMA mappings) exist. One way to ensure this is
1219 * to unregister from kCF before calling this routine. (This is done
1220 * e.g. in detach(9e).)
1221 */
1222 void
1223 dca_uninit(dca_t *dca)
1224 {
1225 int mcr;
1226
1227 mutex_destroy(&dca->dca_ctx_list_lock);
1228
1229 for (mcr = MCR1; mcr <= MCR2; mcr++) {
1230 dca_worklist_t *wlp = WORKLIST(dca, mcr);
1231 dca_work_t *workp;
1232 dca_request_t *reqp;
1233
1234 if (dca->dca_regs_handle == NULL) {
1235 continue;
1236 }
1237
1238 mutex_enter(&wlp->dwl_lock);
1239 while ((workp = dca_getwork(dca, mcr)) != NULL) {
1240 dca_destroywork(workp);
1241 }
1242 mutex_exit(&wlp->dwl_lock);
1243 while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) {
1244 dca_destroyreq(reqp);
1245 }
1246
1247 mutex_destroy(&wlp->dwl_lock);
1248 mutex_destroy(&wlp->dwl_freereqslock);
1249 mutex_destroy(&wlp->dwl_freelock);
1250 cv_destroy(&wlp->dwl_cv);
1251 wlp->dwl_prov = NULL;
1252 }
1253 }
1254
1255 static void
1256 dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock)
1257 {
1258 if (!q || !node)
1259 return;
1260
1261 mutex_enter(lock);
1262 node->dl_next2 = q;
1263 node->dl_prev2 = q->dl_prev2;
1264 node->dl_next2->dl_prev2 = node;
1265 node->dl_prev2->dl_next2 = node;
1266 mutex_exit(lock);
1267 }
1268
1269 static void
1270 dca_rmlist2(dca_listnode_t *node, kmutex_t *lock)
1271 {
1272 if (!node)
1273 return;
1274
1275 mutex_enter(lock);
1276 node->dl_next2->dl_prev2 = node->dl_prev2;
1277 node->dl_prev2->dl_next2 = node->dl_next2;
1278 node->dl_next2 = NULL;
1279 node->dl_prev2 = NULL;
1280 mutex_exit(lock);
1281 }
1282
1283 static dca_listnode_t *
1284 dca_delist2(dca_listnode_t *q, kmutex_t *lock)
1285 {
1286 dca_listnode_t *node;
1287
1288 mutex_enter(lock);
1289 if ((node = q->dl_next2) == q) {
1290 mutex_exit(lock);
1291 return (NULL);
1292 }
1293
1294 node->dl_next2->dl_prev2 = node->dl_prev2;
1295 node->dl_prev2->dl_next2 = node->dl_next2;
1296 node->dl_next2 = NULL;
1297 node->dl_prev2 = NULL;
1298 mutex_exit(lock);
1299
1300 return (node);
1301 }
1302
1303 void
1304 dca_initq(dca_listnode_t *q)
1305 {
1306 q->dl_next = q;
1307 q->dl_prev = q;
1308 q->dl_next2 = q;
1309 q->dl_prev2 = q;
1310 }
1311
1312 void
1313 dca_enqueue(dca_listnode_t *q, dca_listnode_t *node)
1314 {
1315 /*
1316 * Enqueue submits at the "tail" of the list, i.e. just
1317 * behind the sentinel.
1318 */
1319 node->dl_next = q;
1320 node->dl_prev = q->dl_prev;
1321 node->dl_next->dl_prev = node;
1322 node->dl_prev->dl_next = node;
1323 }
1324
1325 void
1326 dca_rmqueue(dca_listnode_t *node)
1327 {
1328 node->dl_next->dl_prev = node->dl_prev;
1329 node->dl_prev->dl_next = node->dl_next;
1330 node->dl_next = NULL;
1331 node->dl_prev = NULL;
1332 }
1333
1334 dca_listnode_t *
1335 dca_dequeue(dca_listnode_t *q)
1336 {
1337 dca_listnode_t *node;
1338 /*
1339 * Dequeue takes from the "head" of the list, i.e. just after
1340 * the sentinel.
1341 */
1342 if ((node = q->dl_next) == q) {
1343 /* queue is empty */
1344 return (NULL);
1345 }
1346 dca_rmqueue(node);
1347 return (node);
1348 }
1349
1350 /* this is the opposite of dequeue, it takes things off in LIFO order */
1351 dca_listnode_t *
1352 dca_unqueue(dca_listnode_t *q)
1353 {
1354 dca_listnode_t *node;
1355 /*
1356 * unqueue takes from the "tail" of the list, i.e. just before
1357 * the sentinel.
1358 */
1359 if ((node = q->dl_prev) == q) {
1360 /* queue is empty */
1361 return (NULL);
1362 }
1363 dca_rmqueue(node);
1364 return (node);
1365 }
1366
1367 dca_listnode_t *
1368 dca_peekqueue(dca_listnode_t *q)
1369 {
1370 dca_listnode_t *node;
1371
1372 if ((node = q->dl_next) == q) {
1373 return (NULL);
1374 } else {
1375 return (node);
1376 }
1377 }
1378
1379 /*
1380 * Interrupt service routine.
1381 */
1382 uint_t
1383 dca_intr(char *arg)
1384 {
1385 dca_t *dca = (dca_t *)arg;
1386 uint32_t status;
1387
1388 mutex_enter(&dca->dca_intrlock);
1389 status = GETCSR(dca, CSR_DMASTAT);
1390 PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS);
1391 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1392 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
1393 mutex_exit(&dca->dca_intrlock);
1394 return ((uint_t)DDI_FAILURE);
1395 }
1396
1397 DBG(dca, DINTR, "interrupted, status = 0x%x!", status);
1398
1399 if ((status & DMASTAT_INTERRUPTS) == 0) {
1400 /* increment spurious interrupt kstat */
1401 if (dca->dca_intrstats) {
1402 KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++;
1403 }
1404 mutex_exit(&dca->dca_intrlock);
1405 return (DDI_INTR_UNCLAIMED);
1406 }
1407
1408 if (dca->dca_intrstats) {
1409 KIOIP(dca)->intrs[KSTAT_INTR_HARD]++;
1410 }
1411 if (status & DMASTAT_MCR1INT) {
1412 DBG(dca, DINTR, "MCR1 interrupted");
1413 mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock));
1414 dca_schedule(dca, MCR1);
1415 dca_reclaim(dca, MCR1);
1416 mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock));
1417 }
1418
1419 if (status & DMASTAT_MCR2INT) {
1420 DBG(dca, DINTR, "MCR2 interrupted");
1421 mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock));
1422 dca_schedule(dca, MCR2);
1423 dca_reclaim(dca, MCR2);
1424 mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock));
1425 }
1426
1427 if (status & DMASTAT_ERRINT) {
1428 uint32_t erraddr;
1429 erraddr = GETCSR(dca, CSR_DMAEA);
1430 mutex_exit(&dca->dca_intrlock);
1431
1432 /*
1433 * bit 1 of the error address indicates failure during
1434 * read if set, during write otherwise.
1435 */
1436 dca_failure(dca, DDI_DEVICE_FAULT,
1437 DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR,
1438 "DMA master access error %s address 0x%x",
1439 erraddr & 0x1 ? "reading" : "writing", erraddr & ~1);
1440 return (DDI_INTR_CLAIMED);
1441 }
1442
1443 mutex_exit(&dca->dca_intrlock);
1444
1445 return (DDI_INTR_CLAIMED);
1446 }
1447
1448 /*
1449 * Reverse a string of bytes from s1 into s2. The reversal happens
1450 * from the tail of s1. If len1 < len2, then null bytes will be
1451 * padded to the end of s2. If len2 < len1, then (presumably null)
1452 * bytes will be dropped from the start of s1.
1453 *
1454 * The rationale here is that when s1 (source) is shorter, then we
1455 * are reversing from big-endian ordering, into device ordering, and
1456 * want to add some extra nulls to the tail (MSB) side of the device.
1457 *
1458 * Similarly, when s2 (dest) is shorter, then we are truncating what
1459 * are presumably null MSB bits from the device.
1460 *
1461 * There is an expectation when reversing from the device back into
1462 * big-endian, that the number of bytes to reverse and the target size
1463 * will match, and no truncation or padding occurs.
1464 */
1465 void
1466 dca_reverse(void *s1, void *s2, int len1, int len2)
1467 {
1468 caddr_t src, dst;
1469
1470 if (len1 == 0) {
1471 if (len2) {
1472 bzero(s2, len2);
1473 }
1474 return;
1475 }
1476 src = (caddr_t)s1 + len1 - 1;
1477 dst = s2;
1478 while ((src >= (caddr_t)s1) && (len2)) {
1479 *dst++ = *src--;
1480 len2--;
1481 }
1482 while (len2 > 0) {
1483 *dst++ = 0;
1484 len2--;
1485 }
1486 }
1487
1488 uint16_t
1489 dca_padfull(int num)
1490 {
1491 if (num <= 512) {
1492 return (BITS2BYTES(512));
1493 }
1494 if (num <= 768) {
1495 return (BITS2BYTES(768));
1496 }
1497 if (num <= 1024) {
1498 return (BITS2BYTES(1024));
1499 }
1500 if (num <= 1536) {
1501 return (BITS2BYTES(1536));
1502 }
1503 if (num <= 2048) {
1504 return (BITS2BYTES(2048));
1505 }
1506 return (0);
1507 }
1508
1509 uint16_t
1510 dca_padhalf(int num)
1511 {
1512 if (num <= 256) {
1513 return (BITS2BYTES(256));
1514 }
1515 if (num <= 384) {
1516 return (BITS2BYTES(384));
1517 }
1518 if (num <= 512) {
1519 return (BITS2BYTES(512));
1520 }
1521 if (num <= 768) {
1522 return (BITS2BYTES(768));
1523 }
1524 if (num <= 1024) {
1525 return (BITS2BYTES(1024));
1526 }
1527 return (0);
1528 }
1529
1530 dca_work_t *
1531 dca_newwork(dca_t *dca)
1532 {
1533 dca_work_t *workp;
1534 size_t size;
1535 ddi_dma_cookie_t c;
1536 unsigned nc;
1537 int rv;
1538
1539 workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP);
1540
1541 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1542 DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah);
1543 if (rv != 0) {
1544 dca_error(dca, "unable to alloc MCR DMA handle");
1545 dca_destroywork(workp);
1546 return (NULL);
1547 }
1548
1549 rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah,
1550 ROUNDUP(MCR_SIZE, dca->dca_pagesize),
1551 &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
1552 &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch);
1553 if (rv != 0) {
1554 dca_error(dca, "unable to alloc MCR DMA memory");
1555 dca_destroywork(workp);
1556 return (NULL);
1557 }
1558
1559 rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL,
1560 workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR,
1561 DDI_DMA_SLEEP, NULL, &c, &nc);
1562 if (rv != DDI_DMA_MAPPED) {
1563 dca_error(dca, "unable to map MCR DMA memory");
1564 dca_destroywork(workp);
1565 return (NULL);
1566 }
1567
1568 workp->dw_mcr_paddr = c.dmac_address;
1569 return (workp);
1570 }
1571
1572 void
1573 dca_destroywork(dca_work_t *workp)
1574 {
1575 if (workp->dw_mcr_paddr) {
1576 (void) ddi_dma_unbind_handle(workp->dw_mcr_dmah);
1577 }
1578 if (workp->dw_mcr_acch) {
1579 ddi_dma_mem_free(&workp->dw_mcr_acch);
1580 }
1581 if (workp->dw_mcr_dmah) {
1582 ddi_dma_free_handle(&workp->dw_mcr_dmah);
1583 }
1584 kmem_free(workp, sizeof (dca_work_t));
1585 }
1586
1587 dca_request_t *
1588 dca_newreq(dca_t *dca)
1589 {
1590 dca_request_t *reqp;
1591 size_t size;
1592 ddi_dma_cookie_t c;
1593 unsigned nc;
1594 int rv;
1595 int n_chain = 0;
1596
1597 size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH;
1598
1599 reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP);
1600
1601 reqp->dr_dca = dca;
1602
1603 /*
1604 * Setup the DMA region for the context and descriptors.
1605 */
1606 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP,
1607 NULL, &reqp->dr_ctx_dmah);
1608 if (rv != DDI_SUCCESS) {
1609 dca_error(dca, "failure allocating request DMA handle");
1610 dca_destroyreq(reqp);
1611 return (NULL);
1612 }
1613
1614 /* for driver hardening, allocate in whole pages */
1615 rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah,
1616 ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT,
1617 DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size,
1618 &reqp->dr_ctx_acch);
1619 if (rv != DDI_SUCCESS) {
1620 dca_error(dca, "unable to alloc request DMA memory");
1621 dca_destroyreq(reqp);
1622 return (NULL);
1623 }
1624
1625 rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL,
1626 reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE,
1627 DDI_DMA_SLEEP, 0, &c, &nc);
1628 if (rv != DDI_DMA_MAPPED) {
1629 dca_error(dca, "failed binding request DMA handle");
1630 dca_destroyreq(reqp);
1631 return (NULL);
1632 }
1633 reqp->dr_ctx_paddr = c.dmac_address;
1634
1635 reqp->dr_dma_size = size;
1636
1637 /*
1638 * Set up the dma for our scratch/shared buffers.
1639 */
1640 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1641 DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah);
1642 if (rv != DDI_SUCCESS) {
1643 dca_error(dca, "failure allocating ibuf DMA handle");
1644 dca_destroyreq(reqp);
1645 return (NULL);
1646 }
1647 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1648 DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah);
1649 if (rv != DDI_SUCCESS) {
1650 dca_error(dca, "failure allocating obuf DMA handle");
1651 dca_destroyreq(reqp);
1652 return (NULL);
1653 }
1654
1655 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1656 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah);
1657 if (rv != DDI_SUCCESS) {
1658 dca_error(dca, "failure allocating chain_in DMA handle");
1659 dca_destroyreq(reqp);
1660 return (NULL);
1661 }
1662
1663 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1664 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah);
1665 if (rv != DDI_SUCCESS) {
1666 dca_error(dca, "failure allocating chain_out DMA handle");
1667 dca_destroyreq(reqp);
1668 return (NULL);
1669 }
1670
1671 /*
1672 * for driver hardening, allocate in whole pages.
1673 */
1674 size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
1675 #if defined(i386) || defined(__i386)
1676 /*
1677 * Use kmem_alloc instead of ddi_dma_mem_alloc here since the latter
1678 * may fail on x86 platform if a physically contiguous memory chunk
1679 * cannot be found. From initial testing, we did not see performance
1680 * degradation as seen on Sparc.
1681 */
1682 if ((reqp->dr_ibuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) {
1683 dca_error(dca, "unable to alloc request ibuf memory");
1684 dca_destroyreq(reqp);
1685 return (NULL);
1686 }
1687 if ((reqp->dr_obuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) {
1688 dca_error(dca, "unable to alloc request obuf memory");
1689 dca_destroyreq(reqp);
1690 return (NULL);
1691 }
1692 #else
1693 /*
1694 * We could kmem_alloc for Sparc too. However, it gives worse
1695 * performance when transferring more than one page data. For example,
1696 * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system,
1697 * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for
1698 * the same throughput.
1699 */
1700 rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah,
1701 size, &dca_bufattr,
1702 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr,
1703 &size, &reqp->dr_ibuf_acch);
1704 if (rv != DDI_SUCCESS) {
1705 dca_error(dca, "unable to alloc request DMA memory");
1706 dca_destroyreq(reqp);
1707 return (NULL);
1708 }
1709
1710 rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah,
1711 size, &dca_bufattr,
1712 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr,
1713 &size, &reqp->dr_obuf_acch);
1714 if (rv != DDI_SUCCESS) {
1715 dca_error(dca, "unable to alloc request DMA memory");
1716 dca_destroyreq(reqp);
1717 return (NULL);
1718 }
1719 #endif
1720
1721 /* Skip the used portion in the context page */
1722 reqp->dr_offset = CTX_MAXLENGTH;
1723 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
1724 reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah,
1725 DDI_DMA_WRITE | DDI_DMA_STREAMING,
1726 &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) {
1727 (void) dca_destroyreq(reqp);
1728 return (NULL);
1729 }
1730 reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr;
1731 /* Skip the space used by the input buffer */
1732 reqp->dr_offset += DESC_SIZE * n_chain;
1733
1734 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
1735 reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah,
1736 DDI_DMA_READ | DDI_DMA_STREAMING,
1737 &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) {
1738 (void) dca_destroyreq(reqp);
1739 return (NULL);
1740 }
1741 reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr;
1742 /* Skip the space used by the output buffer */
1743 reqp->dr_offset += DESC_SIZE * n_chain;
1744
1745 DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d",
1746 reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH);
1747 return (reqp);
1748 }
1749
1750 void
1751 dca_destroyreq(dca_request_t *reqp)
1752 {
1753 #if defined(i386) || defined(__i386)
1754 dca_t *dca = reqp->dr_dca;
1755 size_t size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
1756 #endif
1757
1758 /*
1759 * Clean up DMA for the context structure.
1760 */
1761 if (reqp->dr_ctx_paddr) {
1762 (void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah);
1763 }
1764
1765 if (reqp->dr_ctx_acch) {
1766 ddi_dma_mem_free(&reqp->dr_ctx_acch);
1767 }
1768
1769 if (reqp->dr_ctx_dmah) {
1770 ddi_dma_free_handle(&reqp->dr_ctx_dmah);
1771 }
1772
1773 /*
1774 * Clean up DMA for the scratch buffer.
1775 */
1776 #if defined(i386) || defined(__i386)
1777 if (reqp->dr_ibuf_dmah) {
1778 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
1779 ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
1780 }
1781 if (reqp->dr_obuf_dmah) {
1782 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
1783 ddi_dma_free_handle(&reqp->dr_obuf_dmah);
1784 }
1785
1786 kmem_free(reqp->dr_ibuf_kaddr, size);
1787 kmem_free(reqp->dr_obuf_kaddr, size);
1788 #else
1789 if (reqp->dr_ibuf_paddr) {
1790 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
1791 }
1792 if (reqp->dr_obuf_paddr) {
1793 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
1794 }
1795
1796 if (reqp->dr_ibuf_acch) {
1797 ddi_dma_mem_free(&reqp->dr_ibuf_acch);
1798 }
1799 if (reqp->dr_obuf_acch) {
1800 ddi_dma_mem_free(&reqp->dr_obuf_acch);
1801 }
1802
1803 if (reqp->dr_ibuf_dmah) {
1804 ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
1805 }
1806 if (reqp->dr_obuf_dmah) {
1807 ddi_dma_free_handle(&reqp->dr_obuf_dmah);
1808 }
1809 #endif
1810 /*
1811 * These two DMA handles should have been unbinded in
1812 * dca_unbindchains() function
1813 */
1814 if (reqp->dr_chain_in_dmah) {
1815 ddi_dma_free_handle(&reqp->dr_chain_in_dmah);
1816 }
1817 if (reqp->dr_chain_out_dmah) {
1818 ddi_dma_free_handle(&reqp->dr_chain_out_dmah);
1819 }
1820
1821 kmem_free(reqp, sizeof (dca_request_t));
1822 }
1823
1824 dca_work_t *
1825 dca_getwork(dca_t *dca, int mcr)
1826 {
1827 dca_worklist_t *wlp = WORKLIST(dca, mcr);
1828 dca_work_t *workp;
1829
1830 mutex_enter(&wlp->dwl_freelock);
1831 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework);
1832 mutex_exit(&wlp->dwl_freelock);
1833 if (workp) {
1834 int nreqs;
1835 bzero(workp->dw_mcr_kaddr, 8);
1836
1837 /* clear out old requests */
1838 for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) {
1839 workp->dw_reqs[nreqs] = NULL;
1840 }
1841 }
1842 return (workp);
1843 }
1844
1845 void
1846 dca_freework(dca_work_t *workp)
1847 {
1848 mutex_enter(&workp->dw_wlp->dwl_freelock);
1849 dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp);
1850 mutex_exit(&workp->dw_wlp->dwl_freelock);
1851 }
1852
1853 dca_request_t *
1854 dca_getreq(dca_t *dca, int mcr, int tryhard)
1855 {
1856 dca_worklist_t *wlp = WORKLIST(dca, mcr);
1857 dca_request_t *reqp;
1858
1859 mutex_enter(&wlp->dwl_freereqslock);
1860 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs);
1861 mutex_exit(&wlp->dwl_freereqslock);
1862 if (reqp) {
1863 reqp->dr_flags = 0;
1864 reqp->dr_callback = NULL;
1865 } else if (tryhard) {
1866 /*
1867 * failed to get a free one, try an allocation, the hard way.
1868 * XXX: Kstat desired here.
1869 */
1870 if ((reqp = dca_newreq(dca)) != NULL) {
1871 reqp->dr_wlp = wlp;
1872 reqp->dr_dca = dca;
1873 reqp->dr_flags = 0;
1874 reqp->dr_callback = NULL;
1875 }
1876 }
1877 return (reqp);
1878 }
1879
1880 void
1881 dca_freereq(dca_request_t *reqp)
1882 {
1883 reqp->dr_kcf_req = NULL;
1884 if (!(reqp->dr_flags & DR_NOCACHE)) {
1885 mutex_enter(&reqp->dr_wlp->dwl_freereqslock);
1886 dca_enqueue(&reqp->dr_wlp->dwl_freereqs,
1887 (dca_listnode_t *)reqp);
1888 mutex_exit(&reqp->dr_wlp->dwl_freereqslock);
1889 }
1890 }
1891
1892 /*
1893 * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer
1894 * is mapped to a single physical address. On x86, a user buffer is mapped
1895 * to multiple physical addresses. These physical addresses are chained
1896 * using the method specified in Broadcom BCM5820 specification.
1897 */
1898 int
1899 dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt)
1900 {
1901 int rv;
1902 caddr_t kaddr;
1903 uint_t flags;
1904 int n_chain = 0;
1905
1906 if (reqp->dr_flags & DR_INPLACE) {
1907 flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
1908 } else {
1909 flags = DDI_DMA_WRITE | DDI_DMA_STREAMING;
1910 }
1911
1912 /* first the input */
1913 if (incnt) {
1914 if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) {
1915 DBG(NULL, DWARN, "unrecognised crypto data format");
1916 return (DDI_FAILURE);
1917 }
1918 if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset,
1919 kaddr, reqp->dr_chain_in_dmah, flags,
1920 &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) {
1921 (void) dca_unbindchains(reqp);
1922 return (rv);
1923 }
1924
1925 /*
1926 * The offset and length are altered by the calling routine
1927 * reqp->dr_in->cd_offset += incnt;
1928 * reqp->dr_in->cd_length -= incnt;
1929 */
1930 /* Save the first one in the chain for MCR */
1931 reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr;
1932 reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr;
1933 reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length;
1934 } else {
1935 reqp->dr_in_paddr = NULL;
1936 reqp->dr_in_next = 0;
1937 reqp->dr_in_len = 0;
1938 }
1939
1940 if (reqp->dr_flags & DR_INPLACE) {
1941 reqp->dr_out_paddr = reqp->dr_in_paddr;
1942 reqp->dr_out_len = reqp->dr_in_len;
1943 reqp->dr_out_next = reqp->dr_in_next;
1944 return (DDI_SUCCESS);
1945 }
1946
1947 /* then the output */
1948 if (outcnt) {
1949 flags = DDI_DMA_READ | DDI_DMA_STREAMING;
1950 if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) {
1951 DBG(NULL, DWARN, "unrecognised crypto data format");
1952 (void) dca_unbindchains(reqp);
1953 return (DDI_FAILURE);
1954 }
1955 rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset +
1956 n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah,
1957 flags, &reqp->dr_chain_out_head, &n_chain);
1958 if (rv != DDI_SUCCESS) {
1959 (void) dca_unbindchains(reqp);
1960 return (DDI_FAILURE);
1961 }
1962
1963 /* Save the first one in the chain for MCR */
1964 reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr;
1965 reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr;
1966 reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length;
1967 } else {
1968 reqp->dr_out_paddr = NULL;
1969 reqp->dr_out_next = 0;
1970 reqp->dr_out_len = 0;
1971 }
1972
1973 return (DDI_SUCCESS);
1974 }
1975
1976 /*
1977 * Unbind the user buffers from the DMA handles.
1978 */
1979 int
1980 dca_unbindchains(dca_request_t *reqp)
1981 {
1982 int rv = DDI_SUCCESS;
1983 int rv1 = DDI_SUCCESS;
1984
1985 /* Clear the input chain */
1986 if (reqp->dr_chain_in_head.dc_buffer_paddr != NULL) {
1987 (void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah);
1988 reqp->dr_chain_in_head.dc_buffer_paddr = 0;
1989 }
1990
1991 if (reqp->dr_flags & DR_INPLACE) {
1992 return (rv);
1993 }
1994
1995 /* Clear the output chain */
1996 if (reqp->dr_chain_out_head.dc_buffer_paddr != NULL) {
1997 (void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah);
1998 reqp->dr_chain_out_head.dc_buffer_paddr = 0;
1999 }
2000
2001 return ((rv != DDI_SUCCESS)? rv : rv1);
2002 }
2003
2004 /*
2005 * Build either input chain or output chain. It is single-item chain for Sparc,
2006 * and possible mutiple-item chain for x86.
2007 */
2008 static int
2009 dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
2010 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
2011 dca_chain_t *head, int *n_chain)
2012 {
2013 ddi_dma_cookie_t c;
2014 uint_t nc;
2015 int rv;
2016 caddr_t chain_kaddr_pre;
2017 caddr_t chain_kaddr;
2018 uint32_t chain_paddr;
2019 int i;
2020
2021 /* Advance past the context structure to the starting address */
2022 chain_paddr = reqp->dr_ctx_paddr + dr_offset;
2023 chain_kaddr = reqp->dr_ctx_kaddr + dr_offset;
2024
2025 /*
2026 * Bind the kernel address to the DMA handle. On x86, the actual
2027 * buffer is mapped into multiple physical addresses. On Sparc,
2028 * the actual buffer is mapped into a single address.
2029 */
2030 rv = ddi_dma_addr_bind_handle(handle,
2031 NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc);
2032 if (rv != DDI_DMA_MAPPED) {
2033 return (DDI_FAILURE);
2034 }
2035
2036 (void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV);
2037 if ((rv = dca_check_dma_handle(reqp->dr_dca, handle,
2038 DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) {
2039 reqp->destroy = TRUE;
2040 return (rv);
2041 }
2042
2043 *n_chain = nc;
2044
2045 /* Setup the data buffer chain for DMA transfer */
2046 chain_kaddr_pre = NULL;
2047 head->dc_buffer_paddr = 0;
2048 head->dc_next_paddr = 0;
2049 head->dc_buffer_length = 0;
2050 for (i = 0; i < nc; i++) {
2051 /* PIO */
2052 PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address);
2053 PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0);
2054 PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size);
2055
2056 /* Remember the head of the chain */
2057 if (head->dc_buffer_paddr == 0) {
2058 head->dc_buffer_paddr = c.dmac_address;
2059 head->dc_buffer_length = c.dmac_size;
2060 }
2061
2062 /* Link to the previous one if one exists */
2063 if (chain_kaddr_pre) {
2064 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT,
2065 chain_paddr);
2066 if (head->dc_next_paddr == 0)
2067 head->dc_next_paddr = chain_paddr;
2068 }
2069 chain_kaddr_pre = chain_kaddr;
2070
2071 /* Maintain pointers */
2072 chain_paddr += DESC_SIZE;
2073 chain_kaddr += DESC_SIZE;
2074
2075 /* Retrieve the next cookie if there is one */
2076 if (i < nc-1)
2077 ddi_dma_nextcookie(handle, &c);
2078 }
2079
2080 /* Set the next pointer in the last entry to NULL */
2081 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0);
2082
2083 return (DDI_SUCCESS);
2084 }
2085
2086 /*
2087 * Schedule some work.
2088 */
2089 int
2090 dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched)
2091 {
2092 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2093
2094 mutex_enter(&wlp->dwl_lock);
2095
2096 DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p",
2097 reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr,
2098 reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr);
2099 DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x",
2100 reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr);
2101 /* sync out the entire context and descriptor chains */
2102 (void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV);
2103 if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah,
2104 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2105 reqp->destroy = TRUE;
2106 mutex_exit(&wlp->dwl_lock);
2107 return (CRYPTO_DEVICE_ERROR);
2108 }
2109
2110 dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp);
2111 wlp->dwl_count++;
2112 wlp->dwl_lastsubmit = ddi_get_lbolt();
2113 reqp->dr_wlp = wlp;
2114
2115 if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) {
2116 /* we are fully loaded now, let kCF know */
2117
2118 wlp->dwl_flowctl++;
2119 wlp->dwl_busy = 1;
2120
2121 crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY);
2122 }
2123
2124 if (dosched) {
2125 #ifdef SCHEDDELAY
2126 /* possibly wait for more work to arrive */
2127 if (wlp->dwl_count >= wlp->dwl_reqspermcr) {
2128 dca_schedule(dca, mcr);
2129 } else if (!wlp->dwl_schedtid) {
2130 /* wait 1 msec for more work before doing it */
2131 wlp->dwl_schedtid = timeout(dca_schedtimeout,
2132 (void *)wlp, drv_usectohz(MSEC));
2133 }
2134 #else
2135 dca_schedule(dca, mcr);
2136 #endif
2137 }
2138 mutex_exit(&wlp->dwl_lock);
2139
2140 return (CRYPTO_QUEUED);
2141 }
2142
2143 void
2144 dca_schedule(dca_t *dca, int mcr)
2145 {
2146 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2147 int csr;
2148 int full;
2149 uint32_t status;
2150
2151 ASSERT(mutex_owned(&wlp->dwl_lock));
2152 /*
2153 * If the card is draining or has an outstanding failure,
2154 * don't schedule any more work on it right now
2155 */
2156 if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) {
2157 return;
2158 }
2159
2160 if (mcr == MCR2) {
2161 csr = CSR_MCR2;
2162 full = DMASTAT_MCR2FULL;
2163 } else {
2164 csr = CSR_MCR1;
2165 full = DMASTAT_MCR1FULL;
2166 }
2167
2168 for (;;) {
2169 dca_work_t *workp;
2170 uint32_t offset;
2171 int nreqs;
2172
2173 status = GETCSR(dca, CSR_DMASTAT);
2174 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
2175 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
2176 return;
2177
2178 if ((status & full) != 0)
2179 break;
2180
2181 #ifdef SCHEDDELAY
2182 /* if there isn't enough to do, don't bother now */
2183 if ((wlp->dwl_count < wlp->dwl_reqspermcr) &&
2184 (ddi_get_lbolt() < (wlp->dwl_lastsubmit +
2185 drv_usectohz(MSEC)))) {
2186 /* wait a bit longer... */
2187 if (wlp->dwl_schedtid == 0) {
2188 wlp->dwl_schedtid = timeout(dca_schedtimeout,
2189 (void *)wlp, drv_usectohz(MSEC));
2190 }
2191 return;
2192 }
2193 #endif
2194
2195 /* grab a work structure */
2196 workp = dca_getwork(dca, mcr);
2197
2198 if (workp == NULL) {
2199 /*
2200 * There must be work ready to be reclaimed,
2201 * in this case, since the chip can only hold
2202 * less work outstanding than there are total.
2203 */
2204 dca_reclaim(dca, mcr);
2205 continue;
2206 }
2207
2208 nreqs = 0;
2209 offset = MCR_CTXADDR;
2210
2211 while (nreqs < wlp->dwl_reqspermcr) {
2212 dca_request_t *reqp;
2213
2214 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq);
2215 if (reqp == NULL) {
2216 /* nothing left to process */
2217 break;
2218 }
2219 /*
2220 * Update flow control.
2221 */
2222 wlp->dwl_count--;
2223 if ((wlp->dwl_count == wlp->dwl_lowater) &&
2224 (wlp->dwl_busy)) {
2225 wlp->dwl_busy = 0;
2226 crypto_prov_notify(wlp->dwl_prov,
2227 CRYPTO_PROVIDER_READY);
2228 }
2229
2230 /*
2231 * Context address.
2232 */
2233 PUTMCR32(workp, offset, reqp->dr_ctx_paddr);
2234 offset += 4;
2235
2236 /*
2237 * Input chain.
2238 */
2239 /* input buffer address */
2240 PUTMCR32(workp, offset, reqp->dr_in_paddr);
2241 offset += 4;
2242 /* next input buffer entry */
2243 PUTMCR32(workp, offset, reqp->dr_in_next);
2244 offset += 4;
2245 /* input buffer length */
2246 PUTMCR16(workp, offset, reqp->dr_in_len);
2247 offset += 2;
2248 /* zero the reserved field */
2249 PUTMCR16(workp, offset, 0);
2250 offset += 2;
2251
2252 /*
2253 * Overall length.
2254 */
2255 /* reserved field */
2256 PUTMCR16(workp, offset, 0);
2257 offset += 2;
2258 /* total packet length */
2259 PUTMCR16(workp, offset, reqp->dr_pkt_length);
2260 offset += 2;
2261
2262 /*
2263 * Output chain.
2264 */
2265 /* output buffer address */
2266 PUTMCR32(workp, offset, reqp->dr_out_paddr);
2267 offset += 4;
2268 /* next output buffer entry */
2269 PUTMCR32(workp, offset, reqp->dr_out_next);
2270 offset += 4;
2271 /* output buffer length */
2272 PUTMCR16(workp, offset, reqp->dr_out_len);
2273 offset += 2;
2274 /* zero the reserved field */
2275 PUTMCR16(workp, offset, 0);
2276 offset += 2;
2277
2278 /*
2279 * Note submission.
2280 */
2281 workp->dw_reqs[nreqs] = reqp;
2282 nreqs++;
2283 }
2284
2285 if (nreqs == 0) {
2286 /* nothing in the queue! */
2287 dca_freework(workp);
2288 return;
2289 }
2290
2291 wlp->dwl_submit++;
2292
2293 PUTMCR16(workp, MCR_FLAGS, 0);
2294 PUTMCR16(workp, MCR_COUNT, nreqs);
2295
2296 DBG(dca, DCHATTY,
2297 "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d",
2298 workp->dw_mcr_paddr, workp->dw_mcr_kaddr,
2299 nreqs, mcr);
2300
2301 workp->dw_lbolt = ddi_get_lbolt();
2302 /* Make sure MCR is synced out to device. */
2303 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0,
2304 DDI_DMA_SYNC_FORDEV);
2305 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
2306 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2307 dca_destroywork(workp);
2308 return;
2309 }
2310
2311 PUTCSR(dca, csr, workp->dw_mcr_paddr);
2312 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
2313 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2314 dca_destroywork(workp);
2315 return;
2316 } else {
2317 dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp);
2318 }
2319
2320 DBG(dca, DCHATTY, "posted");
2321 }
2322 }
2323
2324 /*
2325 * Reclaim completed work, called in interrupt context.
2326 */
2327 void
2328 dca_reclaim(dca_t *dca, int mcr)
2329 {
2330 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2331 dca_work_t *workp;
2332 ushort_t flags;
2333 int nreclaimed = 0;
2334 int i;
2335
2336 DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr);
2337 ASSERT(mutex_owned(&wlp->dwl_lock));
2338 /*
2339 * For each MCR in the submitted (runq), we check to see if
2340 * it has been processed. If so, then we note each individual
2341 * job in the MCR, and and do the completion processing for
2342 * each of such job.
2343 */
2344 for (;;) {
2345
2346 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
2347 if (workp == NULL) {
2348 break;
2349 }
2350
2351 /* only sync the MCR flags, since that's all we need */
2352 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4,
2353 DDI_DMA_SYNC_FORKERNEL);
2354 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
2355 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2356 dca_rmqueue((dca_listnode_t *)workp);
2357 dca_destroywork(workp);
2358 return;
2359 }
2360
2361 flags = GETMCR16(workp, MCR_FLAGS);
2362 if ((flags & MCRFLAG_FINISHED) == 0) {
2363 /* chip is still working on it */
2364 DBG(dca, DRECLAIM,
2365 "chip still working on it (MCR%d)", mcr);
2366 break;
2367 }
2368
2369 /* its really for us, so remove it from the queue */
2370 dca_rmqueue((dca_listnode_t *)workp);
2371
2372 /* if we were draining, signal on the cv */
2373 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
2374 cv_signal(&wlp->dwl_cv);
2375 }
2376
2377 /* update statistics, done under the lock */
2378 for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2379 dca_request_t *reqp = workp->dw_reqs[i];
2380 if (reqp == NULL) {
2381 continue;
2382 }
2383 if (reqp->dr_byte_stat >= 0) {
2384 dca->dca_stats[reqp->dr_byte_stat] +=
2385 reqp->dr_pkt_length;
2386 }
2387 if (reqp->dr_job_stat >= 0) {
2388 dca->dca_stats[reqp->dr_job_stat]++;
2389 }
2390 }
2391 mutex_exit(&wlp->dwl_lock);
2392
2393 for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2394 dca_request_t *reqp = workp->dw_reqs[i];
2395
2396 if (reqp == NULL) {
2397 continue;
2398 }
2399
2400 /* Do the callback. */
2401 workp->dw_reqs[i] = NULL;
2402 dca_done(reqp, CRYPTO_SUCCESS);
2403
2404 nreclaimed++;
2405 }
2406
2407 /* now we can release the work */
2408 dca_freework(workp);
2409
2410 mutex_enter(&wlp->dwl_lock);
2411 }
2412 DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed);
2413 }
2414
2415 int
2416 dca_length(crypto_data_t *cdata)
2417 {
2418 return (cdata->cd_length);
2419 }
2420
2421 /*
2422 * This is the callback function called from the interrupt when a kCF job
2423 * completes. It does some driver-specific things, and then calls the
2424 * kCF-provided callback. Finally, it cleans up the state for the work
2425 * request and drops the reference count to allow for DR.
2426 */
2427 void
2428 dca_done(dca_request_t *reqp, int err)
2429 {
2430 uint64_t ena = 0;
2431
2432 /* unbind any chains we were using */
2433 if (dca_unbindchains(reqp) != DDI_SUCCESS) {
2434 /* DMA failure */
2435 ena = dca_ena(ena);
2436 dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT,
2437 DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR,
2438 "fault on buffer DMA handle");
2439 if (err == CRYPTO_SUCCESS) {
2440 err = CRYPTO_DEVICE_ERROR;
2441 }
2442 }
2443
2444 if (reqp->dr_callback != NULL) {
2445 reqp->dr_callback(reqp, err);
2446 } else {
2447 dca_freereq(reqp);
2448 }
2449 }
2450
2451 /*
2452 * Call this when a failure is detected. It will reset the chip,
2453 * log a message, alert kCF, and mark jobs in the runq as failed.
2454 */
2455 /* ARGSUSED */
2456 void
2457 dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index,
2458 uint64_t ena, int errno, char *mess, ...)
2459 {
2460 va_list ap;
2461 char buf[256];
2462 int mcr;
2463 char *eclass;
2464 int have_mutex;
2465
2466 va_start(ap, mess);
2467 (void) vsprintf(buf, mess, ap);
2468 va_end(ap);
2469
2470 eclass = dca_fma_eclass_string(dca->dca_model, index);
2471
2472 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) &&
2473 index != DCA_FM_ECLASS_NONE) {
2474 ddi_fm_ereport_post(dca->dca_dip, eclass, ena,
2475 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
2476 FM_EREPORT_VERS0, NULL);
2477
2478 /* Report the impact of the failure to the DDI. */
2479 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST);
2480 } else {
2481 /* Just log the error string to the message log */
2482 dca_error(dca, buf);
2483 }
2484
2485 /*
2486 * Indicate a failure (keeps schedule from running).
2487 */
2488 dca->dca_flags |= DCA_FAILED;
2489
2490 /*
2491 * Reset the chip. This should also have as a side effect, the
2492 * disabling of all interrupts from the device.
2493 */
2494 (void) dca_reset(dca, 1);
2495
2496 /*
2497 * Report the failure to kCF.
2498 */
2499 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2500 if (WORKLIST(dca, mcr)->dwl_prov) {
2501 crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov,
2502 CRYPTO_PROVIDER_FAILED);
2503 }
2504 }
2505
2506 /*
2507 * Return jobs not sent to hardware back to kCF.
2508 */
2509 dca_rejectjobs(dca);
2510
2511 /*
2512 * From this point on, no new work should be arriving, and the
2513 * chip should not be doing any active DMA.
2514 */
2515
2516 /*
2517 * Now find all the work submitted to the device and fail
2518 * them.
2519 */
2520 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2521 dca_worklist_t *wlp;
2522 int i;
2523
2524 wlp = WORKLIST(dca, mcr);
2525
2526 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
2527 continue;
2528 }
2529 for (;;) {
2530 dca_work_t *workp;
2531
2532 have_mutex = mutex_tryenter(&wlp->dwl_lock);
2533 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq);
2534 if (workp == NULL) {
2535 if (have_mutex)
2536 mutex_exit(&wlp->dwl_lock);
2537 break;
2538 }
2539 mutex_exit(&wlp->dwl_lock);
2540
2541 /*
2542 * Free up requests
2543 */
2544 for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2545 dca_request_t *reqp = workp->dw_reqs[i];
2546 if (reqp) {
2547 dca_done(reqp, errno);
2548 workp->dw_reqs[i] = NULL;
2549 }
2550 }
2551
2552 mutex_enter(&wlp->dwl_lock);
2553 /*
2554 * If waiting to drain, signal on the waiter.
2555 */
2556 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
2557 cv_signal(&wlp->dwl_cv);
2558 }
2559
2560 /*
2561 * Return the work and request structures to
2562 * the free pool.
2563 */
2564 dca_freework(workp);
2565 if (have_mutex)
2566 mutex_exit(&wlp->dwl_lock);
2567 }
2568 }
2569
2570 }
2571
2572 #ifdef SCHEDDELAY
2573 /*
2574 * Reschedule worklist as needed.
2575 */
2576 void
2577 dca_schedtimeout(void *arg)
2578 {
2579 dca_worklist_t *wlp = (dca_worklist_t *)arg;
2580 mutex_enter(&wlp->dwl_lock);
2581 wlp->dwl_schedtid = 0;
2582 dca_schedule(wlp->dwl_dca, wlp->dwl_mcr);
2583 mutex_exit(&wlp->dwl_lock);
2584 }
2585 #endif
2586
2587 /*
2588 * Check for stalled jobs.
2589 */
2590 void
2591 dca_jobtimeout(void *arg)
2592 {
2593 int mcr;
2594 dca_t *dca = (dca_t *)arg;
2595 int hung = 0;
2596
2597 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2598 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2599 dca_work_t *workp;
2600 clock_t when;
2601
2602 mutex_enter(&wlp->dwl_lock);
2603 when = ddi_get_lbolt();
2604
2605 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
2606 if (workp == NULL) {
2607 /* nothing sitting in the queue */
2608 mutex_exit(&wlp->dwl_lock);
2609 continue;
2610 }
2611
2612 if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) {
2613 /* request has been queued for less than STALETIME */
2614 mutex_exit(&wlp->dwl_lock);
2615 continue;
2616 }
2617
2618 /* job has been sitting around for over 1 second, badness */
2619 DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp,
2620 mcr);
2621
2622 /* put it back in the queue, until we reset the chip */
2623 hung++;
2624 mutex_exit(&wlp->dwl_lock);
2625 }
2626
2627 if (hung) {
2628 dca_failure(dca, DDI_DEVICE_FAULT,
2629 DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR,
2630 "timeout processing job.)");
2631 }
2632
2633 /* reschedule ourself */
2634 mutex_enter(&dca->dca_intrlock);
2635 if (dca->dca_jobtid == 0) {
2636 /* timeout has been canceled, prior to DR */
2637 mutex_exit(&dca->dca_intrlock);
2638 return;
2639 }
2640
2641 /* check again in 1 second */
2642 dca->dca_jobtid = timeout(dca_jobtimeout, arg,
2643 drv_usectohz(SECOND));
2644 mutex_exit(&dca->dca_intrlock);
2645 }
2646
2647 /*
2648 * This returns all jobs back to kCF. It assumes that processing
2649 * on the worklist has halted.
2650 */
2651 void
2652 dca_rejectjobs(dca_t *dca)
2653 {
2654 int mcr;
2655 int have_mutex;
2656 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2657 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2658 dca_request_t *reqp;
2659
2660 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
2661 continue;
2662 }
2663 have_mutex = mutex_tryenter(&wlp->dwl_lock);
2664 for (;;) {
2665 reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq);
2666 if (reqp == NULL) {
2667 break;
2668 }
2669 /* update flow control */
2670 wlp->dwl_count--;
2671 if ((wlp->dwl_count == wlp->dwl_lowater) &&
2672 (wlp->dwl_busy)) {
2673 wlp->dwl_busy = 0;
2674 crypto_prov_notify(wlp->dwl_prov,
2675 CRYPTO_PROVIDER_READY);
2676 }
2677 mutex_exit(&wlp->dwl_lock);
2678
2679 (void) dca_unbindchains(reqp);
2680 reqp->dr_callback(reqp, EAGAIN);
2681 mutex_enter(&wlp->dwl_lock);
2682 }
2683 if (have_mutex)
2684 mutex_exit(&wlp->dwl_lock);
2685 }
2686 }
2687
2688 int
2689 dca_drain(dca_t *dca)
2690 {
2691 int mcr;
2692 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2693 #ifdef SCHEDDELAY
2694 timeout_id_t tid;
2695 #endif
2696 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2697
2698 mutex_enter(&wlp->dwl_lock);
2699 wlp->dwl_drain = 1;
2700
2701 /* give it up to a second to drain from the chip */
2702 if (!QEMPTY(&wlp->dwl_runq)) {
2703 (void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock,
2704 drv_usectohz(STALETIME), TR_CLOCK_TICK);
2705
2706 if (!QEMPTY(&wlp->dwl_runq)) {
2707 dca_error(dca, "unable to drain device");
2708 mutex_exit(&wlp->dwl_lock);
2709 dca_undrain(dca);
2710 return (EBUSY);
2711 }
2712 }
2713
2714 #ifdef SCHEDDELAY
2715 tid = wlp->dwl_schedtid;
2716 mutex_exit(&wlp->dwl_lock);
2717
2718 /*
2719 * untimeout outside the lock -- this is safe because we
2720 * have set the drain flag, so dca_schedule() will not
2721 * reschedule another timeout
2722 */
2723 if (tid) {
2724 untimeout(tid);
2725 }
2726 #else
2727 mutex_exit(&wlp->dwl_lock);
2728 #endif
2729 }
2730 return (0);
2731 }
2732
2733 void
2734 dca_undrain(dca_t *dca)
2735 {
2736 int mcr;
2737
2738 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2739 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2740 mutex_enter(&wlp->dwl_lock);
2741 wlp->dwl_drain = 0;
2742 dca_schedule(dca, mcr);
2743 mutex_exit(&wlp->dwl_lock);
2744 }
2745 }
2746
2747 /*
2748 * Duplicate the crypto_data_t structure, but point to the original
2749 * buffers.
2750 */
2751 int
2752 dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput)
2753 {
2754 ninput->cd_format = input->cd_format;
2755 ninput->cd_offset = input->cd_offset;
2756 ninput->cd_length = input->cd_length;
2757 ninput->cd_miscdata = input->cd_miscdata;
2758
2759 switch (input->cd_format) {
2760 case CRYPTO_DATA_RAW:
2761 ninput->cd_raw.iov_base = input->cd_raw.iov_base;
2762 ninput->cd_raw.iov_len = input->cd_raw.iov_len;
2763 break;
2764
2765 case CRYPTO_DATA_UIO:
2766 ninput->cd_uio = input->cd_uio;
2767 break;
2768
2769 case CRYPTO_DATA_MBLK:
2770 ninput->cd_mp = input->cd_mp;
2771 break;
2772
2773 default:
2774 DBG(NULL, DWARN,
2775 "dca_dupcrypto: unrecognised crypto data format");
2776 return (CRYPTO_FAILED);
2777 }
2778
2779 return (CRYPTO_SUCCESS);
2780 }
2781
2782 /*
2783 * Performs validation checks on the input and output data structures.
2784 */
2785 int
2786 dca_verifyio(crypto_data_t *input, crypto_data_t *output)
2787 {
2788 int rv = CRYPTO_SUCCESS;
2789
2790 switch (input->cd_format) {
2791 case CRYPTO_DATA_RAW:
2792 break;
2793
2794 case CRYPTO_DATA_UIO:
2795 /* we support only kernel buffer */
2796 if (input->cd_uio->uio_segflg != UIO_SYSSPACE) {
2797 DBG(NULL, DWARN, "non kernel input uio buffer");
2798 rv = CRYPTO_ARGUMENTS_BAD;
2799 }
2800 break;
2801
2802 case CRYPTO_DATA_MBLK:
2803 break;
2804
2805 default:
2806 DBG(NULL, DWARN, "unrecognised input crypto data format");
2807 rv = CRYPTO_ARGUMENTS_BAD;
2808 }
2809
2810 switch (output->cd_format) {
2811 case CRYPTO_DATA_RAW:
2812 break;
2813
2814 case CRYPTO_DATA_UIO:
2815 /* we support only kernel buffer */
2816 if (output->cd_uio->uio_segflg != UIO_SYSSPACE) {
2817 DBG(NULL, DWARN, "non kernel output uio buffer");
2818 rv = CRYPTO_ARGUMENTS_BAD;
2819 }
2820 break;
2821
2822 case CRYPTO_DATA_MBLK:
2823 break;
2824
2825 default:
2826 DBG(NULL, DWARN, "unrecognised output crypto data format");
2827 rv = CRYPTO_ARGUMENTS_BAD;
2828 }
2829
2830 return (rv);
2831 }
2832
2833 /*
2834 * data: source crypto_data_t struct
2835 * off: offset into the source before commencing copy
2836 * count: the amount of data to copy
2837 * dest: destination buffer
2838 */
2839 int
2840 dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest)
2841 {
2842 int rv = CRYPTO_SUCCESS;
2843 uio_t *uiop;
2844 uint_t vec_idx;
2845 size_t cur_len;
2846 mblk_t *mp;
2847
2848 if (count == 0) {
2849 /* We don't want anything so we're done. */
2850 return (rv);
2851 }
2852
2853 /*
2854 * Sanity check that we haven't specified a length greater than the
2855 * offset adjusted size of the buffer.
2856 */
2857 if (count > (data->cd_length - off)) {
2858 return (CRYPTO_DATA_LEN_RANGE);
2859 }
2860
2861 /* Add the internal crypto_data offset to the requested offset. */
2862 off += data->cd_offset;
2863
2864 switch (data->cd_format) {
2865 case CRYPTO_DATA_RAW:
2866 bcopy(data->cd_raw.iov_base + off, dest, count);
2867 break;
2868
2869 case CRYPTO_DATA_UIO:
2870 /*
2871 * Jump to the first iovec containing data to be
2872 * processed.
2873 */
2874 uiop = data->cd_uio;
2875 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
2876 off >= uiop->uio_iov[vec_idx].iov_len;
2877 off -= uiop->uio_iov[vec_idx++].iov_len)
2878 ;
2879 if (vec_idx == uiop->uio_iovcnt) {
2880 /*
2881 * The caller specified an offset that is larger than
2882 * the total size of the buffers it provided.
2883 */
2884 return (CRYPTO_DATA_LEN_RANGE);
2885 }
2886
2887 /*
2888 * Now process the iovecs.
2889 */
2890 while (vec_idx < uiop->uio_iovcnt && count > 0) {
2891 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
2892 off, count);
2893 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
2894 cur_len);
2895 count -= cur_len;
2896 dest += cur_len;
2897 vec_idx++;
2898 off = 0;
2899 }
2900
2901 if (vec_idx == uiop->uio_iovcnt && count > 0) {
2902 /*
2903 * The end of the specified iovec's was reached but
2904 * the length requested could not be processed
2905 * (requested to digest more data than it provided).
2906 */
2907 return (CRYPTO_DATA_LEN_RANGE);
2908 }
2909 break;
2910
2911 case CRYPTO_DATA_MBLK:
2912 /*
2913 * Jump to the first mblk_t containing data to be processed.
2914 */
2915 for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp);
2916 off -= MBLKL(mp), mp = mp->b_cont)
2917 ;
2918 if (mp == NULL) {
2919 /*
2920 * The caller specified an offset that is larger than
2921 * the total size of the buffers it provided.
2922 */
2923 return (CRYPTO_DATA_LEN_RANGE);
2924 }
2925
2926 /*
2927 * Now do the processing on the mblk chain.
2928 */
2929 while (mp != NULL && count > 0) {
2930 cur_len = min(MBLKL(mp) - off, count);
2931 bcopy((char *)(mp->b_rptr + off), dest, cur_len);
2932 count -= cur_len;
2933 dest += cur_len;
2934 mp = mp->b_cont;
2935 off = 0;
2936 }
2937
2938 if (mp == NULL && count > 0) {
2939 /*
2940 * The end of the mblk was reached but the length
2941 * requested could not be processed, (requested to
2942 * digest more data than it provided).
2943 */
2944 return (CRYPTO_DATA_LEN_RANGE);
2945 }
2946 break;
2947
2948 default:
2949 DBG(NULL, DWARN, "unrecognised crypto data format");
2950 rv = CRYPTO_ARGUMENTS_BAD;
2951 }
2952 return (rv);
2953 }
2954
2955
2956 /*
2957 * Performs the input, output or hard scatter/gather checks on the specified
2958 * crypto_data_t struct. Returns true if the data is scatter/gather in nature
2959 * ie fails the test.
2960 */
2961 int
2962 dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val)
2963 {
2964 uio_t *uiop;
2965 mblk_t *mp;
2966 int rv = FALSE;
2967
2968 switch (val) {
2969 case DCA_SG_CONTIG:
2970 /*
2971 * Check for a contiguous data buffer.
2972 */
2973 switch (data->cd_format) {
2974 case CRYPTO_DATA_RAW:
2975 /* Contiguous in nature */
2976 break;
2977
2978 case CRYPTO_DATA_UIO:
2979 if (data->cd_uio->uio_iovcnt > 1)
2980 rv = TRUE;
2981 break;
2982
2983 case CRYPTO_DATA_MBLK:
2984 mp = data->cd_mp;
2985 if (mp->b_cont != NULL)
2986 rv = TRUE;
2987 break;
2988
2989 default:
2990 DBG(NULL, DWARN, "unrecognised crypto data format");
2991 }
2992 break;
2993
2994 case DCA_SG_WALIGN:
2995 /*
2996 * Check for a contiguous data buffer that is 32-bit word
2997 * aligned and is of word multiples in size.
2998 */
2999 switch (data->cd_format) {
3000 case CRYPTO_DATA_RAW:
3001 if ((data->cd_raw.iov_len % sizeof (uint32_t)) ||
3002 ((uintptr_t)data->cd_raw.iov_base %
3003 sizeof (uint32_t))) {
3004 rv = TRUE;
3005 }
3006 break;
3007
3008 case CRYPTO_DATA_UIO:
3009 uiop = data->cd_uio;
3010 if (uiop->uio_iovcnt > 1) {
3011 return (TRUE);
3012 }
3013 /* So there is only one iovec */
3014 if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) ||
3015 ((uintptr_t)uiop->uio_iov[0].iov_base %
3016 sizeof (uint32_t))) {
3017 rv = TRUE;
3018 }
3019 break;
3020
3021 case CRYPTO_DATA_MBLK:
3022 mp = data->cd_mp;
3023 if (mp->b_cont != NULL) {
3024 return (TRUE);
3025 }
3026 /* So there is only one mblk in the chain */
3027 if ((MBLKL(mp) % sizeof (uint32_t)) ||
3028 ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) {
3029 rv = TRUE;
3030 }
3031 break;
3032
3033 default:
3034 DBG(NULL, DWARN, "unrecognised crypto data format");
3035 }
3036 break;
3037
3038 case DCA_SG_PALIGN:
3039 /*
3040 * Check that the data buffer is page aligned and is of
3041 * page multiples in size.
3042 */
3043 switch (data->cd_format) {
3044 case CRYPTO_DATA_RAW:
3045 if ((data->cd_length % dca->dca_pagesize) ||
3046 ((uintptr_t)data->cd_raw.iov_base %
3047 dca->dca_pagesize)) {
3048 rv = TRUE;
3049 }
3050 break;
3051
3052 case CRYPTO_DATA_UIO:
3053 uiop = data->cd_uio;
3054 if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) ||
3055 ((uintptr_t)uiop->uio_iov[0].iov_base %
3056 dca->dca_pagesize)) {
3057 rv = TRUE;
3058 }
3059 break;
3060
3061 case CRYPTO_DATA_MBLK:
3062 mp = data->cd_mp;
3063 if ((MBLKL(mp) % dca->dca_pagesize) ||
3064 ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) {
3065 rv = TRUE;
3066 }
3067 break;
3068
3069 default:
3070 DBG(NULL, DWARN, "unrecognised crypto data format");
3071 }
3072 break;
3073
3074 default:
3075 DBG(NULL, DWARN, "unrecognised scatter/gather param type");
3076 }
3077
3078 return (rv);
3079 }
3080
3081 /*
3082 * Increments the cd_offset and decrements the cd_length as the data is
3083 * gathered from the crypto_data_t struct.
3084 * The data is reverse-copied into the dest buffer if the flag is true.
3085 */
3086 int
3087 dca_gather(crypto_data_t *in, char *dest, int count, int reverse)
3088 {
3089 int rv = CRYPTO_SUCCESS;
3090 uint_t vec_idx;
3091 uio_t *uiop;
3092 off_t off = in->cd_offset;
3093 size_t cur_len;
3094 mblk_t *mp;
3095
3096 switch (in->cd_format) {
3097 case CRYPTO_DATA_RAW:
3098 if (count > in->cd_length) {
3099 /*
3100 * The caller specified a length greater than the
3101 * size of the buffer.
3102 */
3103 return (CRYPTO_DATA_LEN_RANGE);
3104 }
3105 if (reverse)
3106 dca_reverse(in->cd_raw.iov_base + off, dest, count,
3107 count);
3108 else
3109 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
3110 in->cd_offset += count;
3111 in->cd_length -= count;
3112 break;
3113
3114 case CRYPTO_DATA_UIO:
3115 /*
3116 * Jump to the first iovec containing data to be processed.
3117 */
3118 uiop = in->cd_uio;
3119 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3120 off >= uiop->uio_iov[vec_idx].iov_len;
3121 off -= uiop->uio_iov[vec_idx++].iov_len)
3122 ;
3123 if (vec_idx == uiop->uio_iovcnt) {
3124 /*
3125 * The caller specified an offset that is larger than
3126 * the total size of the buffers it provided.
3127 */
3128 return (CRYPTO_DATA_LEN_RANGE);
3129 }
3130
3131 /*
3132 * Now process the iovecs.
3133 */
3134 while (vec_idx < uiop->uio_iovcnt && count > 0) {
3135 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3136 off, count);
3137 count -= cur_len;
3138 if (reverse) {
3139 /* Fill the dest buffer from the end */
3140 dca_reverse(uiop->uio_iov[vec_idx].iov_base +
3141 off, dest+count, cur_len, cur_len);
3142 } else {
3143 bcopy(uiop->uio_iov[vec_idx].iov_base + off,
3144 dest, cur_len);
3145 dest += cur_len;
3146 }
3147 in->cd_offset += cur_len;
3148 in->cd_length -= cur_len;
3149 vec_idx++;
3150 off = 0;
3151 }
3152
3153 if (vec_idx == uiop->uio_iovcnt && count > 0) {
3154 /*
3155 * The end of the specified iovec's was reached but
3156 * the length requested could not be processed
3157 * (requested to digest more data than it provided).
3158 */
3159 return (CRYPTO_DATA_LEN_RANGE);
3160 }
3161 break;
3162
3163 case CRYPTO_DATA_MBLK:
3164 /*
3165 * Jump to the first mblk_t containing data to be processed.
3166 */
3167 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
3168 off -= MBLKL(mp), mp = mp->b_cont)
3169 ;
3170 if (mp == NULL) {
3171 /*
3172 * The caller specified an offset that is larger than
3173 * the total size of the buffers it provided.
3174 */
3175 return (CRYPTO_DATA_LEN_RANGE);
3176 }
3177
3178 /*
3179 * Now do the processing on the mblk chain.
3180 */
3181 while (mp != NULL && count > 0) {
3182 cur_len = min(MBLKL(mp) - off, count);
3183 count -= cur_len;
3184 if (reverse) {
3185 /* Fill the dest buffer from the end */
3186 dca_reverse((char *)(mp->b_rptr + off),
3187 dest+count, cur_len, cur_len);
3188 } else {
3189 bcopy((char *)(mp->b_rptr + off), dest,
3190 cur_len);
3191 dest += cur_len;
3192 }
3193 in->cd_offset += cur_len;
3194 in->cd_length -= cur_len;
3195 mp = mp->b_cont;
3196 off = 0;
3197 }
3198
3199 if (mp == NULL && count > 0) {
3200 /*
3201 * The end of the mblk was reached but the length
3202 * requested could not be processed, (requested to
3203 * digest more data than it provided).
3204 */
3205 return (CRYPTO_DATA_LEN_RANGE);
3206 }
3207 break;
3208
3209 default:
3210 DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format");
3211 rv = CRYPTO_ARGUMENTS_BAD;
3212 }
3213 return (rv);
3214 }
3215
3216 /*
3217 * Increments the cd_offset and decrements the cd_length as the data is
3218 * gathered from the crypto_data_t struct.
3219 */
3220 int
3221 dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest,
3222 int count)
3223 {
3224 int rv = CRYPTO_SUCCESS;
3225 caddr_t baddr;
3226 uint_t vec_idx;
3227 uio_t *uiop;
3228 off_t off = in->cd_offset;
3229 size_t cur_len;
3230 mblk_t *mp;
3231
3232 /* Process the residual first */
3233 if (*residlen > 0) {
3234 uint_t num = min(count, *residlen);
3235 bcopy(resid, dest, num);
3236 *residlen -= num;
3237 if (*residlen > 0) {
3238 /*
3239 * Requested amount 'count' is less than what's in
3240 * the residual, so shuffle any remaining resid to
3241 * the front.
3242 */
3243 baddr = resid + num;
3244 bcopy(baddr, resid, *residlen);
3245 }
3246 dest += num;
3247 count -= num;
3248 }
3249
3250 /* Now process what's in the crypto_data_t structs */
3251 switch (in->cd_format) {
3252 case CRYPTO_DATA_RAW:
3253 if (count > in->cd_length) {
3254 /*
3255 * The caller specified a length greater than the
3256 * size of the buffer.
3257 */
3258 return (CRYPTO_DATA_LEN_RANGE);
3259 }
3260 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
3261 in->cd_offset += count;
3262 in->cd_length -= count;
3263 break;
3264
3265 case CRYPTO_DATA_UIO:
3266 /*
3267 * Jump to the first iovec containing data to be processed.
3268 */
3269 uiop = in->cd_uio;
3270 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3271 off >= uiop->uio_iov[vec_idx].iov_len;
3272 off -= uiop->uio_iov[vec_idx++].iov_len)
3273 ;
3274 if (vec_idx == uiop->uio_iovcnt) {
3275 /*
3276 * The caller specified an offset that is larger than
3277 * the total size of the buffers it provided.
3278 */
3279 return (CRYPTO_DATA_LEN_RANGE);
3280 }
3281
3282 /*
3283 * Now process the iovecs.
3284 */
3285 while (vec_idx < uiop->uio_iovcnt && count > 0) {
3286 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3287 off, count);
3288 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
3289 cur_len);
3290 count -= cur_len;
3291 dest += cur_len;
3292 in->cd_offset += cur_len;
3293 in->cd_length -= cur_len;
3294 vec_idx++;
3295 off = 0;
3296 }
3297
3298 if (vec_idx == uiop->uio_iovcnt && count > 0) {
3299 /*
3300 * The end of the specified iovec's was reached but
3301 * the length requested could not be processed
3302 * (requested to digest more data than it provided).
3303 */
3304 return (CRYPTO_DATA_LEN_RANGE);
3305 }
3306 break;
3307
3308 case CRYPTO_DATA_MBLK:
3309 /*
3310 * Jump to the first mblk_t containing data to be processed.
3311 */
3312 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
3313 off -= MBLKL(mp), mp = mp->b_cont)
3314 ;
3315 if (mp == NULL) {
3316 /*
3317 * The caller specified an offset that is larger than
3318 * the total size of the buffers it provided.
3319 */
3320 return (CRYPTO_DATA_LEN_RANGE);
3321 }
3322
3323 /*
3324 * Now do the processing on the mblk chain.
3325 */
3326 while (mp != NULL && count > 0) {
3327 cur_len = min(MBLKL(mp) - off, count);
3328 bcopy((char *)(mp->b_rptr + off), dest, cur_len);
3329 count -= cur_len;
3330 dest += cur_len;
3331 in->cd_offset += cur_len;
3332 in->cd_length -= cur_len;
3333 mp = mp->b_cont;
3334 off = 0;
3335 }
3336
3337 if (mp == NULL && count > 0) {
3338 /*
3339 * The end of the mblk was reached but the length
3340 * requested could not be processed, (requested to
3341 * digest more data than it provided).
3342 */
3343 return (CRYPTO_DATA_LEN_RANGE);
3344 }
3345 break;
3346
3347 default:
3348 DBG(NULL, DWARN,
3349 "dca_resid_gather: unrecognised crypto data format");
3350 rv = CRYPTO_ARGUMENTS_BAD;
3351 }
3352 return (rv);
3353 }
3354
3355 /*
3356 * Appends the data to the crypto_data_t struct increasing cd_length.
3357 * cd_offset is left unchanged.
3358 * Data is reverse-copied if the flag is TRUE.
3359 */
3360 int
3361 dca_scatter(const char *src, crypto_data_t *out, int count, int reverse)
3362 {
3363 int rv = CRYPTO_SUCCESS;
3364 off_t offset = out->cd_offset + out->cd_length;
3365 uint_t vec_idx;
3366 uio_t *uiop;
3367 size_t cur_len;
3368 mblk_t *mp;
3369
3370 switch (out->cd_format) {
3371 case CRYPTO_DATA_RAW:
3372 if (out->cd_raw.iov_len - offset < count) {
3373 /* Trying to write out more than space available. */
3374 return (CRYPTO_DATA_LEN_RANGE);
3375 }
3376 if (reverse)
3377 dca_reverse((void*) src, out->cd_raw.iov_base + offset,
3378 count, count);
3379 else
3380 bcopy(src, out->cd_raw.iov_base + offset, count);
3381 out->cd_length += count;
3382 break;
3383
3384 case CRYPTO_DATA_UIO:
3385 /*
3386 * Jump to the first iovec that can be written to.
3387 */
3388 uiop = out->cd_uio;
3389 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3390 offset >= uiop->uio_iov[vec_idx].iov_len;
3391 offset -= uiop->uio_iov[vec_idx++].iov_len)
3392 ;
3393 if (vec_idx == uiop->uio_iovcnt) {
3394 /*
3395 * The caller specified an offset that is larger than
3396 * the total size of the buffers it provided.
3397 */
3398 return (CRYPTO_DATA_LEN_RANGE);
3399 }
3400
3401 /*
3402 * Now process the iovecs.
3403 */
3404 while (vec_idx < uiop->uio_iovcnt && count > 0) {
3405 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3406 offset, count);
3407 count -= cur_len;
3408 if (reverse) {
3409 dca_reverse((void*) (src+count),
3410 uiop->uio_iov[vec_idx].iov_base +
3411 offset, cur_len, cur_len);
3412 } else {
3413 bcopy(src, uiop->uio_iov[vec_idx].iov_base +
3414 offset, cur_len);
3415 src += cur_len;
3416 }
3417 out->cd_length += cur_len;
3418 vec_idx++;
3419 offset = 0;
3420 }
3421
3422 if (vec_idx == uiop->uio_iovcnt && count > 0) {
3423 /*
3424 * The end of the specified iovec's was reached but
3425 * the length requested could not be processed
3426 * (requested to write more data than space provided).
3427 */
3428 return (CRYPTO_DATA_LEN_RANGE);
3429 }
3430 break;
3431
3432 case CRYPTO_DATA_MBLK:
3433 /*
3434 * Jump to the first mblk_t that can be written to.
3435 */
3436 for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp);
3437 offset -= MBLKL(mp), mp = mp->b_cont)
3438 ;
3439 if (mp == NULL) {
3440 /*
3441 * The caller specified an offset that is larger than
3442 * the total size of the buffers it provided.
3443 */
3444 return (CRYPTO_DATA_LEN_RANGE);
3445 }
3446
3447 /*
3448 * Now do the processing on the mblk chain.
3449 */
3450 while (mp != NULL && count > 0) {
3451 cur_len = min(MBLKL(mp) - offset, count);
3452 count -= cur_len;
3453 if (reverse) {
3454 dca_reverse((void*) (src+count),
3455 (char *)(mp->b_rptr + offset), cur_len,
3456 cur_len);
3457 } else {
3458 bcopy(src, (char *)(mp->b_rptr + offset),
3459 cur_len);
3460 src += cur_len;
3461 }
3462 out->cd_length += cur_len;
3463 mp = mp->b_cont;
3464 offset = 0;
3465 }
3466
3467 if (mp == NULL && count > 0) {
3468 /*
3469 * The end of the mblk was reached but the length
3470 * requested could not be processed, (requested to
3471 * digest more data than it provided).
3472 */
3473 return (CRYPTO_DATA_LEN_RANGE);
3474 }
3475 break;
3476
3477 default:
3478 DBG(NULL, DWARN, "unrecognised crypto data format");
3479 rv = CRYPTO_ARGUMENTS_BAD;
3480 }
3481 return (rv);
3482 }
3483
3484 /*
3485 * Compare two byte arrays in reverse order.
3486 * Return 0 if they are identical, 1 otherwise.
3487 */
3488 int
3489 dca_bcmp_reverse(const void *s1, const void *s2, size_t n)
3490 {
3491 int i;
3492 caddr_t src, dst;
3493
3494 if (!n)
3495 return (0);
3496
3497 src = ((caddr_t)s1) + n - 1;
3498 dst = (caddr_t)s2;
3499 for (i = 0; i < n; i++) {
3500 if (*src != *dst)
3501 return (1);
3502 src--;
3503 dst++;
3504 }
3505
3506 return (0);
3507 }
3508
3509
3510 /*
3511 * This calculates the size of a bignum in bits, specifically not counting
3512 * leading zero bits. This size calculation must be done *before* any
3513 * endian reversal takes place (i.e. the numbers are in absolute big-endian
3514 * order.)
3515 */
3516 int
3517 dca_bitlen(unsigned char *bignum, int bytelen)
3518 {
3519 unsigned char msbyte;
3520 int i, j;
3521
3522 for (i = 0; i < bytelen - 1; i++) {
3523 if (bignum[i] != 0) {
3524 break;
3525 }
3526 }
3527 msbyte = bignum[i];
3528 for (j = 8; j > 1; j--) {
3529 if (msbyte & 0x80) {
3530 break;
3531 }
3532 msbyte <<= 1;
3533 }
3534 return ((8 * (bytelen - i - 1)) + j);
3535 }
3536
3537 /*
3538 * This compares to bignums (in big-endian order). It ignores leading
3539 * null bytes. The result semantics follow bcmp, mempcmp, strcmp, etc.
3540 */
3541 int
3542 dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len)
3543 {
3544 while ((n1len > 1) && (*n1 == 0)) {
3545 n1len--;
3546 n1++;
3547 }
3548 while ((n2len > 1) && (*n2 == 0)) {
3549 n2len--;
3550 n2++;
3551 }
3552 if (n1len != n2len) {
3553 return (n1len - n2len);
3554 }
3555 while ((n1len > 1) && (*n1 == *n2)) {
3556 n1++;
3557 n2++;
3558 n1len--;
3559 }
3560 return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2));
3561 }
3562
3563 /*
3564 * Return array of key attributes.
3565 */
3566 crypto_object_attribute_t *
3567 dca_get_key_attr(crypto_key_t *key)
3568 {
3569 if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) ||
3570 (key->ck_count == 0)) {
3571 return (NULL);
3572 }
3573
3574 return (key->ck_attrs);
3575 }
3576
3577 /*
3578 * If attribute type exists valp points to it's 32-bit value.
3579 */
3580 int
3581 dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum,
3582 uint64_t atype, uint32_t *valp)
3583 {
3584 crypto_object_attribute_t *bap;
3585
3586 bap = dca_find_attribute(attrp, atnum, atype);
3587 if (bap == NULL) {
3588 return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
3589 }
3590
3591 *valp = *bap->oa_value;
3592
3593 return (CRYPTO_SUCCESS);
3594 }
3595
3596 /*
3597 * If attribute type exists data contains the start address of the value,
3598 * and numelems contains it's length.
3599 */
3600 int
3601 dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum,
3602 uint64_t atype, void **data, unsigned int *numelems)
3603 {
3604 crypto_object_attribute_t *bap;
3605
3606 bap = dca_find_attribute(attrp, atnum, atype);
3607 if (bap == NULL) {
3608 return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
3609 }
3610
3611 *data = bap->oa_value;
3612 *numelems = bap->oa_value_len;
3613
3614 return (CRYPTO_SUCCESS);
3615 }
3616
3617 /*
3618 * Finds entry of specified name. If it is not found dca_find_attribute returns
3619 * NULL.
3620 */
3621 crypto_object_attribute_t *
3622 dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum,
3623 uint64_t atype)
3624 {
3625 while (atnum) {
3626 if (attrp->oa_type == atype)
3627 return (attrp);
3628 atnum--;
3629 attrp++;
3630 }
3631 return (NULL);
3632 }
3633
3634 /*
3635 * Return the address of the first data buffer. If the data format is
3636 * unrecognised return NULL.
3637 */
3638 caddr_t
3639 dca_bufdaddr(crypto_data_t *data)
3640 {
3641 switch (data->cd_format) {
3642 case CRYPTO_DATA_RAW:
3643 return (data->cd_raw.iov_base + data->cd_offset);
3644 case CRYPTO_DATA_UIO:
3645 return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset);
3646 case CRYPTO_DATA_MBLK:
3647 return ((char *)data->cd_mp->b_rptr + data->cd_offset);
3648 default:
3649 DBG(NULL, DWARN,
3650 "dca_bufdaddr: unrecognised crypto data format");
3651 return (NULL);
3652 }
3653 }
3654
3655 static caddr_t
3656 dca_bufdaddr_out(crypto_data_t *data)
3657 {
3658 size_t offset = data->cd_offset + data->cd_length;
3659
3660 switch (data->cd_format) {
3661 case CRYPTO_DATA_RAW:
3662 return (data->cd_raw.iov_base + offset);
3663 case CRYPTO_DATA_UIO:
3664 return (data->cd_uio->uio_iov[0].iov_base + offset);
3665 case CRYPTO_DATA_MBLK:
3666 return ((char *)data->cd_mp->b_rptr + offset);
3667 default:
3668 DBG(NULL, DWARN,
3669 "dca_bufdaddr_out: unrecognised crypto data format");
3670 return (NULL);
3671 }
3672 }
3673
3674 /*
3675 * Control entry points.
3676 */
3677
3678 /* ARGSUSED */
3679 static void
3680 dca_provider_status(crypto_provider_handle_t provider, uint_t *status)
3681 {
3682 *status = CRYPTO_PROVIDER_READY;
3683 }
3684
3685 /*
3686 * Cipher (encrypt/decrypt) entry points.
3687 */
3688
3689 /* ARGSUSED */
3690 static int
3691 dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
3692 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
3693 crypto_req_handle_t req)
3694 {
3695 int error = CRYPTO_FAILED;
3696 dca_t *softc;
3697 /* LINTED E_FUNC_SET_NOT_USED */
3698 int instance;
3699
3700 /* extract softc and instance number from context */
3701 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3702 DBG(softc, DENTRY, "dca_encrypt_init: started");
3703
3704 /* check mechanism */
3705 switch (mechanism->cm_type) {
3706 case DES_CBC_MECH_INFO_TYPE:
3707 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3708 DR_ENCRYPT);
3709 break;
3710 case DES3_CBC_MECH_INFO_TYPE:
3711 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3712 DR_ENCRYPT | DR_TRIPLE);
3713 break;
3714 case RSA_PKCS_MECH_INFO_TYPE:
3715 case RSA_X_509_MECH_INFO_TYPE:
3716 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
3717 break;
3718 default:
3719 cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type "
3720 "0x%llx\n", (unsigned long long)mechanism->cm_type);
3721 error = CRYPTO_MECHANISM_INVALID;
3722 }
3723
3724 DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error);
3725
3726 if (error == CRYPTO_SUCCESS)
3727 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
3728 &softc->dca_ctx_list_lock);
3729
3730 return (error);
3731 }
3732
3733 /* ARGSUSED */
3734 static int
3735 dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
3736 crypto_data_t *ciphertext, crypto_req_handle_t req)
3737 {
3738 int error = CRYPTO_FAILED;
3739 dca_t *softc;
3740 /* LINTED E_FUNC_SET_NOT_USED */
3741 int instance;
3742
3743 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3744 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3745
3746 /* extract softc and instance number from context */
3747 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3748 DBG(softc, DENTRY, "dca_encrypt: started");
3749
3750 /* handle inplace ops */
3751 if (!ciphertext) {
3752 dca_request_t *reqp = ctx->cc_provider_private;
3753 reqp->dr_flags |= DR_INPLACE;
3754 ciphertext = plaintext;
3755 }
3756
3757 /* check mechanism */
3758 switch (DCA_MECH_FROM_CTX(ctx)) {
3759 case DES_CBC_MECH_INFO_TYPE:
3760 error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT);
3761 break;
3762 case DES3_CBC_MECH_INFO_TYPE:
3763 error = dca_3des(ctx, plaintext, ciphertext, req,
3764 DR_ENCRYPT | DR_TRIPLE);
3765 break;
3766 case RSA_PKCS_MECH_INFO_TYPE:
3767 case RSA_X_509_MECH_INFO_TYPE:
3768 error = dca_rsastart(ctx, plaintext, ciphertext, req,
3769 DCA_RSA_ENC);
3770 break;
3771 default:
3772 /* Should never reach here */
3773 cmn_err(CE_WARN, "dca_encrypt: unexpected mech type "
3774 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3775 error = CRYPTO_MECHANISM_INVALID;
3776 }
3777
3778 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
3779 (error != CRYPTO_BUFFER_TOO_SMALL)) {
3780 ciphertext->cd_length = 0;
3781 }
3782
3783 DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error);
3784
3785 return (error);
3786 }
3787
3788 /* ARGSUSED */
3789 static int
3790 dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext,
3791 crypto_data_t *ciphertext, crypto_req_handle_t req)
3792 {
3793 int error = CRYPTO_FAILED;
3794 dca_t *softc;
3795 /* LINTED E_FUNC_SET_NOT_USED */
3796 int instance;
3797
3798 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3799 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3800
3801 /* extract softc and instance number from context */
3802 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3803 DBG(softc, DENTRY, "dca_encrypt_update: started");
3804
3805 /* handle inplace ops */
3806 if (!ciphertext) {
3807 dca_request_t *reqp = ctx->cc_provider_private;
3808 reqp->dr_flags |= DR_INPLACE;
3809 ciphertext = plaintext;
3810 }
3811
3812 /* check mechanism */
3813 switch (DCA_MECH_FROM_CTX(ctx)) {
3814 case DES_CBC_MECH_INFO_TYPE:
3815 error = dca_3desupdate(ctx, plaintext, ciphertext, req,
3816 DR_ENCRYPT);
3817 break;
3818 case DES3_CBC_MECH_INFO_TYPE:
3819 error = dca_3desupdate(ctx, plaintext, ciphertext, req,
3820 DR_ENCRYPT | DR_TRIPLE);
3821 break;
3822 default:
3823 /* Should never reach here */
3824 cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type "
3825 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3826 error = CRYPTO_MECHANISM_INVALID;
3827 }
3828
3829 DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error);
3830
3831 return (error);
3832 }
3833
3834 /* ARGSUSED */
3835 static int
3836 dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
3837 crypto_req_handle_t req)
3838 {
3839 int error = CRYPTO_FAILED;
3840 dca_t *softc;
3841 /* LINTED E_FUNC_SET_NOT_USED */
3842 int instance;
3843
3844 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3845 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3846
3847 /* extract softc and instance number from context */
3848 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3849 DBG(softc, DENTRY, "dca_encrypt_final: started");
3850
3851 /* check mechanism */
3852 switch (DCA_MECH_FROM_CTX(ctx)) {
3853 case DES_CBC_MECH_INFO_TYPE:
3854 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT);
3855 break;
3856 case DES3_CBC_MECH_INFO_TYPE:
3857 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE);
3858 break;
3859 default:
3860 /* Should never reach here */
3861 cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type "
3862 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3863 error = CRYPTO_MECHANISM_INVALID;
3864 }
3865
3866 DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error);
3867
3868 return (error);
3869 }
3870
3871 /* ARGSUSED */
3872 static int
3873 dca_encrypt_atomic(crypto_provider_handle_t provider,
3874 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
3875 crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
3876 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
3877 {
3878 int error = CRYPTO_FAILED;
3879 dca_t *softc = (dca_t *)provider;
3880
3881 DBG(softc, DENTRY, "dca_encrypt_atomic: started");
3882
3883 if (ctx_template != NULL)
3884 return (CRYPTO_ARGUMENTS_BAD);
3885
3886 /* handle inplace ops */
3887 if (!ciphertext) {
3888 ciphertext = plaintext;
3889 }
3890
3891 /* check mechanism */
3892 switch (mechanism->cm_type) {
3893 case DES_CBC_MECH_INFO_TYPE:
3894 error = dca_3desatomic(provider, session_id, mechanism, key,
3895 plaintext, ciphertext, KM_SLEEP, req,
3896 DR_ENCRYPT | DR_ATOMIC);
3897 break;
3898 case DES3_CBC_MECH_INFO_TYPE:
3899 error = dca_3desatomic(provider, session_id, mechanism, key,
3900 plaintext, ciphertext, KM_SLEEP, req,
3901 DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC);
3902 break;
3903 case RSA_PKCS_MECH_INFO_TYPE:
3904 case RSA_X_509_MECH_INFO_TYPE:
3905 error = dca_rsaatomic(provider, session_id, mechanism, key,
3906 plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC);
3907 break;
3908 default:
3909 cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type "
3910 "0x%llx\n", (unsigned long long)mechanism->cm_type);
3911 error = CRYPTO_MECHANISM_INVALID;
3912 }
3913
3914 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
3915 ciphertext->cd_length = 0;
3916 }
3917
3918 DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error);
3919
3920 return (error);
3921 }
3922
3923 /* ARGSUSED */
3924 static int
3925 dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
3926 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
3927 crypto_req_handle_t req)
3928 {
3929 int error = CRYPTO_FAILED;
3930 dca_t *softc;
3931 /* LINTED E_FUNC_SET_NOT_USED */
3932 int instance;
3933
3934 /* extract softc and instance number from context */
3935 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3936 DBG(softc, DENTRY, "dca_decrypt_init: started");
3937
3938 /* check mechanism */
3939 switch (mechanism->cm_type) {
3940 case DES_CBC_MECH_INFO_TYPE:
3941 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3942 DR_DECRYPT);
3943 break;
3944 case DES3_CBC_MECH_INFO_TYPE:
3945 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3946 DR_DECRYPT | DR_TRIPLE);
3947 break;
3948 case RSA_PKCS_MECH_INFO_TYPE:
3949 case RSA_X_509_MECH_INFO_TYPE:
3950 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
3951 break;
3952 default:
3953 cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type "
3954 "0x%llx\n", (unsigned long long)mechanism->cm_type);
3955 error = CRYPTO_MECHANISM_INVALID;
3956 }
3957
3958 DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error);
3959
3960 if (error == CRYPTO_SUCCESS)
3961 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
3962 &softc->dca_ctx_list_lock);
3963
3964 return (error);
3965 }
3966
3967 /* ARGSUSED */
3968 static int
3969 dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
3970 crypto_data_t *plaintext, crypto_req_handle_t req)
3971 {
3972 int error = CRYPTO_FAILED;
3973 dca_t *softc;
3974 /* LINTED E_FUNC_SET_NOT_USED */
3975 int instance;
3976
3977 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3978 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3979
3980 /* extract softc and instance number from context */
3981 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3982 DBG(softc, DENTRY, "dca_decrypt: started");
3983
3984 /* handle inplace ops */
3985 if (!plaintext) {
3986 dca_request_t *reqp = ctx->cc_provider_private;
3987 reqp->dr_flags |= DR_INPLACE;
3988 plaintext = ciphertext;
3989 }
3990
3991 /* check mechanism */
3992 switch (DCA_MECH_FROM_CTX(ctx)) {
3993 case DES_CBC_MECH_INFO_TYPE:
3994 error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT);
3995 break;
3996 case DES3_CBC_MECH_INFO_TYPE:
3997 error = dca_3des(ctx, ciphertext, plaintext, req,
3998 DR_DECRYPT | DR_TRIPLE);
3999 break;
4000 case RSA_PKCS_MECH_INFO_TYPE:
4001 case RSA_X_509_MECH_INFO_TYPE:
4002 error = dca_rsastart(ctx, ciphertext, plaintext, req,
4003 DCA_RSA_DEC);
4004 break;
4005 default:
4006 /* Should never reach here */
4007 cmn_err(CE_WARN, "dca_decrypt: unexpected mech type "
4008 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4009 error = CRYPTO_MECHANISM_INVALID;
4010 }
4011
4012 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
4013 (error != CRYPTO_BUFFER_TOO_SMALL)) {
4014 if (plaintext)
4015 plaintext->cd_length = 0;
4016 }
4017
4018 DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error);
4019
4020 return (error);
4021 }
4022
4023 /* ARGSUSED */
4024 static int
4025 dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
4026 crypto_data_t *plaintext, crypto_req_handle_t req)
4027 {
4028 int error = CRYPTO_FAILED;
4029 dca_t *softc;
4030 /* LINTED E_FUNC_SET_NOT_USED */
4031 int instance;
4032
4033 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4034 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4035
4036 /* extract softc and instance number from context */
4037 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4038 DBG(softc, DENTRY, "dca_decrypt_update: started");
4039
4040 /* handle inplace ops */
4041 if (!plaintext) {
4042 dca_request_t *reqp = ctx->cc_provider_private;
4043 reqp->dr_flags |= DR_INPLACE;
4044 plaintext = ciphertext;
4045 }
4046
4047 /* check mechanism */
4048 switch (DCA_MECH_FROM_CTX(ctx)) {
4049 case DES_CBC_MECH_INFO_TYPE:
4050 error = dca_3desupdate(ctx, ciphertext, plaintext, req,
4051 DR_DECRYPT);
4052 break;
4053 case DES3_CBC_MECH_INFO_TYPE:
4054 error = dca_3desupdate(ctx, ciphertext, plaintext, req,
4055 DR_DECRYPT | DR_TRIPLE);
4056 break;
4057 default:
4058 /* Should never reach here */
4059 cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type "
4060 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4061 error = CRYPTO_MECHANISM_INVALID;
4062 }
4063
4064 DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error);
4065
4066 return (error);
4067 }
4068
4069 /* ARGSUSED */
4070 static int
4071 dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext,
4072 crypto_req_handle_t req)
4073 {
4074 int error = CRYPTO_FAILED;
4075 dca_t *softc;
4076 /* LINTED E_FUNC_SET_NOT_USED */
4077 int instance;
4078
4079 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4080 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4081
4082 /* extract softc and instance number from context */
4083 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4084 DBG(softc, DENTRY, "dca_decrypt_final: started");
4085
4086 /* check mechanism */
4087 switch (DCA_MECH_FROM_CTX(ctx)) {
4088 case DES_CBC_MECH_INFO_TYPE:
4089 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT);
4090 break;
4091 case DES3_CBC_MECH_INFO_TYPE:
4092 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE);
4093 break;
4094 default:
4095 /* Should never reach here */
4096 cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type "
4097 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4098 error = CRYPTO_MECHANISM_INVALID;
4099 }
4100
4101 DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error);
4102
4103 return (error);
4104 }
4105
4106 /* ARGSUSED */
4107 static int
4108 dca_decrypt_atomic(crypto_provider_handle_t provider,
4109 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4110 crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
4111 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4112 {
4113 int error = CRYPTO_FAILED;
4114 dca_t *softc = (dca_t *)provider;
4115
4116 DBG(softc, DENTRY, "dca_decrypt_atomic: started");
4117
4118 if (ctx_template != NULL)
4119 return (CRYPTO_ARGUMENTS_BAD);
4120
4121 /* handle inplace ops */
4122 if (!plaintext) {
4123 plaintext = ciphertext;
4124 }
4125
4126 /* check mechanism */
4127 switch (mechanism->cm_type) {
4128 case DES_CBC_MECH_INFO_TYPE:
4129 error = dca_3desatomic(provider, session_id, mechanism, key,
4130 ciphertext, plaintext, KM_SLEEP, req,
4131 DR_DECRYPT | DR_ATOMIC);
4132 break;
4133 case DES3_CBC_MECH_INFO_TYPE:
4134 error = dca_3desatomic(provider, session_id, mechanism, key,
4135 ciphertext, plaintext, KM_SLEEP, req,
4136 DR_DECRYPT | DR_TRIPLE | DR_ATOMIC);
4137 break;
4138 case RSA_PKCS_MECH_INFO_TYPE:
4139 case RSA_X_509_MECH_INFO_TYPE:
4140 error = dca_rsaatomic(provider, session_id, mechanism, key,
4141 ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC);
4142 break;
4143 default:
4144 cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type "
4145 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4146 error = CRYPTO_MECHANISM_INVALID;
4147 }
4148
4149 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
4150 plaintext->cd_length = 0;
4151 }
4152
4153 DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error);
4154
4155 return (error);
4156 }
4157
4158 /*
4159 * Sign entry points.
4160 */
4161
4162 /* ARGSUSED */
4163 static int
4164 dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4165 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4166 crypto_req_handle_t req)
4167 {
4168 int error = CRYPTO_FAILED;
4169 dca_t *softc;
4170 /* LINTED E_FUNC_SET_NOT_USED */
4171 int instance;
4172
4173 /* extract softc and instance number from context */
4174 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4175 DBG(softc, DENTRY, "dca_sign_init: started\n");
4176
4177 if (ctx_template != NULL)
4178 return (CRYPTO_ARGUMENTS_BAD);
4179
4180 /* check mechanism */
4181 switch (mechanism->cm_type) {
4182 case RSA_PKCS_MECH_INFO_TYPE:
4183 case RSA_X_509_MECH_INFO_TYPE:
4184 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4185 break;
4186 case DSA_MECH_INFO_TYPE:
4187 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
4188 DCA_DSA_SIGN);
4189 break;
4190 default:
4191 cmn_err(CE_WARN, "dca_sign_init: unexpected mech type "
4192 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4193 error = CRYPTO_MECHANISM_INVALID;
4194 }
4195
4196 DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error);
4197
4198 if (error == CRYPTO_SUCCESS)
4199 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4200 &softc->dca_ctx_list_lock);
4201
4202 return (error);
4203 }
4204
4205 static int
4206 dca_sign(crypto_ctx_t *ctx, crypto_data_t *data,
4207 crypto_data_t *signature, crypto_req_handle_t req)
4208 {
4209 int error = CRYPTO_FAILED;
4210 dca_t *softc;
4211 /* LINTED E_FUNC_SET_NOT_USED */
4212 int instance;
4213
4214 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4215 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4216
4217 /* extract softc and instance number from context */
4218 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4219 DBG(softc, DENTRY, "dca_sign: started\n");
4220
4221 /* check mechanism */
4222 switch (DCA_MECH_FROM_CTX(ctx)) {
4223 case RSA_PKCS_MECH_INFO_TYPE:
4224 case RSA_X_509_MECH_INFO_TYPE:
4225 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN);
4226 break;
4227 case DSA_MECH_INFO_TYPE:
4228 error = dca_dsa_sign(ctx, data, signature, req);
4229 break;
4230 default:
4231 cmn_err(CE_WARN, "dca_sign: unexpected mech type "
4232 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4233 error = CRYPTO_MECHANISM_INVALID;
4234 }
4235
4236 DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error);
4237
4238 return (error);
4239 }
4240
4241 /* ARGSUSED */
4242 static int
4243 dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data,
4244 crypto_req_handle_t req)
4245 {
4246 int error = CRYPTO_MECHANISM_INVALID;
4247 dca_t *softc;
4248 /* LINTED E_FUNC_SET_NOT_USED */
4249 int instance;
4250
4251 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4252 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4253
4254 /* extract softc and instance number from context */
4255 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4256 DBG(softc, DENTRY, "dca_sign_update: started\n");
4257
4258 cmn_err(CE_WARN, "dca_sign_update: unexpected mech type "
4259 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4260
4261 DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error);
4262
4263 return (error);
4264 }
4265
4266 /* ARGSUSED */
4267 static int
4268 dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
4269 crypto_req_handle_t req)
4270 {
4271 int error = CRYPTO_MECHANISM_INVALID;
4272 dca_t *softc;
4273 /* LINTED E_FUNC_SET_NOT_USED */
4274 int instance;
4275
4276 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4277 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4278
4279 /* extract softc and instance number from context */
4280 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4281 DBG(softc, DENTRY, "dca_sign_final: started\n");
4282
4283 cmn_err(CE_WARN, "dca_sign_final: unexpected mech type "
4284 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4285
4286 DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error);
4287
4288 return (error);
4289 }
4290
4291 static int
4292 dca_sign_atomic(crypto_provider_handle_t provider,
4293 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4294 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4295 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4296 {
4297 int error = CRYPTO_FAILED;
4298 dca_t *softc = (dca_t *)provider;
4299
4300 DBG(softc, DENTRY, "dca_sign_atomic: started\n");
4301
4302 if (ctx_template != NULL)
4303 return (CRYPTO_ARGUMENTS_BAD);
4304
4305 /* check mechanism */
4306 switch (mechanism->cm_type) {
4307 case RSA_PKCS_MECH_INFO_TYPE:
4308 case RSA_X_509_MECH_INFO_TYPE:
4309 error = dca_rsaatomic(provider, session_id, mechanism, key,
4310 data, signature, KM_SLEEP, req, DCA_RSA_SIGN);
4311 break;
4312 case DSA_MECH_INFO_TYPE:
4313 error = dca_dsaatomic(provider, session_id, mechanism, key,
4314 data, signature, KM_SLEEP, req, DCA_DSA_SIGN);
4315 break;
4316 default:
4317 cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type "
4318 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4319 error = CRYPTO_MECHANISM_INVALID;
4320 }
4321
4322 DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error);
4323
4324 return (error);
4325 }
4326
4327 /* ARGSUSED */
4328 static int
4329 dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4330 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4331 crypto_req_handle_t req)
4332 {
4333 int error = CRYPTO_FAILED;
4334 dca_t *softc;
4335 /* LINTED E_FUNC_SET_NOT_USED */
4336 int instance;
4337
4338 /* extract softc and instance number from context */
4339 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4340 DBG(softc, DENTRY, "dca_sign_recover_init: started\n");
4341
4342 if (ctx_template != NULL)
4343 return (CRYPTO_ARGUMENTS_BAD);
4344
4345 /* check mechanism */
4346 switch (mechanism->cm_type) {
4347 case RSA_PKCS_MECH_INFO_TYPE:
4348 case RSA_X_509_MECH_INFO_TYPE:
4349 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4350 break;
4351 default:
4352 cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type "
4353 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4354 error = CRYPTO_MECHANISM_INVALID;
4355 }
4356
4357 DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error);
4358
4359 if (error == CRYPTO_SUCCESS)
4360 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4361 &softc->dca_ctx_list_lock);
4362
4363 return (error);
4364 }
4365
4366 static int
4367 dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data,
4368 crypto_data_t *signature, crypto_req_handle_t req)
4369 {
4370 int error = CRYPTO_FAILED;
4371 dca_t *softc;
4372 /* LINTED E_FUNC_SET_NOT_USED */
4373 int instance;
4374
4375 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4376 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4377
4378 /* extract softc and instance number from context */
4379 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4380 DBG(softc, DENTRY, "dca_sign_recover: started\n");
4381
4382 /* check mechanism */
4383 switch (DCA_MECH_FROM_CTX(ctx)) {
4384 case RSA_PKCS_MECH_INFO_TYPE:
4385 case RSA_X_509_MECH_INFO_TYPE:
4386 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR);
4387 break;
4388 default:
4389 cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type "
4390 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4391 error = CRYPTO_MECHANISM_INVALID;
4392 }
4393
4394 DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error);
4395
4396 return (error);
4397 }
4398
4399 static int
4400 dca_sign_recover_atomic(crypto_provider_handle_t provider,
4401 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4402 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4403 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4404 {
4405 int error = CRYPTO_FAILED;
4406 dca_t *softc = (dca_t *)provider;
4407 /* LINTED E_FUNC_SET_NOT_USED */
4408 int instance;
4409
4410 instance = ddi_get_instance(softc->dca_dip);
4411 DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n");
4412
4413 if (ctx_template != NULL)
4414 return (CRYPTO_ARGUMENTS_BAD);
4415
4416 /* check mechanism */
4417 switch (mechanism->cm_type) {
4418 case RSA_PKCS_MECH_INFO_TYPE:
4419 case RSA_X_509_MECH_INFO_TYPE:
4420 error = dca_rsaatomic(provider, session_id, mechanism, key,
4421 data, signature, KM_SLEEP, req, DCA_RSA_SIGNR);
4422 break;
4423 default:
4424 cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type"
4425 " 0x%llx\n", (unsigned long long)mechanism->cm_type);
4426 error = CRYPTO_MECHANISM_INVALID;
4427 }
4428
4429 DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error);
4430
4431 return (error);
4432 }
4433
4434 /*
4435 * Verify entry points.
4436 */
4437
4438 /* ARGSUSED */
4439 static int
4440 dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4441 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4442 crypto_req_handle_t req)
4443 {
4444 int error = CRYPTO_FAILED;
4445 dca_t *softc;
4446 /* LINTED E_FUNC_SET_NOT_USED */
4447 int instance;
4448
4449 /* extract softc and instance number from context */
4450 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4451 DBG(softc, DENTRY, "dca_verify_init: started\n");
4452
4453 if (ctx_template != NULL)
4454 return (CRYPTO_ARGUMENTS_BAD);
4455
4456 /* check mechanism */
4457 switch (mechanism->cm_type) {
4458 case RSA_PKCS_MECH_INFO_TYPE:
4459 case RSA_X_509_MECH_INFO_TYPE:
4460 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4461 break;
4462 case DSA_MECH_INFO_TYPE:
4463 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
4464 DCA_DSA_VRFY);
4465 break;
4466 default:
4467 cmn_err(CE_WARN, "dca_verify_init: unexpected mech type "
4468 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4469 error = CRYPTO_MECHANISM_INVALID;
4470 }
4471
4472 DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error);
4473
4474 if (error == CRYPTO_SUCCESS)
4475 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4476 &softc->dca_ctx_list_lock);
4477
4478 return (error);
4479 }
4480
4481 static int
4482 dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
4483 crypto_req_handle_t req)
4484 {
4485 int error = CRYPTO_FAILED;
4486 dca_t *softc;
4487 /* LINTED E_FUNC_SET_NOT_USED */
4488 int instance;
4489
4490 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4491 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4492
4493 /* extract softc and instance number from context */
4494 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4495 DBG(softc, DENTRY, "dca_verify: started\n");
4496
4497 /* check mechanism */
4498 switch (DCA_MECH_FROM_CTX(ctx)) {
4499 case RSA_PKCS_MECH_INFO_TYPE:
4500 case RSA_X_509_MECH_INFO_TYPE:
4501 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY);
4502 break;
4503 case DSA_MECH_INFO_TYPE:
4504 error = dca_dsa_verify(ctx, data, signature, req);
4505 break;
4506 default:
4507 cmn_err(CE_WARN, "dca_verify: unexpected mech type "
4508 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4509 error = CRYPTO_MECHANISM_INVALID;
4510 }
4511
4512 DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error);
4513
4514 return (error);
4515 }
4516
4517 /* ARGSUSED */
4518 static int
4519 dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
4520 crypto_req_handle_t req)
4521 {
4522 int error = CRYPTO_MECHANISM_INVALID;
4523 dca_t *softc;
4524 /* LINTED E_FUNC_SET_NOT_USED */
4525 int instance;
4526
4527 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4528 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4529
4530 /* extract softc and instance number from context */
4531 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4532 DBG(softc, DENTRY, "dca_verify_update: started\n");
4533
4534 cmn_err(CE_WARN, "dca_verify_update: unexpected mech type "
4535 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4536
4537 DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error);
4538
4539 return (error);
4540 }
4541
4542 /* ARGSUSED */
4543 static int
4544 dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
4545 crypto_req_handle_t req)
4546 {
4547 int error = CRYPTO_MECHANISM_INVALID;
4548 dca_t *softc;
4549 /* LINTED E_FUNC_SET_NOT_USED */
4550 int instance;
4551
4552 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4553 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4554
4555 /* extract softc and instance number from context */
4556 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4557 DBG(softc, DENTRY, "dca_verify_final: started\n");
4558
4559 cmn_err(CE_WARN, "dca_verify_final: unexpected mech type "
4560 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4561
4562 DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error);
4563
4564 return (error);
4565 }
4566
4567 static int
4568 dca_verify_atomic(crypto_provider_handle_t provider,
4569 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4570 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4571 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4572 {
4573 int error = CRYPTO_FAILED;
4574 dca_t *softc = (dca_t *)provider;
4575
4576 DBG(softc, DENTRY, "dca_verify_atomic: started\n");
4577
4578 if (ctx_template != NULL)
4579 return (CRYPTO_ARGUMENTS_BAD);
4580
4581 /* check mechanism */
4582 switch (mechanism->cm_type) {
4583 case RSA_PKCS_MECH_INFO_TYPE:
4584 case RSA_X_509_MECH_INFO_TYPE:
4585 error = dca_rsaatomic(provider, session_id, mechanism, key,
4586 signature, data, KM_SLEEP, req, DCA_RSA_VRFY);
4587 break;
4588 case DSA_MECH_INFO_TYPE:
4589 error = dca_dsaatomic(provider, session_id, mechanism, key,
4590 data, signature, KM_SLEEP, req, DCA_DSA_VRFY);
4591 break;
4592 default:
4593 cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type "
4594 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4595 error = CRYPTO_MECHANISM_INVALID;
4596 }
4597
4598 DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error);
4599
4600 return (error);
4601 }
4602
4603 /* ARGSUSED */
4604 static int
4605 dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4606 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4607 crypto_req_handle_t req)
4608 {
4609 int error = CRYPTO_MECHANISM_INVALID;
4610 dca_t *softc;
4611 /* LINTED E_FUNC_SET_NOT_USED */
4612 int instance;
4613
4614 /* extract softc and instance number from context */
4615 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4616 DBG(softc, DENTRY, "dca_verify_recover_init: started\n");
4617
4618 if (ctx_template != NULL)
4619 return (CRYPTO_ARGUMENTS_BAD);
4620
4621 /* check mechanism */
4622 switch (mechanism->cm_type) {
4623 case RSA_PKCS_MECH_INFO_TYPE:
4624 case RSA_X_509_MECH_INFO_TYPE:
4625 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4626 break;
4627 default:
4628 cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type"
4629 " 0x%llx\n", (unsigned long long)mechanism->cm_type);
4630 }
4631
4632 DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error);
4633
4634 if (error == CRYPTO_SUCCESS)
4635 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4636 &softc->dca_ctx_list_lock);
4637
4638 return (error);
4639 }
4640
4641 static int
4642 dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
4643 crypto_data_t *data, crypto_req_handle_t req)
4644 {
4645 int error = CRYPTO_MECHANISM_INVALID;
4646 dca_t *softc;
4647 /* LINTED E_FUNC_SET_NOT_USED */
4648 int instance;
4649
4650 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4651 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4652
4653 /* extract softc and instance number from context */
4654 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4655 DBG(softc, DENTRY, "dca_verify_recover: started\n");
4656
4657 /* check mechanism */
4658 switch (DCA_MECH_FROM_CTX(ctx)) {
4659 case RSA_PKCS_MECH_INFO_TYPE:
4660 case RSA_X_509_MECH_INFO_TYPE:
4661 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR);
4662 break;
4663 default:
4664 cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type "
4665 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4666 }
4667
4668 DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error);
4669
4670 return (error);
4671 }
4672
4673 static int
4674 dca_verify_recover_atomic(crypto_provider_handle_t provider,
4675 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4676 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4677 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4678 {
4679 int error = CRYPTO_MECHANISM_INVALID;
4680 dca_t *softc = (dca_t *)provider;
4681
4682 DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n");
4683
4684 if (ctx_template != NULL)
4685 return (CRYPTO_ARGUMENTS_BAD);
4686
4687 /* check mechanism */
4688 switch (mechanism->cm_type) {
4689 case RSA_PKCS_MECH_INFO_TYPE:
4690 case RSA_X_509_MECH_INFO_TYPE:
4691 error = dca_rsaatomic(provider, session_id, mechanism, key,
4692 signature, data, KM_SLEEP, req, DCA_RSA_VRFYR);
4693 break;
4694 default:
4695 cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech "
4696 "type 0x%llx\n", (unsigned long long)mechanism->cm_type);
4697 error = CRYPTO_MECHANISM_INVALID;
4698 }
4699
4700 DBG(softc, DENTRY,
4701 "dca_verify_recover_atomic: done, err = 0x%x", error);
4702
4703 return (error);
4704 }
4705
4706 /*
4707 * Random number entry points.
4708 */
4709
4710 /* ARGSUSED */
4711 static int
4712 dca_generate_random(crypto_provider_handle_t provider,
4713 crypto_session_id_t session_id,
4714 uchar_t *buf, size_t len, crypto_req_handle_t req)
4715 {
4716 int error = CRYPTO_FAILED;
4717 dca_t *softc = (dca_t *)provider;
4718 /* LINTED E_FUNC_SET_NOT_USED */
4719 int instance;
4720
4721 instance = ddi_get_instance(softc->dca_dip);
4722 DBG(softc, DENTRY, "dca_generate_random: started");
4723
4724 error = dca_rng(softc, buf, len, req);
4725
4726 DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error);
4727
4728 return (error);
4729 }
4730
4731 /*
4732 * Context management entry points.
4733 */
4734
4735 int
4736 dca_free_context(crypto_ctx_t *ctx)
4737 {
4738 int error = CRYPTO_SUCCESS;
4739 dca_t *softc;
4740 /* LINTED E_FUNC_SET_NOT_USED */
4741 int instance;
4742
4743 /* extract softc and instance number from context */
4744 DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4745 DBG(softc, DENTRY, "dca_free_context: entered");
4746
4747 if (ctx->cc_provider_private == NULL)
4748 return (error);
4749
4750 dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock);
4751
4752 error = dca_free_context_low(ctx);
4753
4754 DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error);
4755
4756 return (error);
4757 }
4758
4759 static int
4760 dca_free_context_low(crypto_ctx_t *ctx)
4761 {
4762 int error = CRYPTO_SUCCESS;
4763
4764 /* check mechanism */
4765 switch (DCA_MECH_FROM_CTX(ctx)) {
4766 case DES_CBC_MECH_INFO_TYPE:
4767 case DES3_CBC_MECH_INFO_TYPE:
4768 dca_3desctxfree(ctx);
4769 break;
4770 case RSA_PKCS_MECH_INFO_TYPE:
4771 case RSA_X_509_MECH_INFO_TYPE:
4772 dca_rsactxfree(ctx);
4773 break;
4774 case DSA_MECH_INFO_TYPE:
4775 dca_dsactxfree(ctx);
4776 break;
4777 default:
4778 /* Should never reach here */
4779 cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type "
4780 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4781 error = CRYPTO_MECHANISM_INVALID;
4782 }
4783
4784 return (error);
4785 }
4786
4787
4788 /* Free any unfreed private context. It is called in detach. */
4789 static void
4790 dca_free_context_list(dca_t *dca)
4791 {
4792 dca_listnode_t *node;
4793 crypto_ctx_t ctx;
4794
4795 (void) memset(&ctx, 0, sizeof (ctx));
4796 ctx.cc_provider = dca;
4797
4798 while ((node = dca_delist2(&dca->dca_ctx_list,
4799 &dca->dca_ctx_list_lock)) != NULL) {
4800 ctx.cc_provider_private = node;
4801 (void) dca_free_context_low(&ctx);
4802 }
4803 }
4804
4805 static int
4806 ext_info_sym(crypto_provider_handle_t prov,
4807 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
4808 {
4809 return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM));
4810 }
4811
4812 static int
4813 ext_info_asym(crypto_provider_handle_t prov,
4814 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
4815 {
4816 int rv;
4817
4818 rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM);
4819 /* The asymmetric cipher slot supports random */
4820 ext_info->ei_flags |= CRYPTO_EXTF_RNG;
4821
4822 return (rv);
4823 }
4824
4825 /* ARGSUSED */
4826 static int
4827 ext_info_base(crypto_provider_handle_t prov,
4828 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id)
4829 {
4830 dca_t *dca = (dca_t *)prov;
4831 int len;
4832
4833 /* Label */
4834 (void) sprintf((char *)ext_info->ei_label, "%s/%d %s",
4835 ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id);
4836 len = strlen((char *)ext_info->ei_label);
4837 (void) memset(ext_info->ei_label + len, ' ',
4838 CRYPTO_EXT_SIZE_LABEL - len);
4839
4840 /* Manufacturer ID */
4841 (void) sprintf((char *)ext_info->ei_manufacturerID, "%s",
4842 DCA_MANUFACTURER_ID);
4843 len = strlen((char *)ext_info->ei_manufacturerID);
4844 (void) memset(ext_info->ei_manufacturerID + len, ' ',
4845 CRYPTO_EXT_SIZE_MANUF - len);
4846
4847 /* Model */
4848 (void) sprintf((char *)ext_info->ei_model, dca->dca_model);
4849
4850 DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model);
4851
4852 len = strlen((char *)ext_info->ei_model);
4853 (void) memset(ext_info->ei_model + len, ' ',
4854 CRYPTO_EXT_SIZE_MODEL - len);
4855
4856 /* Serial Number. Blank for Deimos */
4857 (void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL);
4858
4859 ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED;
4860
4861 ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO;
4862 ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO;
4863 ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO;
4864 ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO;
4865 ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO;
4866 ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO;
4867 ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO;
4868 ext_info->ei_hardware_version.cv_major = 0;
4869 ext_info->ei_hardware_version.cv_minor = 0;
4870 ext_info->ei_firmware_version.cv_major = 0;
4871 ext_info->ei_firmware_version.cv_minor = 0;
4872
4873 /* Time. No need to be supplied for token without a clock */
4874 ext_info->ei_time[0] = '\000';
4875
4876 return (CRYPTO_SUCCESS);
4877 }
4878
4879 static void
4880 dca_fma_init(dca_t *dca)
4881 {
4882 ddi_iblock_cookie_t fm_ibc;
4883 int fm_capabilities = DDI_FM_EREPORT_CAPABLE |
4884 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE |
4885 DDI_FM_ERRCB_CAPABLE;
4886
4887 /* Read FMA capabilities from dca.conf file (if present) */
4888 dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip,
4889 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
4890 fm_capabilities);
4891
4892 DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities);
4893
4894 /* Only register with IO Fault Services if we have some capability */
4895 if (dca->fm_capabilities) {
4896 dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC;
4897 dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR;
4898
4899 /* Register capabilities with IO Fault Services */
4900 ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc);
4901 DBG(dca, DWARN, "fm_capable() = 0x%x",
4902 ddi_fm_capable(dca->dca_dip));
4903
4904 /*
4905 * Initialize pci ereport capabilities if ereport capable
4906 */
4907 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
4908 DDI_FM_ERRCB_CAP(dca->fm_capabilities))
4909 pci_ereport_setup(dca->dca_dip);
4910
4911 /*
4912 * Initialize callback mutex and register error callback if
4913 * error callback capable.
4914 */
4915 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4916 ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb,
4917 (void *)dca);
4918 }
4919 } else {
4920 /*
4921 * These fields have to be cleared of FMA if there are no
4922 * FMA capabilities at runtime.
4923 */
4924 dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC;
4925 dca_dmaattr.dma_attr_flags = 0;
4926 }
4927 }
4928
4929
4930 static void
4931 dca_fma_fini(dca_t *dca)
4932 {
4933 /* Only unregister FMA capabilities if we registered some */
4934 if (dca->fm_capabilities) {
4935
4936 /*
4937 * Release any resources allocated by pci_ereport_setup()
4938 */
4939 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
4940 DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4941 pci_ereport_teardown(dca->dca_dip);
4942 }
4943
4944 /*
4945 * Free callback mutex and un-register error callback if
4946 * error callback capable.
4947 */
4948 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4949 ddi_fm_handler_unregister(dca->dca_dip);
4950 }
4951
4952 /* Unregister from IO Fault Services */
4953 ddi_fm_fini(dca->dca_dip);
4954 DBG(dca, DWARN, "fm_capable() = 0x%x",
4955 ddi_fm_capable(dca->dca_dip));
4956 }
4957 }
4958
4959
4960 /*
4961 * The IO fault service error handling callback function
4962 */
4963 /*ARGSUSED*/
4964 static int
4965 dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
4966 {
4967 dca_t *dca = (dca_t *)impl_data;
4968
4969 pci_ereport_post(dip, err, NULL);
4970 if (err->fme_status == DDI_FM_FATAL) {
4971 dca_failure(dca, DDI_DATAPATH_FAULT,
4972 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
4973 "fault PCI in FMA callback.");
4974 }
4975 return (err->fme_status);
4976 }
4977
4978
4979 static int
4980 dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
4981 dca_fma_eclass_t eclass_index)
4982 {
4983 ddi_fm_error_t de;
4984 int version = 0;
4985
4986 ddi_fm_acc_err_get(handle, &de, version);
4987 if (de.fme_status != DDI_FM_OK) {
4988 dca_failure(dca, DDI_DATAPATH_FAULT,
4989 eclass_index, fm_ena_increment(de.fme_ena),
4990 CRYPTO_DEVICE_ERROR, "");
4991 return (DDI_FAILURE);
4992 }
4993
4994 return (DDI_SUCCESS);
4995 }
4996
4997 int
4998 dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle,
4999 dca_fma_eclass_t eclass_index)
5000 {
5001 ddi_fm_error_t de;
5002 int version = 0;
5003
5004 ddi_fm_dma_err_get(handle, &de, version);
5005 if (de.fme_status != DDI_FM_OK) {
5006 dca_failure(dca, DDI_DATAPATH_FAULT,
5007 eclass_index, fm_ena_increment(de.fme_ena),
5008 CRYPTO_DEVICE_ERROR, "");
5009 return (DDI_FAILURE);
5010 }
5011 return (DDI_SUCCESS);
5012 }
5013
5014 static uint64_t
5015 dca_ena(uint64_t ena)
5016 {
5017 if (ena == 0)
5018 ena = fm_ena_generate(0, FM_ENA_FMT1);
5019 else
5020 ena = fm_ena_increment(ena);
5021 return (ena);
5022 }
5023
5024 static char *
5025 dca_fma_eclass_string(char *model, dca_fma_eclass_t index)
5026 {
5027 if (strstr(model, "500"))
5028 return (dca_fma_eclass_sca500[index]);
5029 else
5030 return (dca_fma_eclass_sca1000[index]);
5031 }