1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27
28 /*
29 * Deimos - cryptographic acceleration based upon Broadcom 582x.
30 */
31
32 #include <sys/types.h>
33 #include <sys/modctl.h>
34 #include <sys/conf.h>
35 #include <sys/devops.h>
36 #include <sys/ddi.h>
37 #include <sys/sunddi.h>
38 #include <sys/cmn_err.h>
39 #include <sys/varargs.h>
40 #include <sys/file.h>
41 #include <sys/stat.h>
42 #include <sys/kmem.h>
43 #include <sys/ioccom.h>
44 #include <sys/open.h>
45 #include <sys/cred.h>
46 #include <sys/kstat.h>
47 #include <sys/strsun.h>
48 #include <sys/note.h>
49 #include <sys/crypto/common.h>
50 #include <sys/crypto/spi.h>
51 #include <sys/ddifm.h>
52 #include <sys/fm/protocol.h>
53 #include <sys/fm/util.h>
54 #include <sys/fm/io/ddi.h>
55 #include <sys/crypto/dca.h>
56
57 /*
58 * Core Deimos driver.
59 */
60
61 static void dca_enlist2(dca_listnode_t *, dca_listnode_t *,
62 kmutex_t *);
63 static void dca_rmlist2(dca_listnode_t *node, kmutex_t *);
64 static dca_listnode_t *dca_delist2(dca_listnode_t *q, kmutex_t *);
65 static void dca_free_context_list(dca_t *dca);
66 static int dca_free_context_low(crypto_ctx_t *ctx);
67 static int dca_attach(dev_info_t *, ddi_attach_cmd_t);
68 static int dca_detach(dev_info_t *, ddi_detach_cmd_t);
69 static int dca_suspend(dca_t *);
70 static int dca_resume(dca_t *);
71 static int dca_init(dca_t *);
72 static int dca_reset(dca_t *, int);
73 static int dca_initworklist(dca_t *, dca_worklist_t *);
74 static void dca_uninit(dca_t *);
75 static void dca_initq(dca_listnode_t *);
76 static void dca_enqueue(dca_listnode_t *, dca_listnode_t *);
77 static dca_listnode_t *dca_dequeue(dca_listnode_t *);
78 static dca_listnode_t *dca_unqueue(dca_listnode_t *);
79 static dca_request_t *dca_newreq(dca_t *);
80 static dca_work_t *dca_getwork(dca_t *, int);
81 static void dca_freework(dca_work_t *);
82 static dca_work_t *dca_newwork(dca_t *);
83 static void dca_destroywork(dca_work_t *);
84 static void dca_schedule(dca_t *, int);
85 static void dca_reclaim(dca_t *, int);
86 static uint_t dca_intr(char *);
87 static void dca_failure(dca_t *, ddi_fault_location_t,
88 dca_fma_eclass_t index, uint64_t, int, char *, ...);
89 static void dca_jobtimeout(void *);
90 static int dca_drain(dca_t *);
91 static void dca_undrain(dca_t *);
92 static void dca_rejectjobs(dca_t *);
93
94 #ifdef SCHEDDELAY
95 static void dca_schedtimeout(void *);
96 #endif
97
98 /*
99 * We want these inlined for performance.
100 */
101 #ifndef DEBUG
102 #pragma inline(dca_freereq, dca_getreq, dca_freework, dca_getwork)
103 #pragma inline(dca_enqueue, dca_dequeue, dca_rmqueue, dca_done)
104 #pragma inline(dca_reverse, dca_length)
105 #endif
106
107 /*
108 * Device operations.
109 */
110 static struct dev_ops devops = {
111 DEVO_REV, /* devo_rev */
112 0, /* devo_refcnt */
113 nodev, /* devo_getinfo */
114 nulldev, /* devo_identify */
115 nulldev, /* devo_probe */
116 dca_attach, /* devo_attach */
117 dca_detach, /* devo_detach */
118 nodev, /* devo_reset */
119 NULL, /* devo_cb_ops */
120 NULL, /* devo_bus_ops */
121 ddi_power, /* devo_power */
122 ddi_quiesce_not_supported, /* devo_quiesce */
123 };
124
125 #define IDENT "PCI Crypto Accelerator"
126 #define IDENT_SYM "Crypto Accel Sym 2.0"
127 #define IDENT_ASYM "Crypto Accel Asym 2.0"
128
129 /* Space-padded, will be filled in dynamically during registration */
130 #define IDENT3 "PCI Crypto Accelerator Mod 2.0"
131
132 #define VENDOR "Sun Microsystems, Inc."
133
134 #define STALETIME (30 * SECOND)
135
136 #define crypto_prov_notify crypto_provider_notification
137 /* A 28 char function name doesn't leave much line space */
138
139 /*
140 * Module linkage.
141 */
142 static struct modldrv modldrv = {
143 &mod_driverops, /* drv_modops */
144 IDENT, /* drv_linkinfo */
145 &devops, /* drv_dev_ops */
146 };
147
148 extern struct mod_ops mod_cryptoops;
149
150 static struct modlcrypto modlcrypto = {
151 &mod_cryptoops,
152 IDENT3
153 };
154
155 static struct modlinkage modlinkage = {
156 MODREV_1, /* ml_rev */
157 { &modldrv, /* ml_linkage */
158 &modlcrypto,
159 NULL }
160 };
161
162 /*
163 * CSPI information (entry points, provider info, etc.)
164 */
165
166 /* Mechanisms for the symmetric cipher provider */
167 static crypto_mech_info_t dca_mech_info_tab1[] = {
168 /* DES-CBC */
169 {SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE,
170 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
171 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
172 DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
173 /* 3DES-CBC */
174 {SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE,
175 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
176 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
177 DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}
178 };
179
180 /* Mechanisms for the asymmetric cipher provider */
181 static crypto_mech_info_t dca_mech_info_tab2[] = {
182 /* DSA */
183 {SUN_CKM_DSA, DSA_MECH_INFO_TYPE,
184 CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY |
185 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC,
186 CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN),
187 CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN),
188 CRYPTO_KEYSIZE_UNIT_IN_BITS},
189
190 /* RSA */
191 {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
192 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
193 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
194 CRYPTO_FG_VERIFY_RECOVER |
195 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
196 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
197 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
198 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
199 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
200 CRYPTO_KEYSIZE_UNIT_IN_BITS},
201 {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
202 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
203 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
204 CRYPTO_FG_VERIFY_RECOVER |
205 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
206 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
207 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
208 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
209 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
210 CRYPTO_KEYSIZE_UNIT_IN_BITS}
211 };
212
213 static void dca_provider_status(crypto_provider_handle_t, uint_t *);
214
215 static crypto_control_ops_t dca_control_ops = {
216 dca_provider_status
217 };
218
219 static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
220 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
221 static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
222 crypto_req_handle_t);
223 static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *,
224 crypto_data_t *, crypto_req_handle_t);
225 static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *,
226 crypto_req_handle_t);
227 static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
228 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
229 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
230
231 static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
232 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
233 static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
234 crypto_req_handle_t);
235 static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *,
236 crypto_data_t *, crypto_req_handle_t);
237 static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *,
238 crypto_req_handle_t);
239 static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
240 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
241 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
242
243 static crypto_cipher_ops_t dca_cipher_ops = {
244 dca_encrypt_init,
245 dca_encrypt,
246 dca_encrypt_update,
247 dca_encrypt_final,
248 dca_encrypt_atomic,
249 dca_decrypt_init,
250 dca_decrypt,
251 dca_decrypt_update,
252 dca_decrypt_final,
253 dca_decrypt_atomic
254 };
255
256 static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
257 crypto_spi_ctx_template_t, crypto_req_handle_t);
258 static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
259 crypto_req_handle_t);
260 static int dca_sign_update(crypto_ctx_t *, crypto_data_t *,
261 crypto_req_handle_t);
262 static int dca_sign_final(crypto_ctx_t *, crypto_data_t *,
263 crypto_req_handle_t);
264 static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
265 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
266 crypto_spi_ctx_template_t, crypto_req_handle_t);
267 static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
268 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
269 static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
270 crypto_req_handle_t);
271 static int dca_sign_recover_atomic(crypto_provider_handle_t,
272 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
273 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
274
275 static crypto_sign_ops_t dca_sign_ops = {
276 dca_sign_init,
277 dca_sign,
278 dca_sign_update,
279 dca_sign_final,
280 dca_sign_atomic,
281 dca_sign_recover_init,
282 dca_sign_recover,
283 dca_sign_recover_atomic
284 };
285
286 static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *,
287 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
288 static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
289 crypto_req_handle_t);
290 static int dca_verify_update(crypto_ctx_t *, crypto_data_t *,
291 crypto_req_handle_t);
292 static int dca_verify_final(crypto_ctx_t *, crypto_data_t *,
293 crypto_req_handle_t);
294 static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
295 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
296 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
297 static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
298 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
299 static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *,
300 crypto_data_t *, crypto_req_handle_t);
301 static int dca_verify_recover_atomic(crypto_provider_handle_t,
302 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
303 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
304
305 static crypto_verify_ops_t dca_verify_ops = {
306 dca_verify_init,
307 dca_verify,
308 dca_verify_update,
309 dca_verify_final,
310 dca_verify_atomic,
311 dca_verify_recover_init,
312 dca_verify_recover,
313 dca_verify_recover_atomic
314 };
315
316 static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t,
317 uchar_t *, size_t, crypto_req_handle_t);
318
319 static crypto_random_number_ops_t dca_random_number_ops = {
320 NULL,
321 dca_generate_random
322 };
323
324 static int ext_info_sym(crypto_provider_handle_t prov,
325 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
326 static int ext_info_asym(crypto_provider_handle_t prov,
327 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
328 static int ext_info_base(crypto_provider_handle_t prov,
329 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id);
330
331 static crypto_provider_management_ops_t dca_provmanage_ops_1 = {
332 ext_info_sym, /* ext_info */
333 NULL, /* init_token */
334 NULL, /* init_pin */
335 NULL /* set_pin */
336 };
337
338 static crypto_provider_management_ops_t dca_provmanage_ops_2 = {
339 ext_info_asym, /* ext_info */
340 NULL, /* init_token */
341 NULL, /* init_pin */
342 NULL /* set_pin */
343 };
344
345 int dca_free_context(crypto_ctx_t *);
346
347 static crypto_ctx_ops_t dca_ctx_ops = {
348 NULL,
349 dca_free_context
350 };
351
352 /* Operations for the symmetric cipher provider */
353 static crypto_ops_t dca_crypto_ops1 = {
354 .co_control_ops = &dca_control_ops,
355 .co_digest_ops = NULL,
356 .co_cipher_ops = &dca_cipher_ops,
357 .co_mac_ops = NULL,
358 .co_sign_ops = NULL,
359 .co_verify_ops = NULL,
360 .co_dual_ops = NULL,
361 .co_dual_cipher_mac_ops = NULL,
362 .co_random_ops = NULL,
363 .co_session_ops = NULL,
364 .co_object_ops = NULL,
365 .co_key_ops = NULL,
366 .co_provider_ops = &dca_provmanage_ops_1,
367 .co_ctx_ops = &dca_ctx_ops
368 };
369
370 /* Operations for the asymmetric cipher provider */
371 static crypto_ops_t dca_crypto_ops2 = { .cou.cou_v1 = {
372 &dca_control_ops,
373 NULL, /* digest_ops */
374 &dca_cipher_ops,
375 NULL, /* mac_ops */
376 &dca_sign_ops,
377 &dca_verify_ops,
378 NULL, /* dual_ops */
379 NULL, /* cipher_mac_ops */
380 &dca_random_number_ops,
381 NULL, /* session_ops */
382 NULL, /* object_ops */
383 NULL, /* key_ops */
384 &dca_provmanage_ops_2, /* management_ops */
385 &dca_ctx_ops
386 }};
387
388 /* Provider information for the symmetric cipher provider */
389 static crypto_provider_info_t dca_prov_info1 = {{{{
390 CRYPTO_SPI_VERSION_1,
391 NULL, /* pi_provider_description */
392 CRYPTO_HW_PROVIDER,
393 { NULL }, /* pi_provider_dev */
394 NULL, /* pi_provider_handle */
395 &dca_crypto_ops1,
396 sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t),
397 dca_mech_info_tab1,
398 0, /* pi_logical_provider_count */
399 NULL /* pi_logical_providers */
400 }}}};
401
402 /* Provider information for the asymmetric cipher provider */
403 static crypto_provider_info_t dca_prov_info2 = {{{{
404 CRYPTO_SPI_VERSION_1,
405 NULL, /* pi_provider_description */
406 CRYPTO_HW_PROVIDER,
407 { NULL }, /* pi_provider_dev */
408 NULL, /* pi_provider_handle */
409 &dca_crypto_ops2,
410 sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t),
411 dca_mech_info_tab2,
412 0, /* pi_logical_provider_count */
413 NULL /* pi_logical_providers */
414 }}}};
415
416 /* Convenience macros */
417 #define DCA_SOFTC_FROM_CTX(ctx) ((dca_t *)(ctx)->cc_provider)
418 #define DCA_MECH_FROM_CTX(ctx) \
419 (((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type)
420
421 static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
422 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
423 dca_chain_t *head, int *n_chain);
424 static uint64_t dca_ena(uint64_t ena);
425 static caddr_t dca_bufdaddr_out(crypto_data_t *data);
426 static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index);
427 static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
428 dca_fma_eclass_t eclass_index);
429
430 static void dca_fma_init(dca_t *dca);
431 static void dca_fma_fini(dca_t *dca);
432 static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
433 const void *impl_data);
434
435
436 static dca_device_t dca_devices[] = {
437 /* Broadcom vanilla variants */
438 { 0x14e4, 0x5820, "Broadcom 5820" },
439 { 0x14e4, 0x5821, "Broadcom 5821" },
440 { 0x14e4, 0x5822, "Broadcom 5822" },
441 { 0x14e4, 0x5825, "Broadcom 5825" },
442 /* Sun specific OEMd variants */
443 { 0x108e, 0x5454, "SCA" },
444 { 0x108e, 0x5455, "SCA 1000" },
445 { 0x108e, 0x5457, "SCA 500" },
446 /* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */
447 { 0x108e, 0x1, "SCA 500" },
448 };
449
450 /*
451 * Device attributes.
452 */
453 static struct ddi_device_acc_attr dca_regsattr = {
454 DDI_DEVICE_ATTR_V1,
455 DDI_STRUCTURE_LE_ACC,
456 DDI_STRICTORDER_ACC,
457 DDI_FLAGERR_ACC
458 };
459
460 static struct ddi_device_acc_attr dca_devattr = {
461 DDI_DEVICE_ATTR_V0,
462 DDI_STRUCTURE_LE_ACC,
463 DDI_STRICTORDER_ACC
464 };
465
466 #if !defined(i386) && !defined(__i386)
467 static struct ddi_device_acc_attr dca_bufattr = {
468 DDI_DEVICE_ATTR_V0,
469 DDI_NEVERSWAP_ACC,
470 DDI_STRICTORDER_ACC
471 };
472 #endif
473
474 static struct ddi_dma_attr dca_dmaattr = {
475 DMA_ATTR_V0, /* dma_attr_version */
476 0x0, /* dma_attr_addr_lo */
477 0xffffffffUL, /* dma_attr_addr_hi */
478 0x00ffffffUL, /* dma_attr_count_max */
479 0x40, /* dma_attr_align */
480 0x40, /* dma_attr_burstsizes */
481 0x1, /* dma_attr_minxfer */
482 0x00ffffffUL, /* dma_attr_maxxfer */
483 0xffffffffUL, /* dma_attr_seg */
484 #if defined(i386) || defined(__i386) || defined(__amd64)
485 512, /* dma_attr_sgllen */
486 #else
487 1, /* dma_attr_sgllen */
488 #endif
489 1, /* dma_attr_granular */
490 DDI_DMA_FLAGERR /* dma_attr_flags */
491 };
492
493 static void *dca_state = NULL;
494 int dca_mindma = 2500;
495
496 /*
497 * FMA eclass string definitions. Note that these string arrays must be
498 * consistent with the dca_fma_eclass_t enum.
499 */
500 static char *dca_fma_eclass_sca1000[] = {
501 "sca1000.hw.device",
502 "sca1000.hw.timeout",
503 "sca1000.none"
504 };
505
506 static char *dca_fma_eclass_sca500[] = {
507 "sca500.hw.device",
508 "sca500.hw.timeout",
509 "sca500.none"
510 };
511
512 /*
513 * DDI entry points.
514 */
515 int
516 _init(void)
517 {
518 int rv;
519
520 DBG(NULL, DMOD, "dca: in _init");
521
522 if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) {
523 /* this should *never* happen! */
524 return (rv);
525 }
526
527 if ((rv = mod_install(&modlinkage)) != 0) {
528 /* cleanup here */
529 ddi_soft_state_fini(&dca_state);
530 return (rv);
531 }
532
533 return (0);
534 }
535
536 int
537 _fini(void)
538 {
539 int rv;
540
541 DBG(NULL, DMOD, "dca: in _fini");
542
543 if ((rv = mod_remove(&modlinkage)) == 0) {
544 /* cleanup here */
545 ddi_soft_state_fini(&dca_state);
546 }
547 return (rv);
548 }
549
550 int
551 _info(struct modinfo *modinfop)
552 {
553 DBG(NULL, DMOD, "dca: in _info");
554
555 return (mod_info(&modlinkage, modinfop));
556 }
557
558 int
559 dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
560 {
561 ddi_acc_handle_t pci;
562 int instance;
563 ddi_iblock_cookie_t ibc;
564 int intr_added = 0;
565 dca_t *dca;
566 ushort_t venid;
567 ushort_t devid;
568 ushort_t revid;
569 ushort_t subsysid;
570 ushort_t subvenid;
571 int i;
572 int ret;
573 char ID[64];
574 static char *unknowndev = "Unknown device";
575
576 #if DEBUG
577 /* these are only used for debugging */
578 ushort_t pcicomm;
579 ushort_t pcistat;
580 uchar_t cachelinesz;
581 uchar_t mingnt;
582 uchar_t maxlat;
583 uchar_t lattmr;
584 #endif
585
586 instance = ddi_get_instance(dip);
587
588 DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance);
589
590 switch (cmd) {
591 case DDI_RESUME:
592 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
593 dca_diperror(dip, "no soft state in detach");
594 return (DDI_FAILURE);
595 }
596 /* assumption: we won't be DDI_DETACHed until we return */
597 return (dca_resume(dca));
598 case DDI_ATTACH:
599 break;
600 default:
601 return (DDI_FAILURE);
602 }
603
604 if (ddi_slaveonly(dip) == DDI_SUCCESS) {
605 dca_diperror(dip, "slot does not support PCI bus-master");
606 return (DDI_FAILURE);
607 }
608
609 if (ddi_intr_hilevel(dip, 0) != 0) {
610 dca_diperror(dip, "hilevel interrupts not supported");
611 return (DDI_FAILURE);
612 }
613
614 if (pci_config_setup(dip, &pci) != DDI_SUCCESS) {
615 dca_diperror(dip, "unable to setup PCI config handle");
616 return (DDI_FAILURE);
617 }
618
619 /* common PCI attributes */
620 venid = pci_config_get16(pci, PCI_VENID);
621 devid = pci_config_get16(pci, PCI_DEVID);
622 revid = pci_config_get8(pci, PCI_REVID);
623 subvenid = pci_config_get16(pci, PCI_SUBVENID);
624 subsysid = pci_config_get16(pci, PCI_SUBSYSID);
625
626 /*
627 * Broadcom-specific timings.
628 * We disable these timers/counters since they can cause
629 * incorrect false failures when the bus is just a little
630 * bit slow, or busy.
631 */
632 pci_config_put8(pci, PCI_TRDYTO, 0);
633 pci_config_put8(pci, PCI_RETRIES, 0);
634
635 /* initialize PCI access settings */
636 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
637 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);
638
639 /* set up our PCI latency timer */
640 pci_config_put8(pci, PCI_LATTMR, 0x40);
641
642 #if DEBUG
643 /* read registers (for debugging) */
644 pcicomm = pci_config_get16(pci, PCI_COMM);
645 pcistat = pci_config_get16(pci, PCI_STATUS);
646 cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ);
647 mingnt = pci_config_get8(pci, PCI_MINGNT);
648 maxlat = pci_config_get8(pci, PCI_MAXLAT);
649 lattmr = pci_config_get8(pci, PCI_LATTMR);
650 #endif
651
652 pci_config_teardown(&pci);
653
654 if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) {
655 dca_diperror(dip, "unable to get iblock cookie");
656 return (DDI_FAILURE);
657 }
658
659 if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) {
660 dca_diperror(dip, "unable to allocate soft state");
661 return (DDI_FAILURE);
662 }
663
664 dca = ddi_get_soft_state(dca_state, instance);
665 ASSERT(dca != NULL);
666 dca->dca_dip = dip;
667 WORKLIST(dca, MCR1)->dwl_prov = NULL;
668 WORKLIST(dca, MCR2)->dwl_prov = NULL;
669 /* figure pagesize */
670 dca->dca_pagesize = ddi_ptob(dip, 1);
671
672 /*
673 * Search for the device in our supported devices table. This
674 * is here for two reasons. First, we want to ensure that
675 * only Sun-qualified (and presumably Sun-labeled) devices can
676 * be used with this driver. Second, some devices have
677 * specific differences. E.g. the 5821 has support for a
678 * special mode of RC4, deeper queues, power management, and
679 * other changes. Also, the export versions of some of these
680 * chips don't support RC4 or 3DES, so we catch that here.
681 *
682 * Note that we only look at the upper nibble of the device
683 * id, which is used to distinguish export vs. domestic
684 * versions of the chip. (The lower nibble is used for
685 * stepping information.)
686 */
687 for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) {
688 /*
689 * Try to match the subsystem information first.
690 */
691 if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) &&
692 subsysid && (subsysid == dca_devices[i].dd_device_id)) {
693 dca->dca_model = dca_devices[i].dd_model;
694 dca->dca_devid = dca_devices[i].dd_device_id;
695 break;
696 }
697 /*
698 * Failing that, try the generic vendor and device id.
699 * Even if we find a match, we keep searching anyway,
700 * since we would prefer to find a match based on the
701 * subsystem ids.
702 */
703 if ((venid == dca_devices[i].dd_vendor_id) &&
704 (devid == dca_devices[i].dd_device_id)) {
705 dca->dca_model = dca_devices[i].dd_model;
706 dca->dca_devid = dca_devices[i].dd_device_id;
707 }
708 }
709 /* try and handle an unrecognized device */
710 if (dca->dca_model == NULL) {
711 dca->dca_model = unknowndev;
712 dca_error(dca, "device not recognized, not supported");
713 DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d",
714 i, venid, devid, revid);
715 }
716
717 if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description",
718 dca->dca_model) != DDI_SUCCESS) {
719 dca_error(dca, "unable to create description property");
720 return (DDI_FAILURE);
721 }
722
723 DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x",
724 pcicomm, pcistat, cachelinesz);
725 DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x",
726 mingnt, maxlat, lattmr);
727
728 /*
729 * initialize locks, etc.
730 */
731 (void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc);
732
733 /* use RNGSHA1 by default */
734 if (ddi_getprop(DDI_DEV_T_ANY, dip,
735 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) {
736 dca->dca_flags |= DCA_RNGSHA1;
737 }
738
739 /* initialize FMA */
740 dca_fma_init(dca);
741
742 /* initialize some key data structures */
743 if (dca_init(dca) != DDI_SUCCESS) {
744 goto failed;
745 }
746
747 /* initialize kstats */
748 dca_ksinit(dca);
749
750 /* setup access to registers */
751 if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs,
752 0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) {
753 dca_error(dca, "unable to map registers");
754 goto failed;
755 }
756
757 DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1));
758 DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL));
759 DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT));
760 DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA));
761 DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2));
762
763 /* reset the chip */
764 if (dca_reset(dca, 0) < 0) {
765 goto failed;
766 }
767
768 /* initialize the chip */
769 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
770 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
771 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
772 goto failed;
773 }
774
775 /* add the interrupt */
776 if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr,
777 (void *)dca) != DDI_SUCCESS) {
778 DBG(dca, DWARN, "ddi_add_intr failed");
779 goto failed;
780 } else {
781 intr_added = 1;
782 }
783
784 /* enable interrupts on the device */
785 /*
786 * XXX: Note, 5820A1 errata indicates that this may clobber
787 * bits 24 and 23, which affect the speed of the RNG. Since
788 * we always want to run in full-speed mode, this should be
789 * harmless.
790 */
791 if (dca->dca_devid == 0x5825) {
792 /* for 5825 - increase the DMA read size */
793 SETBIT(dca, CSR_DMACTL,
794 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
795 } else {
796 SETBIT(dca, CSR_DMACTL,
797 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
798 }
799 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
800 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
801 goto failed;
802 }
803
804 /* register MCR1 with the crypto framework */
805 /* Be careful not to exceed 32 chars */
806 (void) sprintf(ID, "%s/%d %s",
807 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM);
808 dca_prov_info1.pi_provider_description = ID;
809 dca_prov_info1.pi_provider_dev.pd_hw = dip;
810 dca_prov_info1.pi_provider_handle = dca;
811 if ((ret = crypto_register_provider(&dca_prov_info1,
812 &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) {
813 cmn_err(CE_WARN,
814 "crypto_register_provider() failed (%d) for MCR1", ret);
815 goto failed;
816 }
817
818 /* register MCR2 with the crypto framework */
819 /* Be careful not to exceed 32 chars */
820 (void) sprintf(ID, "%s/%d %s",
821 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM);
822 dca_prov_info2.pi_provider_description = ID;
823 dca_prov_info2.pi_provider_dev.pd_hw = dip;
824 dca_prov_info2.pi_provider_handle = dca;
825 if ((ret = crypto_register_provider(&dca_prov_info2,
826 &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) {
827 cmn_err(CE_WARN,
828 "crypto_register_provider() failed (%d) for MCR2", ret);
829 goto failed;
830 }
831
832 crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov,
833 CRYPTO_PROVIDER_READY);
834 crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov,
835 CRYPTO_PROVIDER_READY);
836
837 /* Initialize the local random number pool for this instance */
838 if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) {
839 goto failed;
840 }
841
842 mutex_enter(&dca->dca_intrlock);
843 dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca,
844 drv_usectohz(SECOND));
845 mutex_exit(&dca->dca_intrlock);
846
847 ddi_set_driver_private(dip, (caddr_t)dca);
848
849 ddi_report_dev(dip);
850
851 if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) {
852 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED);
853 }
854
855 return (DDI_SUCCESS);
856
857 failed:
858 /* unregister from the crypto framework */
859 if (WORKLIST(dca, MCR1)->dwl_prov != NULL) {
860 (void) crypto_unregister_provider(
861 WORKLIST(dca, MCR1)->dwl_prov);
862 }
863 if (WORKLIST(dca, MCR2)->dwl_prov != NULL) {
864 (void) crypto_unregister_provider(
865 WORKLIST(dca, MCR2)->dwl_prov);
866 }
867 if (intr_added) {
868 CLRBIT(dca, CSR_DMACTL,
869 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
870 /* unregister intr handler */
871 ddi_remove_intr(dip, 0, dca->dca_icookie);
872 }
873 if (dca->dca_regs_handle) {
874 ddi_regs_map_free(&dca->dca_regs_handle);
875 }
876 if (dca->dca_intrstats) {
877 kstat_delete(dca->dca_intrstats);
878 }
879 if (dca->dca_ksp) {
880 kstat_delete(dca->dca_ksp);
881 }
882 dca_uninit(dca);
883
884 /* finalize FMA */
885 dca_fma_fini(dca);
886
887 mutex_destroy(&dca->dca_intrlock);
888 ddi_soft_state_free(dca_state, instance);
889 return (DDI_FAILURE);
890
891 }
892
893 int
894 dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
895 {
896 int instance;
897 dca_t *dca;
898 timeout_id_t tid;
899
900 instance = ddi_get_instance(dip);
901
902 DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance);
903
904 switch (cmd) {
905 case DDI_SUSPEND:
906 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
907 dca_diperror(dip, "no soft state in detach");
908 return (DDI_FAILURE);
909 }
910 /* assumption: we won't be DDI_DETACHed until we return */
911 return (dca_suspend(dca));
912
913 case DDI_DETACH:
914 break;
915 default:
916 return (DDI_FAILURE);
917 }
918
919 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
920 dca_diperror(dip, "no soft state in detach");
921 return (DDI_FAILURE);
922 }
923
924 /*
925 * Unregister from kCF.
926 * This needs to be done at the beginning of detach.
927 */
928 if (WORKLIST(dca, MCR1)->dwl_prov != NULL) {
929 if (crypto_unregister_provider(
930 WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) {
931 dca_error(dca, "unable to unregister MCR1 from kcf");
932 return (DDI_FAILURE);
933 }
934 }
935
936 if (WORKLIST(dca, MCR2)->dwl_prov != NULL) {
937 if (crypto_unregister_provider(
938 WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) {
939 dca_error(dca, "unable to unregister MCR2 from kcf");
940 return (DDI_FAILURE);
941 }
942 }
943
944 /*
945 * Cleanup the private context list. Once the
946 * crypto_unregister_provider returns, it is safe to do so.
947 */
948 dca_free_context_list(dca);
949
950 /* Cleanup the local random number pool */
951 dca_random_fini(dca);
952
953 /* send any jobs in the waitq back to kCF */
954 dca_rejectjobs(dca);
955
956 /* untimeout the timeouts */
957 mutex_enter(&dca->dca_intrlock);
958 tid = dca->dca_jobtid;
959 dca->dca_jobtid = 0;
960 mutex_exit(&dca->dca_intrlock);
961 if (tid) {
962 (void) untimeout(tid);
963 }
964
965 /* disable device interrupts */
966 CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
967
968 /* unregister interrupt handlers */
969 ddi_remove_intr(dip, 0, dca->dca_icookie);
970
971 /* release our regs handle */
972 ddi_regs_map_free(&dca->dca_regs_handle);
973
974 /* toss out kstats */
975 if (dca->dca_intrstats) {
976 kstat_delete(dca->dca_intrstats);
977 }
978 if (dca->dca_ksp) {
979 kstat_delete(dca->dca_ksp);
980 }
981
982 mutex_destroy(&dca->dca_intrlock);
983 dca_uninit(dca);
984
985 /* finalize FMA */
986 dca_fma_fini(dca);
987
988 ddi_soft_state_free(dca_state, instance);
989
990 return (DDI_SUCCESS);
991 }
992
993 int
994 dca_resume(dca_t *dca)
995 {
996 ddi_acc_handle_t pci;
997
998 if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) {
999 dca_error(dca, "unable to setup PCI config handle");
1000 return (DDI_FAILURE);
1001 }
1002
1003 /*
1004 * Reprogram registers in PCI configuration space.
1005 */
1006
1007 /* Broadcom-specific timers -- we disable them. */
1008 pci_config_put8(pci, PCI_TRDYTO, 0);
1009 pci_config_put8(pci, PCI_RETRIES, 0);
1010
1011 /* initialize PCI access settings */
1012 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
1013 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);
1014
1015 /* set up our PCI latency timer */
1016 pci_config_put8(pci, PCI_LATTMR, 0x40);
1017
1018 pci_config_teardown(&pci);
1019
1020 if (dca_reset(dca, 0) < 0) {
1021 dca_error(dca, "unable to reset device during resume");
1022 return (DDI_FAILURE);
1023 }
1024
1025 /*
1026 * Now restore the card-specific CSRs.
1027 */
1028
1029 /* restore endianness settings */
1030 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
1031 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1032 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1033 return (DDI_FAILURE);
1034
1035 /* restore interrupt enables */
1036 if (dca->dca_devid == 0x5825) {
1037 /* for 5825 set 256 byte read size to improve performance */
1038 SETBIT(dca, CSR_DMACTL,
1039 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
1040 } else {
1041 SETBIT(dca, CSR_DMACTL,
1042 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
1043 }
1044 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1045 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1046 return (DDI_FAILURE);
1047
1048 /* resume scheduling jobs on the device */
1049 dca_undrain(dca);
1050
1051 return (DDI_SUCCESS);
1052 }
1053
1054 int
1055 dca_suspend(dca_t *dca)
1056 {
1057 if ((dca_drain(dca)) != 0) {
1058 return (DDI_FAILURE);
1059 }
1060 if (dca_reset(dca, 0) < 0) {
1061 dca_error(dca, "unable to reset device during suspend");
1062 return (DDI_FAILURE);
1063 }
1064 return (DDI_SUCCESS);
1065 }
1066
1067 /*
1068 * Hardware access stuff.
1069 */
1070 int
1071 dca_reset(dca_t *dca, int failreset)
1072 {
1073 int i;
1074
1075 if (dca->dca_regs_handle == NULL) {
1076 return (-1);
1077 }
1078
1079 PUTCSR(dca, CSR_DMACTL, DMACTL_RESET);
1080 if (!failreset) {
1081 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1082 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1083 return (-1);
1084 }
1085
1086 /* now wait for a reset */
1087 for (i = 1; i < 100; i++) {
1088 uint32_t dmactl;
1089 drv_usecwait(100);
1090 dmactl = GETCSR(dca, CSR_DMACTL);
1091 if (!failreset) {
1092 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1093 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1094 return (-1);
1095 }
1096 if ((dmactl & DMACTL_RESET) == 0) {
1097 DBG(dca, DCHATTY, "reset in %d usec", i * 100);
1098 return (0);
1099 }
1100 }
1101 if (!failreset) {
1102 dca_failure(dca, DDI_DEVICE_FAULT,
1103 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
1104 "timeout waiting for reset after %d usec", i * 100);
1105 }
1106 return (-1);
1107 }
1108
1109 int
1110 dca_initworklist(dca_t *dca, dca_worklist_t *wlp)
1111 {
1112 int i;
1113 int reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR);
1114
1115 /*
1116 * Set up work queue.
1117 */
1118 mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie);
1119 mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER,
1120 dca->dca_icookie);
1121 mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie);
1122 cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL);
1123
1124 mutex_enter(&wlp->dwl_lock);
1125
1126 dca_initq(&wlp->dwl_freereqs);
1127 dca_initq(&wlp->dwl_waitq);
1128 dca_initq(&wlp->dwl_freework);
1129 dca_initq(&wlp->dwl_runq);
1130
1131 for (i = 0; i < MAXWORK; i++) {
1132 dca_work_t *workp;
1133
1134 if ((workp = dca_newwork(dca)) == NULL) {
1135 dca_error(dca, "unable to allocate work");
1136 mutex_exit(&wlp->dwl_lock);
1137 return (DDI_FAILURE);
1138 }
1139 workp->dw_wlp = wlp;
1140 dca_freework(workp);
1141 }
1142 mutex_exit(&wlp->dwl_lock);
1143
1144 for (i = 0; i < reqprealloc; i++) {
1145 dca_request_t *reqp;
1146
1147 if ((reqp = dca_newreq(dca)) == NULL) {
1148 dca_error(dca, "unable to allocate request");
1149 return (DDI_FAILURE);
1150 }
1151 reqp->dr_dca = dca;
1152 reqp->dr_wlp = wlp;
1153 dca_freereq(reqp);
1154 }
1155 return (DDI_SUCCESS);
1156 }
1157
1158 int
1159 dca_init(dca_t *dca)
1160 {
1161 dca_worklist_t *wlp;
1162
1163 /* Initialize the private context list and the corresponding lock. */
1164 mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL);
1165 dca_initq(&dca->dca_ctx_list);
1166
1167 /*
1168 * MCR1 algorithms.
1169 */
1170 wlp = WORKLIST(dca, MCR1);
1171 (void) sprintf(wlp->dwl_name, "dca%d:mcr1",
1172 ddi_get_instance(dca->dca_dip));
1173 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
1174 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1175 "mcr1_lowater", MCR1LOWATER);
1176 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
1177 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1178 "mcr1_hiwater", MCR1HIWATER);
1179 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
1180 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1181 "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR);
1182 wlp->dwl_dca = dca;
1183 wlp->dwl_mcr = MCR1;
1184 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
1185 return (DDI_FAILURE);
1186 }
1187
1188 /*
1189 * MCR2 algorithms.
1190 */
1191 wlp = WORKLIST(dca, MCR2);
1192 (void) sprintf(wlp->dwl_name, "dca%d:mcr2",
1193 ddi_get_instance(dca->dca_dip));
1194 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
1195 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1196 "mcr2_lowater", MCR2LOWATER);
1197 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
1198 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1199 "mcr2_hiwater", MCR2HIWATER);
1200 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
1201 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1202 "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR);
1203 wlp->dwl_dca = dca;
1204 wlp->dwl_mcr = MCR2;
1205 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
1206 return (DDI_FAILURE);
1207 }
1208 return (DDI_SUCCESS);
1209 }
1210
1211 /*
1212 * Uninitialize worklists. This routine should only be called when no
1213 * active jobs (hence DMA mappings) exist. One way to ensure this is
1214 * to unregister from kCF before calling this routine. (This is done
1215 * e.g. in detach(9e).)
1216 */
1217 void
1218 dca_uninit(dca_t *dca)
1219 {
1220 int mcr;
1221
1222 mutex_destroy(&dca->dca_ctx_list_lock);
1223
1224 for (mcr = MCR1; mcr <= MCR2; mcr++) {
1225 dca_worklist_t *wlp = WORKLIST(dca, mcr);
1226 dca_work_t *workp;
1227 dca_request_t *reqp;
1228
1229 if (dca->dca_regs_handle == NULL) {
1230 continue;
1231 }
1232
1233 mutex_enter(&wlp->dwl_lock);
1234 while ((workp = dca_getwork(dca, mcr)) != NULL) {
1235 dca_destroywork(workp);
1236 }
1237 mutex_exit(&wlp->dwl_lock);
1238 while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) {
1239 dca_destroyreq(reqp);
1240 }
1241
1242 mutex_destroy(&wlp->dwl_lock);
1243 mutex_destroy(&wlp->dwl_freereqslock);
1244 mutex_destroy(&wlp->dwl_freelock);
1245 cv_destroy(&wlp->dwl_cv);
1246 wlp->dwl_prov = NULL;
1247 }
1248 }
1249
1250 static void
1251 dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock)
1252 {
1253 if (!q || !node)
1254 return;
1255
1256 mutex_enter(lock);
1257 node->dl_next2 = q;
1258 node->dl_prev2 = q->dl_prev2;
1259 node->dl_next2->dl_prev2 = node;
1260 node->dl_prev2->dl_next2 = node;
1261 mutex_exit(lock);
1262 }
1263
1264 static void
1265 dca_rmlist2(dca_listnode_t *node, kmutex_t *lock)
1266 {
1267 if (!node)
1268 return;
1269
1270 mutex_enter(lock);
1271 node->dl_next2->dl_prev2 = node->dl_prev2;
1272 node->dl_prev2->dl_next2 = node->dl_next2;
1273 node->dl_next2 = NULL;
1274 node->dl_prev2 = NULL;
1275 mutex_exit(lock);
1276 }
1277
1278 static dca_listnode_t *
1279 dca_delist2(dca_listnode_t *q, kmutex_t *lock)
1280 {
1281 dca_listnode_t *node;
1282
1283 mutex_enter(lock);
1284 if ((node = q->dl_next2) == q) {
1285 mutex_exit(lock);
1286 return (NULL);
1287 }
1288
1289 node->dl_next2->dl_prev2 = node->dl_prev2;
1290 node->dl_prev2->dl_next2 = node->dl_next2;
1291 node->dl_next2 = NULL;
1292 node->dl_prev2 = NULL;
1293 mutex_exit(lock);
1294
1295 return (node);
1296 }
1297
1298 void
1299 dca_initq(dca_listnode_t *q)
1300 {
1301 q->dl_next = q;
1302 q->dl_prev = q;
1303 q->dl_next2 = q;
1304 q->dl_prev2 = q;
1305 }
1306
1307 void
1308 dca_enqueue(dca_listnode_t *q, dca_listnode_t *node)
1309 {
1310 /*
1311 * Enqueue submits at the "tail" of the list, i.e. just
1312 * behind the sentinel.
1313 */
1314 node->dl_next = q;
1315 node->dl_prev = q->dl_prev;
1316 node->dl_next->dl_prev = node;
1317 node->dl_prev->dl_next = node;
1318 }
1319
1320 void
1321 dca_rmqueue(dca_listnode_t *node)
1322 {
1323 node->dl_next->dl_prev = node->dl_prev;
1324 node->dl_prev->dl_next = node->dl_next;
1325 node->dl_next = NULL;
1326 node->dl_prev = NULL;
1327 }
1328
1329 dca_listnode_t *
1330 dca_dequeue(dca_listnode_t *q)
1331 {
1332 dca_listnode_t *node;
1333 /*
1334 * Dequeue takes from the "head" of the list, i.e. just after
1335 * the sentinel.
1336 */
1337 if ((node = q->dl_next) == q) {
1338 /* queue is empty */
1339 return (NULL);
1340 }
1341 dca_rmqueue(node);
1342 return (node);
1343 }
1344
1345 /* this is the opposite of dequeue, it takes things off in LIFO order */
1346 dca_listnode_t *
1347 dca_unqueue(dca_listnode_t *q)
1348 {
1349 dca_listnode_t *node;
1350 /*
1351 * unqueue takes from the "tail" of the list, i.e. just before
1352 * the sentinel.
1353 */
1354 if ((node = q->dl_prev) == q) {
1355 /* queue is empty */
1356 return (NULL);
1357 }
1358 dca_rmqueue(node);
1359 return (node);
1360 }
1361
1362 dca_listnode_t *
1363 dca_peekqueue(dca_listnode_t *q)
1364 {
1365 dca_listnode_t *node;
1366
1367 if ((node = q->dl_next) == q) {
1368 return (NULL);
1369 } else {
1370 return (node);
1371 }
1372 }
1373
1374 /*
1375 * Interrupt service routine.
1376 */
1377 uint_t
1378 dca_intr(char *arg)
1379 {
1380 dca_t *dca = (dca_t *)arg;
1381 uint32_t status;
1382
1383 mutex_enter(&dca->dca_intrlock);
1384 status = GETCSR(dca, CSR_DMASTAT);
1385 PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS);
1386 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1387 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
1388 mutex_exit(&dca->dca_intrlock);
1389 return ((uint_t)DDI_FAILURE);
1390 }
1391
1392 DBG(dca, DINTR, "interrupted, status = 0x%x!", status);
1393
1394 if ((status & DMASTAT_INTERRUPTS) == 0) {
1395 /* increment spurious interrupt kstat */
1396 if (dca->dca_intrstats) {
1397 KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++;
1398 }
1399 mutex_exit(&dca->dca_intrlock);
1400 return (DDI_INTR_UNCLAIMED);
1401 }
1402
1403 if (dca->dca_intrstats) {
1404 KIOIP(dca)->intrs[KSTAT_INTR_HARD]++;
1405 }
1406 if (status & DMASTAT_MCR1INT) {
1407 DBG(dca, DINTR, "MCR1 interrupted");
1408 mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock));
1409 dca_schedule(dca, MCR1);
1410 dca_reclaim(dca, MCR1);
1411 mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock));
1412 }
1413
1414 if (status & DMASTAT_MCR2INT) {
1415 DBG(dca, DINTR, "MCR2 interrupted");
1416 mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock));
1417 dca_schedule(dca, MCR2);
1418 dca_reclaim(dca, MCR2);
1419 mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock));
1420 }
1421
1422 if (status & DMASTAT_ERRINT) {
1423 uint32_t erraddr;
1424 erraddr = GETCSR(dca, CSR_DMAEA);
1425 mutex_exit(&dca->dca_intrlock);
1426
1427 /*
1428 * bit 1 of the error address indicates failure during
1429 * read if set, during write otherwise.
1430 */
1431 dca_failure(dca, DDI_DEVICE_FAULT,
1432 DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR,
1433 "DMA master access error %s address 0x%x",
1434 erraddr & 0x1 ? "reading" : "writing", erraddr & ~1);
1435 return (DDI_INTR_CLAIMED);
1436 }
1437
1438 mutex_exit(&dca->dca_intrlock);
1439
1440 return (DDI_INTR_CLAIMED);
1441 }
1442
1443 /*
1444 * Reverse a string of bytes from s1 into s2. The reversal happens
1445 * from the tail of s1. If len1 < len2, then null bytes will be
1446 * padded to the end of s2. If len2 < len1, then (presumably null)
1447 * bytes will be dropped from the start of s1.
1448 *
1449 * The rationale here is that when s1 (source) is shorter, then we
1450 * are reversing from big-endian ordering, into device ordering, and
1451 * want to add some extra nulls to the tail (MSB) side of the device.
1452 *
1453 * Similarly, when s2 (dest) is shorter, then we are truncating what
1454 * are presumably null MSB bits from the device.
1455 *
1456 * There is an expectation when reversing from the device back into
1457 * big-endian, that the number of bytes to reverse and the target size
1458 * will match, and no truncation or padding occurs.
1459 */
1460 void
1461 dca_reverse(void *s1, void *s2, int len1, int len2)
1462 {
1463 caddr_t src, dst;
1464
1465 if (len1 == 0) {
1466 if (len2) {
1467 bzero(s2, len2);
1468 }
1469 return;
1470 }
1471 src = (caddr_t)s1 + len1 - 1;
1472 dst = s2;
1473 while ((src >= (caddr_t)s1) && (len2)) {
1474 *dst++ = *src--;
1475 len2--;
1476 }
1477 while (len2 > 0) {
1478 *dst++ = 0;
1479 len2--;
1480 }
1481 }
1482
1483 uint16_t
1484 dca_padfull(int num)
1485 {
1486 if (num <= 512) {
1487 return (BITS2BYTES(512));
1488 }
1489 if (num <= 768) {
1490 return (BITS2BYTES(768));
1491 }
1492 if (num <= 1024) {
1493 return (BITS2BYTES(1024));
1494 }
1495 if (num <= 1536) {
1496 return (BITS2BYTES(1536));
1497 }
1498 if (num <= 2048) {
1499 return (BITS2BYTES(2048));
1500 }
1501 return (0);
1502 }
1503
1504 uint16_t
1505 dca_padhalf(int num)
1506 {
1507 if (num <= 256) {
1508 return (BITS2BYTES(256));
1509 }
1510 if (num <= 384) {
1511 return (BITS2BYTES(384));
1512 }
1513 if (num <= 512) {
1514 return (BITS2BYTES(512));
1515 }
1516 if (num <= 768) {
1517 return (BITS2BYTES(768));
1518 }
1519 if (num <= 1024) {
1520 return (BITS2BYTES(1024));
1521 }
1522 return (0);
1523 }
1524
1525 dca_work_t *
1526 dca_newwork(dca_t *dca)
1527 {
1528 dca_work_t *workp;
1529 size_t size;
1530 ddi_dma_cookie_t c;
1531 unsigned nc;
1532 int rv;
1533
1534 workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP);
1535
1536 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1537 DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah);
1538 if (rv != 0) {
1539 dca_error(dca, "unable to alloc MCR DMA handle");
1540 dca_destroywork(workp);
1541 return (NULL);
1542 }
1543
1544 rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah,
1545 ROUNDUP(MCR_SIZE, dca->dca_pagesize),
1546 &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
1547 &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch);
1548 if (rv != 0) {
1549 dca_error(dca, "unable to alloc MCR DMA memory");
1550 dca_destroywork(workp);
1551 return (NULL);
1552 }
1553
1554 rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL,
1555 workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR,
1556 DDI_DMA_SLEEP, NULL, &c, &nc);
1557 if (rv != DDI_DMA_MAPPED) {
1558 dca_error(dca, "unable to map MCR DMA memory");
1559 dca_destroywork(workp);
1560 return (NULL);
1561 }
1562
1563 workp->dw_mcr_paddr = c.dmac_address;
1564 return (workp);
1565 }
1566
1567 void
1568 dca_destroywork(dca_work_t *workp)
1569 {
1570 if (workp->dw_mcr_paddr) {
1571 (void) ddi_dma_unbind_handle(workp->dw_mcr_dmah);
1572 }
1573 if (workp->dw_mcr_acch) {
1574 ddi_dma_mem_free(&workp->dw_mcr_acch);
1575 }
1576 if (workp->dw_mcr_dmah) {
1577 ddi_dma_free_handle(&workp->dw_mcr_dmah);
1578 }
1579 kmem_free(workp, sizeof (dca_work_t));
1580 }
1581
1582 dca_request_t *
1583 dca_newreq(dca_t *dca)
1584 {
1585 dca_request_t *reqp;
1586 size_t size;
1587 ddi_dma_cookie_t c;
1588 unsigned nc;
1589 int rv;
1590 int n_chain = 0;
1591
1592 size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH;
1593
1594 reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP);
1595
1596 reqp->dr_dca = dca;
1597
1598 /*
1599 * Setup the DMA region for the context and descriptors.
1600 */
1601 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP,
1602 NULL, &reqp->dr_ctx_dmah);
1603 if (rv != DDI_SUCCESS) {
1604 dca_error(dca, "failure allocating request DMA handle");
1605 dca_destroyreq(reqp);
1606 return (NULL);
1607 }
1608
1609 /* for driver hardening, allocate in whole pages */
1610 rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah,
1611 ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT,
1612 DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size,
1613 &reqp->dr_ctx_acch);
1614 if (rv != DDI_SUCCESS) {
1615 dca_error(dca, "unable to alloc request DMA memory");
1616 dca_destroyreq(reqp);
1617 return (NULL);
1618 }
1619
1620 rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL,
1621 reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE,
1622 DDI_DMA_SLEEP, 0, &c, &nc);
1623 if (rv != DDI_DMA_MAPPED) {
1624 dca_error(dca, "failed binding request DMA handle");
1625 dca_destroyreq(reqp);
1626 return (NULL);
1627 }
1628 reqp->dr_ctx_paddr = c.dmac_address;
1629
1630 reqp->dr_dma_size = size;
1631
1632 /*
1633 * Set up the dma for our scratch/shared buffers.
1634 */
1635 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1636 DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah);
1637 if (rv != DDI_SUCCESS) {
1638 dca_error(dca, "failure allocating ibuf DMA handle");
1639 dca_destroyreq(reqp);
1640 return (NULL);
1641 }
1642 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1643 DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah);
1644 if (rv != DDI_SUCCESS) {
1645 dca_error(dca, "failure allocating obuf DMA handle");
1646 dca_destroyreq(reqp);
1647 return (NULL);
1648 }
1649
1650 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1651 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah);
1652 if (rv != DDI_SUCCESS) {
1653 dca_error(dca, "failure allocating chain_in DMA handle");
1654 dca_destroyreq(reqp);
1655 return (NULL);
1656 }
1657
1658 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1659 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah);
1660 if (rv != DDI_SUCCESS) {
1661 dca_error(dca, "failure allocating chain_out DMA handle");
1662 dca_destroyreq(reqp);
1663 return (NULL);
1664 }
1665
1666 /*
1667 * for driver hardening, allocate in whole pages.
1668 */
1669 size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
1670 #if defined(i386) || defined(__i386)
1671 /*
1672 * Use kmem_alloc instead of ddi_dma_mem_alloc here since the latter
1673 * may fail on x86 platform if a physically contiguous memory chunk
1674 * cannot be found. From initial testing, we did not see performance
1675 * degradation as seen on Sparc.
1676 */
1677 if ((reqp->dr_ibuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) {
1678 dca_error(dca, "unable to alloc request ibuf memory");
1679 dca_destroyreq(reqp);
1680 return (NULL);
1681 }
1682 if ((reqp->dr_obuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) {
1683 dca_error(dca, "unable to alloc request obuf memory");
1684 dca_destroyreq(reqp);
1685 return (NULL);
1686 }
1687 #else
1688 /*
1689 * We could kmem_alloc for Sparc too. However, it gives worse
1690 * performance when transferring more than one page data. For example,
1691 * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system,
1692 * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for
1693 * the same throughput.
1694 */
1695 rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah,
1696 size, &dca_bufattr,
1697 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr,
1698 &size, &reqp->dr_ibuf_acch);
1699 if (rv != DDI_SUCCESS) {
1700 dca_error(dca, "unable to alloc request DMA memory");
1701 dca_destroyreq(reqp);
1702 return (NULL);
1703 }
1704
1705 rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah,
1706 size, &dca_bufattr,
1707 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr,
1708 &size, &reqp->dr_obuf_acch);
1709 if (rv != DDI_SUCCESS) {
1710 dca_error(dca, "unable to alloc request DMA memory");
1711 dca_destroyreq(reqp);
1712 return (NULL);
1713 }
1714 #endif
1715
1716 /* Skip the used portion in the context page */
1717 reqp->dr_offset = CTX_MAXLENGTH;
1718 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
1719 reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah,
1720 DDI_DMA_WRITE | DDI_DMA_STREAMING,
1721 &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) {
1722 (void) dca_destroyreq(reqp);
1723 return (NULL);
1724 }
1725 reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr;
1726 /* Skip the space used by the input buffer */
1727 reqp->dr_offset += DESC_SIZE * n_chain;
1728
1729 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
1730 reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah,
1731 DDI_DMA_READ | DDI_DMA_STREAMING,
1732 &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) {
1733 (void) dca_destroyreq(reqp);
1734 return (NULL);
1735 }
1736 reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr;
1737 /* Skip the space used by the output buffer */
1738 reqp->dr_offset += DESC_SIZE * n_chain;
1739
1740 DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d",
1741 reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH);
1742 return (reqp);
1743 }
1744
1745 void
1746 dca_destroyreq(dca_request_t *reqp)
1747 {
1748 #if defined(i386) || defined(__i386)
1749 dca_t *dca = reqp->dr_dca;
1750 size_t size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
1751 #endif
1752
1753 /*
1754 * Clean up DMA for the context structure.
1755 */
1756 if (reqp->dr_ctx_paddr) {
1757 (void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah);
1758 }
1759
1760 if (reqp->dr_ctx_acch) {
1761 ddi_dma_mem_free(&reqp->dr_ctx_acch);
1762 }
1763
1764 if (reqp->dr_ctx_dmah) {
1765 ddi_dma_free_handle(&reqp->dr_ctx_dmah);
1766 }
1767
1768 /*
1769 * Clean up DMA for the scratch buffer.
1770 */
1771 #if defined(i386) || defined(__i386)
1772 if (reqp->dr_ibuf_dmah) {
1773 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
1774 ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
1775 }
1776 if (reqp->dr_obuf_dmah) {
1777 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
1778 ddi_dma_free_handle(&reqp->dr_obuf_dmah);
1779 }
1780
1781 kmem_free(reqp->dr_ibuf_kaddr, size);
1782 kmem_free(reqp->dr_obuf_kaddr, size);
1783 #else
1784 if (reqp->dr_ibuf_paddr) {
1785 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
1786 }
1787 if (reqp->dr_obuf_paddr) {
1788 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
1789 }
1790
1791 if (reqp->dr_ibuf_acch) {
1792 ddi_dma_mem_free(&reqp->dr_ibuf_acch);
1793 }
1794 if (reqp->dr_obuf_acch) {
1795 ddi_dma_mem_free(&reqp->dr_obuf_acch);
1796 }
1797
1798 if (reqp->dr_ibuf_dmah) {
1799 ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
1800 }
1801 if (reqp->dr_obuf_dmah) {
1802 ddi_dma_free_handle(&reqp->dr_obuf_dmah);
1803 }
1804 #endif
1805 /*
1806 * These two DMA handles should have been unbinded in
1807 * dca_unbindchains() function
1808 */
1809 if (reqp->dr_chain_in_dmah) {
1810 ddi_dma_free_handle(&reqp->dr_chain_in_dmah);
1811 }
1812 if (reqp->dr_chain_out_dmah) {
1813 ddi_dma_free_handle(&reqp->dr_chain_out_dmah);
1814 }
1815
1816 kmem_free(reqp, sizeof (dca_request_t));
1817 }
1818
1819 dca_work_t *
1820 dca_getwork(dca_t *dca, int mcr)
1821 {
1822 dca_worklist_t *wlp = WORKLIST(dca, mcr);
1823 dca_work_t *workp;
1824
1825 mutex_enter(&wlp->dwl_freelock);
1826 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework);
1827 mutex_exit(&wlp->dwl_freelock);
1828 if (workp) {
1829 int nreqs;
1830 bzero(workp->dw_mcr_kaddr, 8);
1831
1832 /* clear out old requests */
1833 for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) {
1834 workp->dw_reqs[nreqs] = NULL;
1835 }
1836 }
1837 return (workp);
1838 }
1839
1840 void
1841 dca_freework(dca_work_t *workp)
1842 {
1843 mutex_enter(&workp->dw_wlp->dwl_freelock);
1844 dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp);
1845 mutex_exit(&workp->dw_wlp->dwl_freelock);
1846 }
1847
1848 dca_request_t *
1849 dca_getreq(dca_t *dca, int mcr, int tryhard)
1850 {
1851 dca_worklist_t *wlp = WORKLIST(dca, mcr);
1852 dca_request_t *reqp;
1853
1854 mutex_enter(&wlp->dwl_freereqslock);
1855 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs);
1856 mutex_exit(&wlp->dwl_freereqslock);
1857 if (reqp) {
1858 reqp->dr_flags = 0;
1859 reqp->dr_callback = NULL;
1860 } else if (tryhard) {
1861 /*
1862 * failed to get a free one, try an allocation, the hard way.
1863 * XXX: Kstat desired here.
1864 */
1865 if ((reqp = dca_newreq(dca)) != NULL) {
1866 reqp->dr_wlp = wlp;
1867 reqp->dr_dca = dca;
1868 reqp->dr_flags = 0;
1869 reqp->dr_callback = NULL;
1870 }
1871 }
1872 return (reqp);
1873 }
1874
1875 void
1876 dca_freereq(dca_request_t *reqp)
1877 {
1878 reqp->dr_kcf_req = NULL;
1879 if (!(reqp->dr_flags & DR_NOCACHE)) {
1880 mutex_enter(&reqp->dr_wlp->dwl_freereqslock);
1881 dca_enqueue(&reqp->dr_wlp->dwl_freereqs,
1882 (dca_listnode_t *)reqp);
1883 mutex_exit(&reqp->dr_wlp->dwl_freereqslock);
1884 }
1885 }
1886
1887 /*
1888 * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer
1889 * is mapped to a single physical address. On x86, a user buffer is mapped
1890 * to multiple physical addresses. These physical addresses are chained
1891 * using the method specified in Broadcom BCM5820 specification.
1892 */
1893 int
1894 dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt)
1895 {
1896 int rv;
1897 caddr_t kaddr;
1898 uint_t flags;
1899 int n_chain = 0;
1900
1901 if (reqp->dr_flags & DR_INPLACE) {
1902 flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
1903 } else {
1904 flags = DDI_DMA_WRITE | DDI_DMA_STREAMING;
1905 }
1906
1907 /* first the input */
1908 if (incnt) {
1909 if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) {
1910 DBG(NULL, DWARN, "unrecognised crypto data format");
1911 return (DDI_FAILURE);
1912 }
1913 if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset,
1914 kaddr, reqp->dr_chain_in_dmah, flags,
1915 &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) {
1916 (void) dca_unbindchains(reqp);
1917 return (rv);
1918 }
1919
1920 /*
1921 * The offset and length are altered by the calling routine
1922 * reqp->dr_in->cd_offset += incnt;
1923 * reqp->dr_in->cd_length -= incnt;
1924 */
1925 /* Save the first one in the chain for MCR */
1926 reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr;
1927 reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr;
1928 reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length;
1929 } else {
1930 reqp->dr_in_paddr = NULL;
1931 reqp->dr_in_next = 0;
1932 reqp->dr_in_len = 0;
1933 }
1934
1935 if (reqp->dr_flags & DR_INPLACE) {
1936 reqp->dr_out_paddr = reqp->dr_in_paddr;
1937 reqp->dr_out_len = reqp->dr_in_len;
1938 reqp->dr_out_next = reqp->dr_in_next;
1939 return (DDI_SUCCESS);
1940 }
1941
1942 /* then the output */
1943 if (outcnt) {
1944 flags = DDI_DMA_READ | DDI_DMA_STREAMING;
1945 if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) {
1946 DBG(NULL, DWARN, "unrecognised crypto data format");
1947 (void) dca_unbindchains(reqp);
1948 return (DDI_FAILURE);
1949 }
1950 rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset +
1951 n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah,
1952 flags, &reqp->dr_chain_out_head, &n_chain);
1953 if (rv != DDI_SUCCESS) {
1954 (void) dca_unbindchains(reqp);
1955 return (DDI_FAILURE);
1956 }
1957
1958 /* Save the first one in the chain for MCR */
1959 reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr;
1960 reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr;
1961 reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length;
1962 } else {
1963 reqp->dr_out_paddr = NULL;
1964 reqp->dr_out_next = 0;
1965 reqp->dr_out_len = 0;
1966 }
1967
1968 return (DDI_SUCCESS);
1969 }
1970
1971 /*
1972 * Unbind the user buffers from the DMA handles.
1973 */
1974 int
1975 dca_unbindchains(dca_request_t *reqp)
1976 {
1977 int rv = DDI_SUCCESS;
1978 int rv1 = DDI_SUCCESS;
1979
1980 /* Clear the input chain */
1981 if (reqp->dr_chain_in_head.dc_buffer_paddr != NULL) {
1982 (void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah);
1983 reqp->dr_chain_in_head.dc_buffer_paddr = 0;
1984 }
1985
1986 if (reqp->dr_flags & DR_INPLACE) {
1987 return (rv);
1988 }
1989
1990 /* Clear the output chain */
1991 if (reqp->dr_chain_out_head.dc_buffer_paddr != NULL) {
1992 (void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah);
1993 reqp->dr_chain_out_head.dc_buffer_paddr = 0;
1994 }
1995
1996 return ((rv != DDI_SUCCESS)? rv : rv1);
1997 }
1998
1999 /*
2000 * Build either input chain or output chain. It is single-item chain for Sparc,
2001 * and possible mutiple-item chain for x86.
2002 */
2003 static int
2004 dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
2005 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
2006 dca_chain_t *head, int *n_chain)
2007 {
2008 ddi_dma_cookie_t c;
2009 uint_t nc;
2010 int rv;
2011 caddr_t chain_kaddr_pre;
2012 caddr_t chain_kaddr;
2013 uint32_t chain_paddr;
2014 int i;
2015
2016 /* Advance past the context structure to the starting address */
2017 chain_paddr = reqp->dr_ctx_paddr + dr_offset;
2018 chain_kaddr = reqp->dr_ctx_kaddr + dr_offset;
2019
2020 /*
2021 * Bind the kernel address to the DMA handle. On x86, the actual
2022 * buffer is mapped into multiple physical addresses. On Sparc,
2023 * the actual buffer is mapped into a single address.
2024 */
2025 rv = ddi_dma_addr_bind_handle(handle,
2026 NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc);
2027 if (rv != DDI_DMA_MAPPED) {
2028 return (DDI_FAILURE);
2029 }
2030
2031 (void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV);
2032 if ((rv = dca_check_dma_handle(reqp->dr_dca, handle,
2033 DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) {
2034 reqp->destroy = TRUE;
2035 return (rv);
2036 }
2037
2038 *n_chain = nc;
2039
2040 /* Setup the data buffer chain for DMA transfer */
2041 chain_kaddr_pre = NULL;
2042 head->dc_buffer_paddr = 0;
2043 head->dc_next_paddr = 0;
2044 head->dc_buffer_length = 0;
2045 for (i = 0; i < nc; i++) {
2046 /* PIO */
2047 PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address);
2048 PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0);
2049 PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size);
2050
2051 /* Remember the head of the chain */
2052 if (head->dc_buffer_paddr == 0) {
2053 head->dc_buffer_paddr = c.dmac_address;
2054 head->dc_buffer_length = c.dmac_size;
2055 }
2056
2057 /* Link to the previous one if one exists */
2058 if (chain_kaddr_pre) {
2059 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT,
2060 chain_paddr);
2061 if (head->dc_next_paddr == 0)
2062 head->dc_next_paddr = chain_paddr;
2063 }
2064 chain_kaddr_pre = chain_kaddr;
2065
2066 /* Maintain pointers */
2067 chain_paddr += DESC_SIZE;
2068 chain_kaddr += DESC_SIZE;
2069
2070 /* Retrieve the next cookie if there is one */
2071 if (i < nc-1)
2072 ddi_dma_nextcookie(handle, &c);
2073 }
2074
2075 /* Set the next pointer in the last entry to NULL */
2076 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0);
2077
2078 return (DDI_SUCCESS);
2079 }
2080
2081 /*
2082 * Schedule some work.
2083 */
2084 int
2085 dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched)
2086 {
2087 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2088
2089 mutex_enter(&wlp->dwl_lock);
2090
2091 DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p",
2092 reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr,
2093 reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr);
2094 DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x",
2095 reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr);
2096 /* sync out the entire context and descriptor chains */
2097 (void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV);
2098 if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah,
2099 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2100 reqp->destroy = TRUE;
2101 mutex_exit(&wlp->dwl_lock);
2102 return (CRYPTO_DEVICE_ERROR);
2103 }
2104
2105 dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp);
2106 wlp->dwl_count++;
2107 wlp->dwl_lastsubmit = ddi_get_lbolt();
2108 reqp->dr_wlp = wlp;
2109
2110 if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) {
2111 /* we are fully loaded now, let kCF know */
2112
2113 wlp->dwl_flowctl++;
2114 wlp->dwl_busy = 1;
2115
2116 crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY);
2117 }
2118
2119 if (dosched) {
2120 #ifdef SCHEDDELAY
2121 /* possibly wait for more work to arrive */
2122 if (wlp->dwl_count >= wlp->dwl_reqspermcr) {
2123 dca_schedule(dca, mcr);
2124 } else if (!wlp->dwl_schedtid) {
2125 /* wait 1 msec for more work before doing it */
2126 wlp->dwl_schedtid = timeout(dca_schedtimeout,
2127 (void *)wlp, drv_usectohz(MSEC));
2128 }
2129 #else
2130 dca_schedule(dca, mcr);
2131 #endif
2132 }
2133 mutex_exit(&wlp->dwl_lock);
2134
2135 return (CRYPTO_QUEUED);
2136 }
2137
2138 void
2139 dca_schedule(dca_t *dca, int mcr)
2140 {
2141 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2142 int csr;
2143 int full;
2144 uint32_t status;
2145
2146 ASSERT(mutex_owned(&wlp->dwl_lock));
2147 /*
2148 * If the card is draining or has an outstanding failure,
2149 * don't schedule any more work on it right now
2150 */
2151 if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) {
2152 return;
2153 }
2154
2155 if (mcr == MCR2) {
2156 csr = CSR_MCR2;
2157 full = DMASTAT_MCR2FULL;
2158 } else {
2159 csr = CSR_MCR1;
2160 full = DMASTAT_MCR1FULL;
2161 }
2162
2163 for (;;) {
2164 dca_work_t *workp;
2165 uint32_t offset;
2166 int nreqs;
2167
2168 status = GETCSR(dca, CSR_DMASTAT);
2169 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
2170 DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
2171 return;
2172
2173 if ((status & full) != 0)
2174 break;
2175
2176 #ifdef SCHEDDELAY
2177 /* if there isn't enough to do, don't bother now */
2178 if ((wlp->dwl_count < wlp->dwl_reqspermcr) &&
2179 (ddi_get_lbolt() < (wlp->dwl_lastsubmit +
2180 drv_usectohz(MSEC)))) {
2181 /* wait a bit longer... */
2182 if (wlp->dwl_schedtid == 0) {
2183 wlp->dwl_schedtid = timeout(dca_schedtimeout,
2184 (void *)wlp, drv_usectohz(MSEC));
2185 }
2186 return;
2187 }
2188 #endif
2189
2190 /* grab a work structure */
2191 workp = dca_getwork(dca, mcr);
2192
2193 if (workp == NULL) {
2194 /*
2195 * There must be work ready to be reclaimed,
2196 * in this case, since the chip can only hold
2197 * less work outstanding than there are total.
2198 */
2199 dca_reclaim(dca, mcr);
2200 continue;
2201 }
2202
2203 nreqs = 0;
2204 offset = MCR_CTXADDR;
2205
2206 while (nreqs < wlp->dwl_reqspermcr) {
2207 dca_request_t *reqp;
2208
2209 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq);
2210 if (reqp == NULL) {
2211 /* nothing left to process */
2212 break;
2213 }
2214 /*
2215 * Update flow control.
2216 */
2217 wlp->dwl_count--;
2218 if ((wlp->dwl_count == wlp->dwl_lowater) &&
2219 (wlp->dwl_busy)) {
2220 wlp->dwl_busy = 0;
2221 crypto_prov_notify(wlp->dwl_prov,
2222 CRYPTO_PROVIDER_READY);
2223 }
2224
2225 /*
2226 * Context address.
2227 */
2228 PUTMCR32(workp, offset, reqp->dr_ctx_paddr);
2229 offset += 4;
2230
2231 /*
2232 * Input chain.
2233 */
2234 /* input buffer address */
2235 PUTMCR32(workp, offset, reqp->dr_in_paddr);
2236 offset += 4;
2237 /* next input buffer entry */
2238 PUTMCR32(workp, offset, reqp->dr_in_next);
2239 offset += 4;
2240 /* input buffer length */
2241 PUTMCR16(workp, offset, reqp->dr_in_len);
2242 offset += 2;
2243 /* zero the reserved field */
2244 PUTMCR16(workp, offset, 0);
2245 offset += 2;
2246
2247 /*
2248 * Overall length.
2249 */
2250 /* reserved field */
2251 PUTMCR16(workp, offset, 0);
2252 offset += 2;
2253 /* total packet length */
2254 PUTMCR16(workp, offset, reqp->dr_pkt_length);
2255 offset += 2;
2256
2257 /*
2258 * Output chain.
2259 */
2260 /* output buffer address */
2261 PUTMCR32(workp, offset, reqp->dr_out_paddr);
2262 offset += 4;
2263 /* next output buffer entry */
2264 PUTMCR32(workp, offset, reqp->dr_out_next);
2265 offset += 4;
2266 /* output buffer length */
2267 PUTMCR16(workp, offset, reqp->dr_out_len);
2268 offset += 2;
2269 /* zero the reserved field */
2270 PUTMCR16(workp, offset, 0);
2271 offset += 2;
2272
2273 /*
2274 * Note submission.
2275 */
2276 workp->dw_reqs[nreqs] = reqp;
2277 nreqs++;
2278 }
2279
2280 if (nreqs == 0) {
2281 /* nothing in the queue! */
2282 dca_freework(workp);
2283 return;
2284 }
2285
2286 wlp->dwl_submit++;
2287
2288 PUTMCR16(workp, MCR_FLAGS, 0);
2289 PUTMCR16(workp, MCR_COUNT, nreqs);
2290
2291 DBG(dca, DCHATTY,
2292 "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d",
2293 workp->dw_mcr_paddr, workp->dw_mcr_kaddr,
2294 nreqs, mcr);
2295
2296 workp->dw_lbolt = ddi_get_lbolt();
2297 /* Make sure MCR is synced out to device. */
2298 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0,
2299 DDI_DMA_SYNC_FORDEV);
2300 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
2301 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2302 dca_destroywork(workp);
2303 return;
2304 }
2305
2306 PUTCSR(dca, csr, workp->dw_mcr_paddr);
2307 if (dca_check_acc_handle(dca, dca->dca_regs_handle,
2308 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2309 dca_destroywork(workp);
2310 return;
2311 } else {
2312 dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp);
2313 }
2314
2315 DBG(dca, DCHATTY, "posted");
2316 }
2317 }
2318
2319 /*
2320 * Reclaim completed work, called in interrupt context.
2321 */
2322 void
2323 dca_reclaim(dca_t *dca, int mcr)
2324 {
2325 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2326 dca_work_t *workp;
2327 ushort_t flags;
2328 int nreclaimed = 0;
2329 int i;
2330
2331 DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr);
2332 ASSERT(mutex_owned(&wlp->dwl_lock));
2333 /*
2334 * For each MCR in the submitted (runq), we check to see if
2335 * it has been processed. If so, then we note each individual
2336 * job in the MCR, and and do the completion processing for
2337 * each of such job.
2338 */
2339 for (;;) {
2340
2341 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
2342 if (workp == NULL) {
2343 break;
2344 }
2345
2346 /* only sync the MCR flags, since that's all we need */
2347 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4,
2348 DDI_DMA_SYNC_FORKERNEL);
2349 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
2350 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2351 dca_rmqueue((dca_listnode_t *)workp);
2352 dca_destroywork(workp);
2353 return;
2354 }
2355
2356 flags = GETMCR16(workp, MCR_FLAGS);
2357 if ((flags & MCRFLAG_FINISHED) == 0) {
2358 /* chip is still working on it */
2359 DBG(dca, DRECLAIM,
2360 "chip still working on it (MCR%d)", mcr);
2361 break;
2362 }
2363
2364 /* its really for us, so remove it from the queue */
2365 dca_rmqueue((dca_listnode_t *)workp);
2366
2367 /* if we were draining, signal on the cv */
2368 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
2369 cv_signal(&wlp->dwl_cv);
2370 }
2371
2372 /* update statistics, done under the lock */
2373 for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2374 dca_request_t *reqp = workp->dw_reqs[i];
2375 if (reqp == NULL) {
2376 continue;
2377 }
2378 if (reqp->dr_byte_stat >= 0) {
2379 dca->dca_stats[reqp->dr_byte_stat] +=
2380 reqp->dr_pkt_length;
2381 }
2382 if (reqp->dr_job_stat >= 0) {
2383 dca->dca_stats[reqp->dr_job_stat]++;
2384 }
2385 }
2386 mutex_exit(&wlp->dwl_lock);
2387
2388 for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2389 dca_request_t *reqp = workp->dw_reqs[i];
2390
2391 if (reqp == NULL) {
2392 continue;
2393 }
2394
2395 /* Do the callback. */
2396 workp->dw_reqs[i] = NULL;
2397 dca_done(reqp, CRYPTO_SUCCESS);
2398
2399 nreclaimed++;
2400 }
2401
2402 /* now we can release the work */
2403 dca_freework(workp);
2404
2405 mutex_enter(&wlp->dwl_lock);
2406 }
2407 DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed);
2408 }
2409
2410 int
2411 dca_length(crypto_data_t *cdata)
2412 {
2413 return (cdata->cd_length);
2414 }
2415
2416 /*
2417 * This is the callback function called from the interrupt when a kCF job
2418 * completes. It does some driver-specific things, and then calls the
2419 * kCF-provided callback. Finally, it cleans up the state for the work
2420 * request and drops the reference count to allow for DR.
2421 */
2422 void
2423 dca_done(dca_request_t *reqp, int err)
2424 {
2425 uint64_t ena = 0;
2426
2427 /* unbind any chains we were using */
2428 if (dca_unbindchains(reqp) != DDI_SUCCESS) {
2429 /* DMA failure */
2430 ena = dca_ena(ena);
2431 dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT,
2432 DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR,
2433 "fault on buffer DMA handle");
2434 if (err == CRYPTO_SUCCESS) {
2435 err = CRYPTO_DEVICE_ERROR;
2436 }
2437 }
2438
2439 if (reqp->dr_callback != NULL) {
2440 reqp->dr_callback(reqp, err);
2441 } else {
2442 dca_freereq(reqp);
2443 }
2444 }
2445
2446 /*
2447 * Call this when a failure is detected. It will reset the chip,
2448 * log a message, alert kCF, and mark jobs in the runq as failed.
2449 */
2450 /* ARGSUSED */
2451 void
2452 dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index,
2453 uint64_t ena, int errno, char *mess, ...)
2454 {
2455 va_list ap;
2456 char buf[256];
2457 int mcr;
2458 char *eclass;
2459 int have_mutex;
2460
2461 va_start(ap, mess);
2462 (void) vsprintf(buf, mess, ap);
2463 va_end(ap);
2464
2465 eclass = dca_fma_eclass_string(dca->dca_model, index);
2466
2467 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) &&
2468 index != DCA_FM_ECLASS_NONE) {
2469 ddi_fm_ereport_post(dca->dca_dip, eclass, ena,
2470 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
2471 FM_EREPORT_VERS0, NULL);
2472
2473 /* Report the impact of the failure to the DDI. */
2474 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST);
2475 } else {
2476 /* Just log the error string to the message log */
2477 dca_error(dca, buf);
2478 }
2479
2480 /*
2481 * Indicate a failure (keeps schedule from running).
2482 */
2483 dca->dca_flags |= DCA_FAILED;
2484
2485 /*
2486 * Reset the chip. This should also have as a side effect, the
2487 * disabling of all interrupts from the device.
2488 */
2489 (void) dca_reset(dca, 1);
2490
2491 /*
2492 * Report the failure to kCF.
2493 */
2494 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2495 if (WORKLIST(dca, mcr)->dwl_prov) {
2496 crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov,
2497 CRYPTO_PROVIDER_FAILED);
2498 }
2499 }
2500
2501 /*
2502 * Return jobs not sent to hardware back to kCF.
2503 */
2504 dca_rejectjobs(dca);
2505
2506 /*
2507 * From this point on, no new work should be arriving, and the
2508 * chip should not be doing any active DMA.
2509 */
2510
2511 /*
2512 * Now find all the work submitted to the device and fail
2513 * them.
2514 */
2515 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2516 dca_worklist_t *wlp;
2517 int i;
2518
2519 wlp = WORKLIST(dca, mcr);
2520
2521 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
2522 continue;
2523 }
2524 for (;;) {
2525 dca_work_t *workp;
2526
2527 have_mutex = mutex_tryenter(&wlp->dwl_lock);
2528 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq);
2529 if (workp == NULL) {
2530 if (have_mutex)
2531 mutex_exit(&wlp->dwl_lock);
2532 break;
2533 }
2534 mutex_exit(&wlp->dwl_lock);
2535
2536 /*
2537 * Free up requests
2538 */
2539 for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2540 dca_request_t *reqp = workp->dw_reqs[i];
2541 if (reqp) {
2542 dca_done(reqp, errno);
2543 workp->dw_reqs[i] = NULL;
2544 }
2545 }
2546
2547 mutex_enter(&wlp->dwl_lock);
2548 /*
2549 * If waiting to drain, signal on the waiter.
2550 */
2551 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
2552 cv_signal(&wlp->dwl_cv);
2553 }
2554
2555 /*
2556 * Return the work and request structures to
2557 * the free pool.
2558 */
2559 dca_freework(workp);
2560 if (have_mutex)
2561 mutex_exit(&wlp->dwl_lock);
2562 }
2563 }
2564
2565 }
2566
2567 #ifdef SCHEDDELAY
2568 /*
2569 * Reschedule worklist as needed.
2570 */
2571 void
2572 dca_schedtimeout(void *arg)
2573 {
2574 dca_worklist_t *wlp = (dca_worklist_t *)arg;
2575 mutex_enter(&wlp->dwl_lock);
2576 wlp->dwl_schedtid = 0;
2577 dca_schedule(wlp->dwl_dca, wlp->dwl_mcr);
2578 mutex_exit(&wlp->dwl_lock);
2579 }
2580 #endif
2581
2582 /*
2583 * Check for stalled jobs.
2584 */
2585 void
2586 dca_jobtimeout(void *arg)
2587 {
2588 int mcr;
2589 dca_t *dca = (dca_t *)arg;
2590 int hung = 0;
2591
2592 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2593 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2594 dca_work_t *workp;
2595 clock_t when;
2596
2597 mutex_enter(&wlp->dwl_lock);
2598 when = ddi_get_lbolt();
2599
2600 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
2601 if (workp == NULL) {
2602 /* nothing sitting in the queue */
2603 mutex_exit(&wlp->dwl_lock);
2604 continue;
2605 }
2606
2607 if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) {
2608 /* request has been queued for less than STALETIME */
2609 mutex_exit(&wlp->dwl_lock);
2610 continue;
2611 }
2612
2613 /* job has been sitting around for over 1 second, badness */
2614 DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp,
2615 mcr);
2616
2617 /* put it back in the queue, until we reset the chip */
2618 hung++;
2619 mutex_exit(&wlp->dwl_lock);
2620 }
2621
2622 if (hung) {
2623 dca_failure(dca, DDI_DEVICE_FAULT,
2624 DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR,
2625 "timeout processing job.)");
2626 }
2627
2628 /* reschedule ourself */
2629 mutex_enter(&dca->dca_intrlock);
2630 if (dca->dca_jobtid == 0) {
2631 /* timeout has been canceled, prior to DR */
2632 mutex_exit(&dca->dca_intrlock);
2633 return;
2634 }
2635
2636 /* check again in 1 second */
2637 dca->dca_jobtid = timeout(dca_jobtimeout, arg,
2638 drv_usectohz(SECOND));
2639 mutex_exit(&dca->dca_intrlock);
2640 }
2641
2642 /*
2643 * This returns all jobs back to kCF. It assumes that processing
2644 * on the worklist has halted.
2645 */
2646 void
2647 dca_rejectjobs(dca_t *dca)
2648 {
2649 int mcr;
2650 int have_mutex;
2651 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2652 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2653 dca_request_t *reqp;
2654
2655 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
2656 continue;
2657 }
2658 have_mutex = mutex_tryenter(&wlp->dwl_lock);
2659 for (;;) {
2660 reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq);
2661 if (reqp == NULL) {
2662 break;
2663 }
2664 /* update flow control */
2665 wlp->dwl_count--;
2666 if ((wlp->dwl_count == wlp->dwl_lowater) &&
2667 (wlp->dwl_busy)) {
2668 wlp->dwl_busy = 0;
2669 crypto_prov_notify(wlp->dwl_prov,
2670 CRYPTO_PROVIDER_READY);
2671 }
2672 mutex_exit(&wlp->dwl_lock);
2673
2674 (void) dca_unbindchains(reqp);
2675 reqp->dr_callback(reqp, EAGAIN);
2676 mutex_enter(&wlp->dwl_lock);
2677 }
2678 if (have_mutex)
2679 mutex_exit(&wlp->dwl_lock);
2680 }
2681 }
2682
2683 int
2684 dca_drain(dca_t *dca)
2685 {
2686 int mcr;
2687 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2688 #ifdef SCHEDDELAY
2689 timeout_id_t tid;
2690 #endif
2691 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2692
2693 mutex_enter(&wlp->dwl_lock);
2694 wlp->dwl_drain = 1;
2695
2696 /* give it up to a second to drain from the chip */
2697 if (!QEMPTY(&wlp->dwl_runq)) {
2698 (void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock,
2699 drv_usectohz(STALETIME), TR_CLOCK_TICK);
2700
2701 if (!QEMPTY(&wlp->dwl_runq)) {
2702 dca_error(dca, "unable to drain device");
2703 mutex_exit(&wlp->dwl_lock);
2704 dca_undrain(dca);
2705 return (EBUSY);
2706 }
2707 }
2708
2709 #ifdef SCHEDDELAY
2710 tid = wlp->dwl_schedtid;
2711 mutex_exit(&wlp->dwl_lock);
2712
2713 /*
2714 * untimeout outside the lock -- this is safe because we
2715 * have set the drain flag, so dca_schedule() will not
2716 * reschedule another timeout
2717 */
2718 if (tid) {
2719 untimeout(tid);
2720 }
2721 #else
2722 mutex_exit(&wlp->dwl_lock);
2723 #endif
2724 }
2725 return (0);
2726 }
2727
2728 void
2729 dca_undrain(dca_t *dca)
2730 {
2731 int mcr;
2732
2733 for (mcr = MCR1; mcr <= MCR2; mcr++) {
2734 dca_worklist_t *wlp = WORKLIST(dca, mcr);
2735 mutex_enter(&wlp->dwl_lock);
2736 wlp->dwl_drain = 0;
2737 dca_schedule(dca, mcr);
2738 mutex_exit(&wlp->dwl_lock);
2739 }
2740 }
2741
2742 /*
2743 * Duplicate the crypto_data_t structure, but point to the original
2744 * buffers.
2745 */
2746 int
2747 dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput)
2748 {
2749 ninput->cd_format = input->cd_format;
2750 ninput->cd_offset = input->cd_offset;
2751 ninput->cd_length = input->cd_length;
2752 ninput->cd_miscdata = input->cd_miscdata;
2753
2754 switch (input->cd_format) {
2755 case CRYPTO_DATA_RAW:
2756 ninput->cd_raw.iov_base = input->cd_raw.iov_base;
2757 ninput->cd_raw.iov_len = input->cd_raw.iov_len;
2758 break;
2759
2760 case CRYPTO_DATA_UIO:
2761 ninput->cd_uio = input->cd_uio;
2762 break;
2763
2764 case CRYPTO_DATA_MBLK:
2765 ninput->cd_mp = input->cd_mp;
2766 break;
2767
2768 default:
2769 DBG(NULL, DWARN,
2770 "dca_dupcrypto: unrecognised crypto data format");
2771 return (CRYPTO_FAILED);
2772 }
2773
2774 return (CRYPTO_SUCCESS);
2775 }
2776
2777 /*
2778 * Performs validation checks on the input and output data structures.
2779 */
2780 int
2781 dca_verifyio(crypto_data_t *input, crypto_data_t *output)
2782 {
2783 int rv = CRYPTO_SUCCESS;
2784
2785 switch (input->cd_format) {
2786 case CRYPTO_DATA_RAW:
2787 break;
2788
2789 case CRYPTO_DATA_UIO:
2790 /* we support only kernel buffer */
2791 if (input->cd_uio->uio_segflg != UIO_SYSSPACE) {
2792 DBG(NULL, DWARN, "non kernel input uio buffer");
2793 rv = CRYPTO_ARGUMENTS_BAD;
2794 }
2795 break;
2796
2797 case CRYPTO_DATA_MBLK:
2798 break;
2799
2800 default:
2801 DBG(NULL, DWARN, "unrecognised input crypto data format");
2802 rv = CRYPTO_ARGUMENTS_BAD;
2803 }
2804
2805 switch (output->cd_format) {
2806 case CRYPTO_DATA_RAW:
2807 break;
2808
2809 case CRYPTO_DATA_UIO:
2810 /* we support only kernel buffer */
2811 if (output->cd_uio->uio_segflg != UIO_SYSSPACE) {
2812 DBG(NULL, DWARN, "non kernel output uio buffer");
2813 rv = CRYPTO_ARGUMENTS_BAD;
2814 }
2815 break;
2816
2817 case CRYPTO_DATA_MBLK:
2818 break;
2819
2820 default:
2821 DBG(NULL, DWARN, "unrecognised output crypto data format");
2822 rv = CRYPTO_ARGUMENTS_BAD;
2823 }
2824
2825 return (rv);
2826 }
2827
2828 /*
2829 * data: source crypto_data_t struct
2830 * off: offset into the source before commencing copy
2831 * count: the amount of data to copy
2832 * dest: destination buffer
2833 */
2834 int
2835 dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest)
2836 {
2837 int rv = CRYPTO_SUCCESS;
2838 uio_t *uiop;
2839 uint_t vec_idx;
2840 size_t cur_len;
2841 mblk_t *mp;
2842
2843 if (count == 0) {
2844 /* We don't want anything so we're done. */
2845 return (rv);
2846 }
2847
2848 /*
2849 * Sanity check that we haven't specified a length greater than the
2850 * offset adjusted size of the buffer.
2851 */
2852 if (count > (data->cd_length - off)) {
2853 return (CRYPTO_DATA_LEN_RANGE);
2854 }
2855
2856 /* Add the internal crypto_data offset to the requested offset. */
2857 off += data->cd_offset;
2858
2859 switch (data->cd_format) {
2860 case CRYPTO_DATA_RAW:
2861 bcopy(data->cd_raw.iov_base + off, dest, count);
2862 break;
2863
2864 case CRYPTO_DATA_UIO:
2865 /*
2866 * Jump to the first iovec containing data to be
2867 * processed.
2868 */
2869 uiop = data->cd_uio;
2870 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
2871 off >= uiop->uio_iov[vec_idx].iov_len;
2872 off -= uiop->uio_iov[vec_idx++].iov_len)
2873 ;
2874 if (vec_idx == uiop->uio_iovcnt) {
2875 /*
2876 * The caller specified an offset that is larger than
2877 * the total size of the buffers it provided.
2878 */
2879 return (CRYPTO_DATA_LEN_RANGE);
2880 }
2881
2882 /*
2883 * Now process the iovecs.
2884 */
2885 while (vec_idx < uiop->uio_iovcnt && count > 0) {
2886 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
2887 off, count);
2888 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
2889 cur_len);
2890 count -= cur_len;
2891 dest += cur_len;
2892 vec_idx++;
2893 off = 0;
2894 }
2895
2896 if (vec_idx == uiop->uio_iovcnt && count > 0) {
2897 /*
2898 * The end of the specified iovec's was reached but
2899 * the length requested could not be processed
2900 * (requested to digest more data than it provided).
2901 */
2902 return (CRYPTO_DATA_LEN_RANGE);
2903 }
2904 break;
2905
2906 case CRYPTO_DATA_MBLK:
2907 /*
2908 * Jump to the first mblk_t containing data to be processed.
2909 */
2910 for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp);
2911 off -= MBLKL(mp), mp = mp->b_cont)
2912 ;
2913 if (mp == NULL) {
2914 /*
2915 * The caller specified an offset that is larger than
2916 * the total size of the buffers it provided.
2917 */
2918 return (CRYPTO_DATA_LEN_RANGE);
2919 }
2920
2921 /*
2922 * Now do the processing on the mblk chain.
2923 */
2924 while (mp != NULL && count > 0) {
2925 cur_len = min(MBLKL(mp) - off, count);
2926 bcopy((char *)(mp->b_rptr + off), dest, cur_len);
2927 count -= cur_len;
2928 dest += cur_len;
2929 mp = mp->b_cont;
2930 off = 0;
2931 }
2932
2933 if (mp == NULL && count > 0) {
2934 /*
2935 * The end of the mblk was reached but the length
2936 * requested could not be processed, (requested to
2937 * digest more data than it provided).
2938 */
2939 return (CRYPTO_DATA_LEN_RANGE);
2940 }
2941 break;
2942
2943 default:
2944 DBG(NULL, DWARN, "unrecognised crypto data format");
2945 rv = CRYPTO_ARGUMENTS_BAD;
2946 }
2947 return (rv);
2948 }
2949
2950
2951 /*
2952 * Performs the input, output or hard scatter/gather checks on the specified
2953 * crypto_data_t struct. Returns true if the data is scatter/gather in nature
2954 * ie fails the test.
2955 */
2956 int
2957 dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val)
2958 {
2959 uio_t *uiop;
2960 mblk_t *mp;
2961 int rv = FALSE;
2962
2963 switch (val) {
2964 case DCA_SG_CONTIG:
2965 /*
2966 * Check for a contiguous data buffer.
2967 */
2968 switch (data->cd_format) {
2969 case CRYPTO_DATA_RAW:
2970 /* Contiguous in nature */
2971 break;
2972
2973 case CRYPTO_DATA_UIO:
2974 if (data->cd_uio->uio_iovcnt > 1)
2975 rv = TRUE;
2976 break;
2977
2978 case CRYPTO_DATA_MBLK:
2979 mp = data->cd_mp;
2980 if (mp->b_cont != NULL)
2981 rv = TRUE;
2982 break;
2983
2984 default:
2985 DBG(NULL, DWARN, "unrecognised crypto data format");
2986 }
2987 break;
2988
2989 case DCA_SG_WALIGN:
2990 /*
2991 * Check for a contiguous data buffer that is 32-bit word
2992 * aligned and is of word multiples in size.
2993 */
2994 switch (data->cd_format) {
2995 case CRYPTO_DATA_RAW:
2996 if ((data->cd_raw.iov_len % sizeof (uint32_t)) ||
2997 ((uintptr_t)data->cd_raw.iov_base %
2998 sizeof (uint32_t))) {
2999 rv = TRUE;
3000 }
3001 break;
3002
3003 case CRYPTO_DATA_UIO:
3004 uiop = data->cd_uio;
3005 if (uiop->uio_iovcnt > 1) {
3006 return (TRUE);
3007 }
3008 /* So there is only one iovec */
3009 if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) ||
3010 ((uintptr_t)uiop->uio_iov[0].iov_base %
3011 sizeof (uint32_t))) {
3012 rv = TRUE;
3013 }
3014 break;
3015
3016 case CRYPTO_DATA_MBLK:
3017 mp = data->cd_mp;
3018 if (mp->b_cont != NULL) {
3019 return (TRUE);
3020 }
3021 /* So there is only one mblk in the chain */
3022 if ((MBLKL(mp) % sizeof (uint32_t)) ||
3023 ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) {
3024 rv = TRUE;
3025 }
3026 break;
3027
3028 default:
3029 DBG(NULL, DWARN, "unrecognised crypto data format");
3030 }
3031 break;
3032
3033 case DCA_SG_PALIGN:
3034 /*
3035 * Check that the data buffer is page aligned and is of
3036 * page multiples in size.
3037 */
3038 switch (data->cd_format) {
3039 case CRYPTO_DATA_RAW:
3040 if ((data->cd_length % dca->dca_pagesize) ||
3041 ((uintptr_t)data->cd_raw.iov_base %
3042 dca->dca_pagesize)) {
3043 rv = TRUE;
3044 }
3045 break;
3046
3047 case CRYPTO_DATA_UIO:
3048 uiop = data->cd_uio;
3049 if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) ||
3050 ((uintptr_t)uiop->uio_iov[0].iov_base %
3051 dca->dca_pagesize)) {
3052 rv = TRUE;
3053 }
3054 break;
3055
3056 case CRYPTO_DATA_MBLK:
3057 mp = data->cd_mp;
3058 if ((MBLKL(mp) % dca->dca_pagesize) ||
3059 ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) {
3060 rv = TRUE;
3061 }
3062 break;
3063
3064 default:
3065 DBG(NULL, DWARN, "unrecognised crypto data format");
3066 }
3067 break;
3068
3069 default:
3070 DBG(NULL, DWARN, "unrecognised scatter/gather param type");
3071 }
3072
3073 return (rv);
3074 }
3075
3076 /*
3077 * Increments the cd_offset and decrements the cd_length as the data is
3078 * gathered from the crypto_data_t struct.
3079 * The data is reverse-copied into the dest buffer if the flag is true.
3080 */
3081 int
3082 dca_gather(crypto_data_t *in, char *dest, int count, int reverse)
3083 {
3084 int rv = CRYPTO_SUCCESS;
3085 uint_t vec_idx;
3086 uio_t *uiop;
3087 off_t off = in->cd_offset;
3088 size_t cur_len;
3089 mblk_t *mp;
3090
3091 switch (in->cd_format) {
3092 case CRYPTO_DATA_RAW:
3093 if (count > in->cd_length) {
3094 /*
3095 * The caller specified a length greater than the
3096 * size of the buffer.
3097 */
3098 return (CRYPTO_DATA_LEN_RANGE);
3099 }
3100 if (reverse)
3101 dca_reverse(in->cd_raw.iov_base + off, dest, count,
3102 count);
3103 else
3104 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
3105 in->cd_offset += count;
3106 in->cd_length -= count;
3107 break;
3108
3109 case CRYPTO_DATA_UIO:
3110 /*
3111 * Jump to the first iovec containing data to be processed.
3112 */
3113 uiop = in->cd_uio;
3114 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3115 off >= uiop->uio_iov[vec_idx].iov_len;
3116 off -= uiop->uio_iov[vec_idx++].iov_len)
3117 ;
3118 if (vec_idx == uiop->uio_iovcnt) {
3119 /*
3120 * The caller specified an offset that is larger than
3121 * the total size of the buffers it provided.
3122 */
3123 return (CRYPTO_DATA_LEN_RANGE);
3124 }
3125
3126 /*
3127 * Now process the iovecs.
3128 */
3129 while (vec_idx < uiop->uio_iovcnt && count > 0) {
3130 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3131 off, count);
3132 count -= cur_len;
3133 if (reverse) {
3134 /* Fill the dest buffer from the end */
3135 dca_reverse(uiop->uio_iov[vec_idx].iov_base +
3136 off, dest+count, cur_len, cur_len);
3137 } else {
3138 bcopy(uiop->uio_iov[vec_idx].iov_base + off,
3139 dest, cur_len);
3140 dest += cur_len;
3141 }
3142 in->cd_offset += cur_len;
3143 in->cd_length -= cur_len;
3144 vec_idx++;
3145 off = 0;
3146 }
3147
3148 if (vec_idx == uiop->uio_iovcnt && count > 0) {
3149 /*
3150 * The end of the specified iovec's was reached but
3151 * the length requested could not be processed
3152 * (requested to digest more data than it provided).
3153 */
3154 return (CRYPTO_DATA_LEN_RANGE);
3155 }
3156 break;
3157
3158 case CRYPTO_DATA_MBLK:
3159 /*
3160 * Jump to the first mblk_t containing data to be processed.
3161 */
3162 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
3163 off -= MBLKL(mp), mp = mp->b_cont)
3164 ;
3165 if (mp == NULL) {
3166 /*
3167 * The caller specified an offset that is larger than
3168 * the total size of the buffers it provided.
3169 */
3170 return (CRYPTO_DATA_LEN_RANGE);
3171 }
3172
3173 /*
3174 * Now do the processing on the mblk chain.
3175 */
3176 while (mp != NULL && count > 0) {
3177 cur_len = min(MBLKL(mp) - off, count);
3178 count -= cur_len;
3179 if (reverse) {
3180 /* Fill the dest buffer from the end */
3181 dca_reverse((char *)(mp->b_rptr + off),
3182 dest+count, cur_len, cur_len);
3183 } else {
3184 bcopy((char *)(mp->b_rptr + off), dest,
3185 cur_len);
3186 dest += cur_len;
3187 }
3188 in->cd_offset += cur_len;
3189 in->cd_length -= cur_len;
3190 mp = mp->b_cont;
3191 off = 0;
3192 }
3193
3194 if (mp == NULL && count > 0) {
3195 /*
3196 * The end of the mblk was reached but the length
3197 * requested could not be processed, (requested to
3198 * digest more data than it provided).
3199 */
3200 return (CRYPTO_DATA_LEN_RANGE);
3201 }
3202 break;
3203
3204 default:
3205 DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format");
3206 rv = CRYPTO_ARGUMENTS_BAD;
3207 }
3208 return (rv);
3209 }
3210
3211 /*
3212 * Increments the cd_offset and decrements the cd_length as the data is
3213 * gathered from the crypto_data_t struct.
3214 */
3215 int
3216 dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest,
3217 int count)
3218 {
3219 int rv = CRYPTO_SUCCESS;
3220 caddr_t baddr;
3221 uint_t vec_idx;
3222 uio_t *uiop;
3223 off_t off = in->cd_offset;
3224 size_t cur_len;
3225 mblk_t *mp;
3226
3227 /* Process the residual first */
3228 if (*residlen > 0) {
3229 uint_t num = min(count, *residlen);
3230 bcopy(resid, dest, num);
3231 *residlen -= num;
3232 if (*residlen > 0) {
3233 /*
3234 * Requested amount 'count' is less than what's in
3235 * the residual, so shuffle any remaining resid to
3236 * the front.
3237 */
3238 baddr = resid + num;
3239 bcopy(baddr, resid, *residlen);
3240 }
3241 dest += num;
3242 count -= num;
3243 }
3244
3245 /* Now process what's in the crypto_data_t structs */
3246 switch (in->cd_format) {
3247 case CRYPTO_DATA_RAW:
3248 if (count > in->cd_length) {
3249 /*
3250 * The caller specified a length greater than the
3251 * size of the buffer.
3252 */
3253 return (CRYPTO_DATA_LEN_RANGE);
3254 }
3255 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
3256 in->cd_offset += count;
3257 in->cd_length -= count;
3258 break;
3259
3260 case CRYPTO_DATA_UIO:
3261 /*
3262 * Jump to the first iovec containing data to be processed.
3263 */
3264 uiop = in->cd_uio;
3265 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3266 off >= uiop->uio_iov[vec_idx].iov_len;
3267 off -= uiop->uio_iov[vec_idx++].iov_len)
3268 ;
3269 if (vec_idx == uiop->uio_iovcnt) {
3270 /*
3271 * The caller specified an offset that is larger than
3272 * the total size of the buffers it provided.
3273 */
3274 return (CRYPTO_DATA_LEN_RANGE);
3275 }
3276
3277 /*
3278 * Now process the iovecs.
3279 */
3280 while (vec_idx < uiop->uio_iovcnt && count > 0) {
3281 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3282 off, count);
3283 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
3284 cur_len);
3285 count -= cur_len;
3286 dest += cur_len;
3287 in->cd_offset += cur_len;
3288 in->cd_length -= cur_len;
3289 vec_idx++;
3290 off = 0;
3291 }
3292
3293 if (vec_idx == uiop->uio_iovcnt && count > 0) {
3294 /*
3295 * The end of the specified iovec's was reached but
3296 * the length requested could not be processed
3297 * (requested to digest more data than it provided).
3298 */
3299 return (CRYPTO_DATA_LEN_RANGE);
3300 }
3301 break;
3302
3303 case CRYPTO_DATA_MBLK:
3304 /*
3305 * Jump to the first mblk_t containing data to be processed.
3306 */
3307 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
3308 off -= MBLKL(mp), mp = mp->b_cont)
3309 ;
3310 if (mp == NULL) {
3311 /*
3312 * The caller specified an offset that is larger than
3313 * the total size of the buffers it provided.
3314 */
3315 return (CRYPTO_DATA_LEN_RANGE);
3316 }
3317
3318 /*
3319 * Now do the processing on the mblk chain.
3320 */
3321 while (mp != NULL && count > 0) {
3322 cur_len = min(MBLKL(mp) - off, count);
3323 bcopy((char *)(mp->b_rptr + off), dest, cur_len);
3324 count -= cur_len;
3325 dest += cur_len;
3326 in->cd_offset += cur_len;
3327 in->cd_length -= cur_len;
3328 mp = mp->b_cont;
3329 off = 0;
3330 }
3331
3332 if (mp == NULL && count > 0) {
3333 /*
3334 * The end of the mblk was reached but the length
3335 * requested could not be processed, (requested to
3336 * digest more data than it provided).
3337 */
3338 return (CRYPTO_DATA_LEN_RANGE);
3339 }
3340 break;
3341
3342 default:
3343 DBG(NULL, DWARN,
3344 "dca_resid_gather: unrecognised crypto data format");
3345 rv = CRYPTO_ARGUMENTS_BAD;
3346 }
3347 return (rv);
3348 }
3349
3350 /*
3351 * Appends the data to the crypto_data_t struct increasing cd_length.
3352 * cd_offset is left unchanged.
3353 * Data is reverse-copied if the flag is TRUE.
3354 */
3355 int
3356 dca_scatter(const char *src, crypto_data_t *out, int count, int reverse)
3357 {
3358 int rv = CRYPTO_SUCCESS;
3359 off_t offset = out->cd_offset + out->cd_length;
3360 uint_t vec_idx;
3361 uio_t *uiop;
3362 size_t cur_len;
3363 mblk_t *mp;
3364
3365 switch (out->cd_format) {
3366 case CRYPTO_DATA_RAW:
3367 if (out->cd_raw.iov_len - offset < count) {
3368 /* Trying to write out more than space available. */
3369 return (CRYPTO_DATA_LEN_RANGE);
3370 }
3371 if (reverse)
3372 dca_reverse((void*) src, out->cd_raw.iov_base + offset,
3373 count, count);
3374 else
3375 bcopy(src, out->cd_raw.iov_base + offset, count);
3376 out->cd_length += count;
3377 break;
3378
3379 case CRYPTO_DATA_UIO:
3380 /*
3381 * Jump to the first iovec that can be written to.
3382 */
3383 uiop = out->cd_uio;
3384 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3385 offset >= uiop->uio_iov[vec_idx].iov_len;
3386 offset -= uiop->uio_iov[vec_idx++].iov_len)
3387 ;
3388 if (vec_idx == uiop->uio_iovcnt) {
3389 /*
3390 * The caller specified an offset that is larger than
3391 * the total size of the buffers it provided.
3392 */
3393 return (CRYPTO_DATA_LEN_RANGE);
3394 }
3395
3396 /*
3397 * Now process the iovecs.
3398 */
3399 while (vec_idx < uiop->uio_iovcnt && count > 0) {
3400 cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3401 offset, count);
3402 count -= cur_len;
3403 if (reverse) {
3404 dca_reverse((void*) (src+count),
3405 uiop->uio_iov[vec_idx].iov_base +
3406 offset, cur_len, cur_len);
3407 } else {
3408 bcopy(src, uiop->uio_iov[vec_idx].iov_base +
3409 offset, cur_len);
3410 src += cur_len;
3411 }
3412 out->cd_length += cur_len;
3413 vec_idx++;
3414 offset = 0;
3415 }
3416
3417 if (vec_idx == uiop->uio_iovcnt && count > 0) {
3418 /*
3419 * The end of the specified iovec's was reached but
3420 * the length requested could not be processed
3421 * (requested to write more data than space provided).
3422 */
3423 return (CRYPTO_DATA_LEN_RANGE);
3424 }
3425 break;
3426
3427 case CRYPTO_DATA_MBLK:
3428 /*
3429 * Jump to the first mblk_t that can be written to.
3430 */
3431 for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp);
3432 offset -= MBLKL(mp), mp = mp->b_cont)
3433 ;
3434 if (mp == NULL) {
3435 /*
3436 * The caller specified an offset that is larger than
3437 * the total size of the buffers it provided.
3438 */
3439 return (CRYPTO_DATA_LEN_RANGE);
3440 }
3441
3442 /*
3443 * Now do the processing on the mblk chain.
3444 */
3445 while (mp != NULL && count > 0) {
3446 cur_len = min(MBLKL(mp) - offset, count);
3447 count -= cur_len;
3448 if (reverse) {
3449 dca_reverse((void*) (src+count),
3450 (char *)(mp->b_rptr + offset), cur_len,
3451 cur_len);
3452 } else {
3453 bcopy(src, (char *)(mp->b_rptr + offset),
3454 cur_len);
3455 src += cur_len;
3456 }
3457 out->cd_length += cur_len;
3458 mp = mp->b_cont;
3459 offset = 0;
3460 }
3461
3462 if (mp == NULL && count > 0) {
3463 /*
3464 * The end of the mblk was reached but the length
3465 * requested could not be processed, (requested to
3466 * digest more data than it provided).
3467 */
3468 return (CRYPTO_DATA_LEN_RANGE);
3469 }
3470 break;
3471
3472 default:
3473 DBG(NULL, DWARN, "unrecognised crypto data format");
3474 rv = CRYPTO_ARGUMENTS_BAD;
3475 }
3476 return (rv);
3477 }
3478
3479 /*
3480 * Compare two byte arrays in reverse order.
3481 * Return 0 if they are identical, 1 otherwise.
3482 */
3483 int
3484 dca_bcmp_reverse(const void *s1, const void *s2, size_t n)
3485 {
3486 int i;
3487 caddr_t src, dst;
3488
3489 if (!n)
3490 return (0);
3491
3492 src = ((caddr_t)s1) + n - 1;
3493 dst = (caddr_t)s2;
3494 for (i = 0; i < n; i++) {
3495 if (*src != *dst)
3496 return (1);
3497 src--;
3498 dst++;
3499 }
3500
3501 return (0);
3502 }
3503
3504
3505 /*
3506 * This calculates the size of a bignum in bits, specifically not counting
3507 * leading zero bits. This size calculation must be done *before* any
3508 * endian reversal takes place (i.e. the numbers are in absolute big-endian
3509 * order.)
3510 */
3511 int
3512 dca_bitlen(unsigned char *bignum, int bytelen)
3513 {
3514 unsigned char msbyte;
3515 int i, j;
3516
3517 for (i = 0; i < bytelen - 1; i++) {
3518 if (bignum[i] != 0) {
3519 break;
3520 }
3521 }
3522 msbyte = bignum[i];
3523 for (j = 8; j > 1; j--) {
3524 if (msbyte & 0x80) {
3525 break;
3526 }
3527 msbyte <<= 1;
3528 }
3529 return ((8 * (bytelen - i - 1)) + j);
3530 }
3531
3532 /*
3533 * This compares to bignums (in big-endian order). It ignores leading
3534 * null bytes. The result semantics follow bcmp, mempcmp, strcmp, etc.
3535 */
3536 int
3537 dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len)
3538 {
3539 while ((n1len > 1) && (*n1 == 0)) {
3540 n1len--;
3541 n1++;
3542 }
3543 while ((n2len > 1) && (*n2 == 0)) {
3544 n2len--;
3545 n2++;
3546 }
3547 if (n1len != n2len) {
3548 return (n1len - n2len);
3549 }
3550 while ((n1len > 1) && (*n1 == *n2)) {
3551 n1++;
3552 n2++;
3553 n1len--;
3554 }
3555 return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2));
3556 }
3557
3558 /*
3559 * Return array of key attributes.
3560 */
3561 crypto_object_attribute_t *
3562 dca_get_key_attr(crypto_key_t *key)
3563 {
3564 if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) ||
3565 (key->ck_count == 0)) {
3566 return (NULL);
3567 }
3568
3569 return (key->ck_attrs);
3570 }
3571
3572 /*
3573 * If attribute type exists valp points to it's 32-bit value.
3574 */
3575 int
3576 dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum,
3577 uint64_t atype, uint32_t *valp)
3578 {
3579 crypto_object_attribute_t *bap;
3580
3581 bap = dca_find_attribute(attrp, atnum, atype);
3582 if (bap == NULL) {
3583 return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
3584 }
3585
3586 *valp = *bap->oa_value;
3587
3588 return (CRYPTO_SUCCESS);
3589 }
3590
3591 /*
3592 * If attribute type exists data contains the start address of the value,
3593 * and numelems contains it's length.
3594 */
3595 int
3596 dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum,
3597 uint64_t atype, void **data, unsigned int *numelems)
3598 {
3599 crypto_object_attribute_t *bap;
3600
3601 bap = dca_find_attribute(attrp, atnum, atype);
3602 if (bap == NULL) {
3603 return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
3604 }
3605
3606 *data = bap->oa_value;
3607 *numelems = bap->oa_value_len;
3608
3609 return (CRYPTO_SUCCESS);
3610 }
3611
3612 /*
3613 * Finds entry of specified name. If it is not found dca_find_attribute returns
3614 * NULL.
3615 */
3616 crypto_object_attribute_t *
3617 dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum,
3618 uint64_t atype)
3619 {
3620 while (atnum) {
3621 if (attrp->oa_type == atype)
3622 return (attrp);
3623 atnum--;
3624 attrp++;
3625 }
3626 return (NULL);
3627 }
3628
3629 /*
3630 * Return the address of the first data buffer. If the data format is
3631 * unrecognised return NULL.
3632 */
3633 caddr_t
3634 dca_bufdaddr(crypto_data_t *data)
3635 {
3636 switch (data->cd_format) {
3637 case CRYPTO_DATA_RAW:
3638 return (data->cd_raw.iov_base + data->cd_offset);
3639 case CRYPTO_DATA_UIO:
3640 return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset);
3641 case CRYPTO_DATA_MBLK:
3642 return ((char *)data->cd_mp->b_rptr + data->cd_offset);
3643 default:
3644 DBG(NULL, DWARN,
3645 "dca_bufdaddr: unrecognised crypto data format");
3646 return (NULL);
3647 }
3648 }
3649
3650 static caddr_t
3651 dca_bufdaddr_out(crypto_data_t *data)
3652 {
3653 size_t offset = data->cd_offset + data->cd_length;
3654
3655 switch (data->cd_format) {
3656 case CRYPTO_DATA_RAW:
3657 return (data->cd_raw.iov_base + offset);
3658 case CRYPTO_DATA_UIO:
3659 return (data->cd_uio->uio_iov[0].iov_base + offset);
3660 case CRYPTO_DATA_MBLK:
3661 return ((char *)data->cd_mp->b_rptr + offset);
3662 default:
3663 DBG(NULL, DWARN,
3664 "dca_bufdaddr_out: unrecognised crypto data format");
3665 return (NULL);
3666 }
3667 }
3668
3669 /*
3670 * Control entry points.
3671 */
3672
3673 /* ARGSUSED */
3674 static void
3675 dca_provider_status(crypto_provider_handle_t provider, uint_t *status)
3676 {
3677 *status = CRYPTO_PROVIDER_READY;
3678 }
3679
3680 /*
3681 * Cipher (encrypt/decrypt) entry points.
3682 */
3683
3684 /* ARGSUSED */
3685 static int
3686 dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
3687 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
3688 crypto_req_handle_t req)
3689 {
3690 int error = CRYPTO_FAILED;
3691 dca_t *softc;
3692
3693 softc = DCA_SOFTC_FROM_CTX(ctx);
3694 DBG(softc, DENTRY, "dca_encrypt_init: started");
3695
3696 /* check mechanism */
3697 switch (mechanism->cm_type) {
3698 case DES_CBC_MECH_INFO_TYPE:
3699 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3700 DR_ENCRYPT);
3701 break;
3702 case DES3_CBC_MECH_INFO_TYPE:
3703 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3704 DR_ENCRYPT | DR_TRIPLE);
3705 break;
3706 case RSA_PKCS_MECH_INFO_TYPE:
3707 case RSA_X_509_MECH_INFO_TYPE:
3708 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
3709 break;
3710 default:
3711 cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type "
3712 "0x%llx\n", (unsigned long long)mechanism->cm_type);
3713 error = CRYPTO_MECHANISM_INVALID;
3714 }
3715
3716 DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error);
3717
3718 if (error == CRYPTO_SUCCESS)
3719 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
3720 &softc->dca_ctx_list_lock);
3721
3722 return (error);
3723 }
3724
3725 /* ARGSUSED */
3726 static int
3727 dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
3728 crypto_data_t *ciphertext, crypto_req_handle_t req)
3729 {
3730 int error = CRYPTO_FAILED;
3731 dca_t *softc;
3732
3733 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3734 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3735
3736 softc = DCA_SOFTC_FROM_CTX(ctx);
3737 DBG(softc, DENTRY, "dca_encrypt: started");
3738
3739 /* handle inplace ops */
3740 if (!ciphertext) {
3741 dca_request_t *reqp = ctx->cc_provider_private;
3742 reqp->dr_flags |= DR_INPLACE;
3743 ciphertext = plaintext;
3744 }
3745
3746 /* check mechanism */
3747 switch (DCA_MECH_FROM_CTX(ctx)) {
3748 case DES_CBC_MECH_INFO_TYPE:
3749 error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT);
3750 break;
3751 case DES3_CBC_MECH_INFO_TYPE:
3752 error = dca_3des(ctx, plaintext, ciphertext, req,
3753 DR_ENCRYPT | DR_TRIPLE);
3754 break;
3755 case RSA_PKCS_MECH_INFO_TYPE:
3756 case RSA_X_509_MECH_INFO_TYPE:
3757 error = dca_rsastart(ctx, plaintext, ciphertext, req,
3758 DCA_RSA_ENC);
3759 break;
3760 default:
3761 /* Should never reach here */
3762 cmn_err(CE_WARN, "dca_encrypt: unexpected mech type "
3763 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3764 error = CRYPTO_MECHANISM_INVALID;
3765 }
3766
3767 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
3768 (error != CRYPTO_BUFFER_TOO_SMALL)) {
3769 ciphertext->cd_length = 0;
3770 }
3771
3772 DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error);
3773
3774 return (error);
3775 }
3776
3777 /* ARGSUSED */
3778 static int
3779 dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext,
3780 crypto_data_t *ciphertext, crypto_req_handle_t req)
3781 {
3782 int error = CRYPTO_FAILED;
3783 dca_t *softc;
3784
3785 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3786 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3787
3788 softc = DCA_SOFTC_FROM_CTX(ctx);
3789 DBG(softc, DENTRY, "dca_encrypt_update: started");
3790
3791 /* handle inplace ops */
3792 if (!ciphertext) {
3793 dca_request_t *reqp = ctx->cc_provider_private;
3794 reqp->dr_flags |= DR_INPLACE;
3795 ciphertext = plaintext;
3796 }
3797
3798 /* check mechanism */
3799 switch (DCA_MECH_FROM_CTX(ctx)) {
3800 case DES_CBC_MECH_INFO_TYPE:
3801 error = dca_3desupdate(ctx, plaintext, ciphertext, req,
3802 DR_ENCRYPT);
3803 break;
3804 case DES3_CBC_MECH_INFO_TYPE:
3805 error = dca_3desupdate(ctx, plaintext, ciphertext, req,
3806 DR_ENCRYPT | DR_TRIPLE);
3807 break;
3808 default:
3809 /* Should never reach here */
3810 cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type "
3811 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3812 error = CRYPTO_MECHANISM_INVALID;
3813 }
3814
3815 DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error);
3816
3817 return (error);
3818 }
3819
3820 /* ARGSUSED */
3821 static int
3822 dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
3823 crypto_req_handle_t req)
3824 {
3825 int error = CRYPTO_FAILED;
3826 dca_t *softc;
3827
3828 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3829 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3830
3831 softc = DCA_SOFTC_FROM_CTX(ctx);
3832 DBG(softc, DENTRY, "dca_encrypt_final: started");
3833
3834 /* check mechanism */
3835 switch (DCA_MECH_FROM_CTX(ctx)) {
3836 case DES_CBC_MECH_INFO_TYPE:
3837 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT);
3838 break;
3839 case DES3_CBC_MECH_INFO_TYPE:
3840 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE);
3841 break;
3842 default:
3843 /* Should never reach here */
3844 cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type "
3845 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3846 error = CRYPTO_MECHANISM_INVALID;
3847 }
3848
3849 DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error);
3850
3851 return (error);
3852 }
3853
3854 /* ARGSUSED */
3855 static int
3856 dca_encrypt_atomic(crypto_provider_handle_t provider,
3857 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
3858 crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
3859 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
3860 {
3861 int error = CRYPTO_FAILED;
3862 dca_t *softc = (dca_t *)provider;
3863
3864 DBG(softc, DENTRY, "dca_encrypt_atomic: started");
3865
3866 if (ctx_template != NULL)
3867 return (CRYPTO_ARGUMENTS_BAD);
3868
3869 /* handle inplace ops */
3870 if (!ciphertext) {
3871 ciphertext = plaintext;
3872 }
3873
3874 /* check mechanism */
3875 switch (mechanism->cm_type) {
3876 case DES_CBC_MECH_INFO_TYPE:
3877 error = dca_3desatomic(provider, session_id, mechanism, key,
3878 plaintext, ciphertext, KM_SLEEP, req,
3879 DR_ENCRYPT | DR_ATOMIC);
3880 break;
3881 case DES3_CBC_MECH_INFO_TYPE:
3882 error = dca_3desatomic(provider, session_id, mechanism, key,
3883 plaintext, ciphertext, KM_SLEEP, req,
3884 DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC);
3885 break;
3886 case RSA_PKCS_MECH_INFO_TYPE:
3887 case RSA_X_509_MECH_INFO_TYPE:
3888 error = dca_rsaatomic(provider, session_id, mechanism, key,
3889 plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC);
3890 break;
3891 default:
3892 cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type "
3893 "0x%llx\n", (unsigned long long)mechanism->cm_type);
3894 error = CRYPTO_MECHANISM_INVALID;
3895 }
3896
3897 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
3898 ciphertext->cd_length = 0;
3899 }
3900
3901 DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error);
3902
3903 return (error);
3904 }
3905
3906 /* ARGSUSED */
3907 static int
3908 dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
3909 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
3910 crypto_req_handle_t req)
3911 {
3912 int error = CRYPTO_FAILED;
3913 dca_t *softc;
3914
3915 softc = DCA_SOFTC_FROM_CTX(ctx);
3916 DBG(softc, DENTRY, "dca_decrypt_init: started");
3917
3918 /* check mechanism */
3919 switch (mechanism->cm_type) {
3920 case DES_CBC_MECH_INFO_TYPE:
3921 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3922 DR_DECRYPT);
3923 break;
3924 case DES3_CBC_MECH_INFO_TYPE:
3925 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3926 DR_DECRYPT | DR_TRIPLE);
3927 break;
3928 case RSA_PKCS_MECH_INFO_TYPE:
3929 case RSA_X_509_MECH_INFO_TYPE:
3930 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
3931 break;
3932 default:
3933 cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type "
3934 "0x%llx\n", (unsigned long long)mechanism->cm_type);
3935 error = CRYPTO_MECHANISM_INVALID;
3936 }
3937
3938 DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error);
3939
3940 if (error == CRYPTO_SUCCESS)
3941 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
3942 &softc->dca_ctx_list_lock);
3943
3944 return (error);
3945 }
3946
3947 /* ARGSUSED */
3948 static int
3949 dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
3950 crypto_data_t *plaintext, crypto_req_handle_t req)
3951 {
3952 int error = CRYPTO_FAILED;
3953 dca_t *softc;
3954
3955 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3956 return (CRYPTO_OPERATION_NOT_INITIALIZED);
3957
3958 softc = DCA_SOFTC_FROM_CTX(ctx);
3959 DBG(softc, DENTRY, "dca_decrypt: started");
3960
3961 /* handle inplace ops */
3962 if (!plaintext) {
3963 dca_request_t *reqp = ctx->cc_provider_private;
3964 reqp->dr_flags |= DR_INPLACE;
3965 plaintext = ciphertext;
3966 }
3967
3968 /* check mechanism */
3969 switch (DCA_MECH_FROM_CTX(ctx)) {
3970 case DES_CBC_MECH_INFO_TYPE:
3971 error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT);
3972 break;
3973 case DES3_CBC_MECH_INFO_TYPE:
3974 error = dca_3des(ctx, ciphertext, plaintext, req,
3975 DR_DECRYPT | DR_TRIPLE);
3976 break;
3977 case RSA_PKCS_MECH_INFO_TYPE:
3978 case RSA_X_509_MECH_INFO_TYPE:
3979 error = dca_rsastart(ctx, ciphertext, plaintext, req,
3980 DCA_RSA_DEC);
3981 break;
3982 default:
3983 /* Should never reach here */
3984 cmn_err(CE_WARN, "dca_decrypt: unexpected mech type "
3985 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3986 error = CRYPTO_MECHANISM_INVALID;
3987 }
3988
3989 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
3990 (error != CRYPTO_BUFFER_TOO_SMALL)) {
3991 if (plaintext)
3992 plaintext->cd_length = 0;
3993 }
3994
3995 DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error);
3996
3997 return (error);
3998 }
3999
4000 /* ARGSUSED */
4001 static int
4002 dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
4003 crypto_data_t *plaintext, crypto_req_handle_t req)
4004 {
4005 int error = CRYPTO_FAILED;
4006 dca_t *softc;
4007
4008 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4009 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4010
4011 softc = DCA_SOFTC_FROM_CTX(ctx);
4012 DBG(softc, DENTRY, "dca_decrypt_update: started");
4013
4014 /* handle inplace ops */
4015 if (!plaintext) {
4016 dca_request_t *reqp = ctx->cc_provider_private;
4017 reqp->dr_flags |= DR_INPLACE;
4018 plaintext = ciphertext;
4019 }
4020
4021 /* check mechanism */
4022 switch (DCA_MECH_FROM_CTX(ctx)) {
4023 case DES_CBC_MECH_INFO_TYPE:
4024 error = dca_3desupdate(ctx, ciphertext, plaintext, req,
4025 DR_DECRYPT);
4026 break;
4027 case DES3_CBC_MECH_INFO_TYPE:
4028 error = dca_3desupdate(ctx, ciphertext, plaintext, req,
4029 DR_DECRYPT | DR_TRIPLE);
4030 break;
4031 default:
4032 /* Should never reach here */
4033 cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type "
4034 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4035 error = CRYPTO_MECHANISM_INVALID;
4036 }
4037
4038 DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error);
4039
4040 return (error);
4041 }
4042
4043 /* ARGSUSED */
4044 static int
4045 dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext,
4046 crypto_req_handle_t req)
4047 {
4048 int error = CRYPTO_FAILED;
4049 dca_t *softc;
4050
4051 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4052 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4053
4054 softc = DCA_SOFTC_FROM_CTX(ctx);
4055 DBG(softc, DENTRY, "dca_decrypt_final: started");
4056
4057 /* check mechanism */
4058 switch (DCA_MECH_FROM_CTX(ctx)) {
4059 case DES_CBC_MECH_INFO_TYPE:
4060 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT);
4061 break;
4062 case DES3_CBC_MECH_INFO_TYPE:
4063 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE);
4064 break;
4065 default:
4066 /* Should never reach here */
4067 cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type "
4068 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4069 error = CRYPTO_MECHANISM_INVALID;
4070 }
4071
4072 DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error);
4073
4074 return (error);
4075 }
4076
4077 /* ARGSUSED */
4078 static int
4079 dca_decrypt_atomic(crypto_provider_handle_t provider,
4080 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4081 crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
4082 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4083 {
4084 int error = CRYPTO_FAILED;
4085 dca_t *softc = (dca_t *)provider;
4086
4087 DBG(softc, DENTRY, "dca_decrypt_atomic: started");
4088
4089 if (ctx_template != NULL)
4090 return (CRYPTO_ARGUMENTS_BAD);
4091
4092 /* handle inplace ops */
4093 if (!plaintext) {
4094 plaintext = ciphertext;
4095 }
4096
4097 /* check mechanism */
4098 switch (mechanism->cm_type) {
4099 case DES_CBC_MECH_INFO_TYPE:
4100 error = dca_3desatomic(provider, session_id, mechanism, key,
4101 ciphertext, plaintext, KM_SLEEP, req,
4102 DR_DECRYPT | DR_ATOMIC);
4103 break;
4104 case DES3_CBC_MECH_INFO_TYPE:
4105 error = dca_3desatomic(provider, session_id, mechanism, key,
4106 ciphertext, plaintext, KM_SLEEP, req,
4107 DR_DECRYPT | DR_TRIPLE | DR_ATOMIC);
4108 break;
4109 case RSA_PKCS_MECH_INFO_TYPE:
4110 case RSA_X_509_MECH_INFO_TYPE:
4111 error = dca_rsaatomic(provider, session_id, mechanism, key,
4112 ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC);
4113 break;
4114 default:
4115 cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type "
4116 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4117 error = CRYPTO_MECHANISM_INVALID;
4118 }
4119
4120 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
4121 plaintext->cd_length = 0;
4122 }
4123
4124 DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error);
4125
4126 return (error);
4127 }
4128
4129 /*
4130 * Sign entry points.
4131 */
4132
4133 /* ARGSUSED */
4134 static int
4135 dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4136 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4137 crypto_req_handle_t req)
4138 {
4139 int error = CRYPTO_FAILED;
4140 dca_t *softc;
4141
4142 softc = DCA_SOFTC_FROM_CTX(ctx);
4143 DBG(softc, DENTRY, "dca_sign_init: started\n");
4144
4145 if (ctx_template != NULL)
4146 return (CRYPTO_ARGUMENTS_BAD);
4147
4148 /* check mechanism */
4149 switch (mechanism->cm_type) {
4150 case RSA_PKCS_MECH_INFO_TYPE:
4151 case RSA_X_509_MECH_INFO_TYPE:
4152 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4153 break;
4154 case DSA_MECH_INFO_TYPE:
4155 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
4156 DCA_DSA_SIGN);
4157 break;
4158 default:
4159 cmn_err(CE_WARN, "dca_sign_init: unexpected mech type "
4160 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4161 error = CRYPTO_MECHANISM_INVALID;
4162 }
4163
4164 DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error);
4165
4166 if (error == CRYPTO_SUCCESS)
4167 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4168 &softc->dca_ctx_list_lock);
4169
4170 return (error);
4171 }
4172
4173 static int
4174 dca_sign(crypto_ctx_t *ctx, crypto_data_t *data,
4175 crypto_data_t *signature, crypto_req_handle_t req)
4176 {
4177 int error = CRYPTO_FAILED;
4178 dca_t *softc;
4179
4180 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4181 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4182
4183 softc = DCA_SOFTC_FROM_CTX(ctx);
4184 DBG(softc, DENTRY, "dca_sign: started\n");
4185
4186 /* check mechanism */
4187 switch (DCA_MECH_FROM_CTX(ctx)) {
4188 case RSA_PKCS_MECH_INFO_TYPE:
4189 case RSA_X_509_MECH_INFO_TYPE:
4190 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN);
4191 break;
4192 case DSA_MECH_INFO_TYPE:
4193 error = dca_dsa_sign(ctx, data, signature, req);
4194 break;
4195 default:
4196 cmn_err(CE_WARN, "dca_sign: unexpected mech type "
4197 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4198 error = CRYPTO_MECHANISM_INVALID;
4199 }
4200
4201 DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error);
4202
4203 return (error);
4204 }
4205
4206 /* ARGSUSED */
4207 static int
4208 dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data,
4209 crypto_req_handle_t req)
4210 {
4211 int error = CRYPTO_MECHANISM_INVALID;
4212 dca_t *softc;
4213
4214 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4215 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4216
4217 softc = DCA_SOFTC_FROM_CTX(ctx);
4218 DBG(softc, DENTRY, "dca_sign_update: started\n");
4219
4220 cmn_err(CE_WARN, "dca_sign_update: unexpected mech type "
4221 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4222
4223 DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error);
4224
4225 return (error);
4226 }
4227
4228 /* ARGSUSED */
4229 static int
4230 dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
4231 crypto_req_handle_t req)
4232 {
4233 int error = CRYPTO_MECHANISM_INVALID;
4234 dca_t *softc;
4235
4236 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4237 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4238
4239 softc = DCA_SOFTC_FROM_CTX(ctx);
4240 DBG(softc, DENTRY, "dca_sign_final: started\n");
4241
4242 cmn_err(CE_WARN, "dca_sign_final: unexpected mech type "
4243 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4244
4245 DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error);
4246
4247 return (error);
4248 }
4249
4250 static int
4251 dca_sign_atomic(crypto_provider_handle_t provider,
4252 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4253 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4254 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4255 {
4256 int error = CRYPTO_FAILED;
4257 dca_t *softc = (dca_t *)provider;
4258
4259 DBG(softc, DENTRY, "dca_sign_atomic: started\n");
4260
4261 if (ctx_template != NULL)
4262 return (CRYPTO_ARGUMENTS_BAD);
4263
4264 /* check mechanism */
4265 switch (mechanism->cm_type) {
4266 case RSA_PKCS_MECH_INFO_TYPE:
4267 case RSA_X_509_MECH_INFO_TYPE:
4268 error = dca_rsaatomic(provider, session_id, mechanism, key,
4269 data, signature, KM_SLEEP, req, DCA_RSA_SIGN);
4270 break;
4271 case DSA_MECH_INFO_TYPE:
4272 error = dca_dsaatomic(provider, session_id, mechanism, key,
4273 data, signature, KM_SLEEP, req, DCA_DSA_SIGN);
4274 break;
4275 default:
4276 cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type "
4277 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4278 error = CRYPTO_MECHANISM_INVALID;
4279 }
4280
4281 DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error);
4282
4283 return (error);
4284 }
4285
4286 /* ARGSUSED */
4287 static int
4288 dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4289 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4290 crypto_req_handle_t req)
4291 {
4292 int error = CRYPTO_FAILED;
4293 dca_t *softc;
4294
4295 softc = DCA_SOFTC_FROM_CTX(ctx);
4296 DBG(softc, DENTRY, "dca_sign_recover_init: started\n");
4297
4298 if (ctx_template != NULL)
4299 return (CRYPTO_ARGUMENTS_BAD);
4300
4301 /* check mechanism */
4302 switch (mechanism->cm_type) {
4303 case RSA_PKCS_MECH_INFO_TYPE:
4304 case RSA_X_509_MECH_INFO_TYPE:
4305 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4306 break;
4307 default:
4308 cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type "
4309 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4310 error = CRYPTO_MECHANISM_INVALID;
4311 }
4312
4313 DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error);
4314
4315 if (error == CRYPTO_SUCCESS)
4316 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4317 &softc->dca_ctx_list_lock);
4318
4319 return (error);
4320 }
4321
4322 static int
4323 dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data,
4324 crypto_data_t *signature, crypto_req_handle_t req)
4325 {
4326 int error = CRYPTO_FAILED;
4327 dca_t *softc;
4328
4329 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4330 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4331
4332 softc = DCA_SOFTC_FROM_CTX(ctx);
4333 DBG(softc, DENTRY, "dca_sign_recover: started\n");
4334
4335 /* check mechanism */
4336 switch (DCA_MECH_FROM_CTX(ctx)) {
4337 case RSA_PKCS_MECH_INFO_TYPE:
4338 case RSA_X_509_MECH_INFO_TYPE:
4339 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR);
4340 break;
4341 default:
4342 cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type "
4343 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4344 error = CRYPTO_MECHANISM_INVALID;
4345 }
4346
4347 DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error);
4348
4349 return (error);
4350 }
4351
4352 static int
4353 dca_sign_recover_atomic(crypto_provider_handle_t provider,
4354 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4355 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4356 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4357 {
4358 int error = CRYPTO_FAILED;
4359 dca_t *softc = (dca_t *)provider;
4360
4361 DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n");
4362
4363 if (ctx_template != NULL)
4364 return (CRYPTO_ARGUMENTS_BAD);
4365
4366 /* check mechanism */
4367 switch (mechanism->cm_type) {
4368 case RSA_PKCS_MECH_INFO_TYPE:
4369 case RSA_X_509_MECH_INFO_TYPE:
4370 error = dca_rsaatomic(provider, session_id, mechanism, key,
4371 data, signature, KM_SLEEP, req, DCA_RSA_SIGNR);
4372 break;
4373 default:
4374 cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type"
4375 " 0x%llx\n", (unsigned long long)mechanism->cm_type);
4376 error = CRYPTO_MECHANISM_INVALID;
4377 }
4378
4379 DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error);
4380
4381 return (error);
4382 }
4383
4384 /*
4385 * Verify entry points.
4386 */
4387
4388 /* ARGSUSED */
4389 static int
4390 dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4391 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4392 crypto_req_handle_t req)
4393 {
4394 int error = CRYPTO_FAILED;
4395 dca_t *softc;
4396
4397 softc = DCA_SOFTC_FROM_CTX(ctx);
4398 DBG(softc, DENTRY, "dca_verify_init: started\n");
4399
4400 if (ctx_template != NULL)
4401 return (CRYPTO_ARGUMENTS_BAD);
4402
4403 /* check mechanism */
4404 switch (mechanism->cm_type) {
4405 case RSA_PKCS_MECH_INFO_TYPE:
4406 case RSA_X_509_MECH_INFO_TYPE:
4407 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4408 break;
4409 case DSA_MECH_INFO_TYPE:
4410 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
4411 DCA_DSA_VRFY);
4412 break;
4413 default:
4414 cmn_err(CE_WARN, "dca_verify_init: unexpected mech type "
4415 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4416 error = CRYPTO_MECHANISM_INVALID;
4417 }
4418
4419 DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error);
4420
4421 if (error == CRYPTO_SUCCESS)
4422 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4423 &softc->dca_ctx_list_lock);
4424
4425 return (error);
4426 }
4427
4428 static int
4429 dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
4430 crypto_req_handle_t req)
4431 {
4432 int error = CRYPTO_FAILED;
4433 dca_t *softc;
4434
4435 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4436 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4437
4438 softc = DCA_SOFTC_FROM_CTX(ctx);
4439 DBG(softc, DENTRY, "dca_verify: started\n");
4440
4441 /* check mechanism */
4442 switch (DCA_MECH_FROM_CTX(ctx)) {
4443 case RSA_PKCS_MECH_INFO_TYPE:
4444 case RSA_X_509_MECH_INFO_TYPE:
4445 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY);
4446 break;
4447 case DSA_MECH_INFO_TYPE:
4448 error = dca_dsa_verify(ctx, data, signature, req);
4449 break;
4450 default:
4451 cmn_err(CE_WARN, "dca_verify: unexpected mech type "
4452 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4453 error = CRYPTO_MECHANISM_INVALID;
4454 }
4455
4456 DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error);
4457
4458 return (error);
4459 }
4460
4461 /* ARGSUSED */
4462 static int
4463 dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
4464 crypto_req_handle_t req)
4465 {
4466 int error = CRYPTO_MECHANISM_INVALID;
4467 dca_t *softc;
4468
4469 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4470 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4471
4472 softc = DCA_SOFTC_FROM_CTX(ctx);
4473 DBG(softc, DENTRY, "dca_verify_update: started\n");
4474
4475 cmn_err(CE_WARN, "dca_verify_update: unexpected mech type "
4476 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4477
4478 DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error);
4479
4480 return (error);
4481 }
4482
4483 /* ARGSUSED */
4484 static int
4485 dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
4486 crypto_req_handle_t req)
4487 {
4488 int error = CRYPTO_MECHANISM_INVALID;
4489 dca_t *softc;
4490
4491 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4492 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4493
4494 softc = DCA_SOFTC_FROM_CTX(ctx);
4495 DBG(softc, DENTRY, "dca_verify_final: started\n");
4496
4497 cmn_err(CE_WARN, "dca_verify_final: unexpected mech type "
4498 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4499
4500 DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error);
4501
4502 return (error);
4503 }
4504
4505 static int
4506 dca_verify_atomic(crypto_provider_handle_t provider,
4507 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4508 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4509 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4510 {
4511 int error = CRYPTO_FAILED;
4512 dca_t *softc = (dca_t *)provider;
4513
4514 DBG(softc, DENTRY, "dca_verify_atomic: started\n");
4515
4516 if (ctx_template != NULL)
4517 return (CRYPTO_ARGUMENTS_BAD);
4518
4519 /* check mechanism */
4520 switch (mechanism->cm_type) {
4521 case RSA_PKCS_MECH_INFO_TYPE:
4522 case RSA_X_509_MECH_INFO_TYPE:
4523 error = dca_rsaatomic(provider, session_id, mechanism, key,
4524 signature, data, KM_SLEEP, req, DCA_RSA_VRFY);
4525 break;
4526 case DSA_MECH_INFO_TYPE:
4527 error = dca_dsaatomic(provider, session_id, mechanism, key,
4528 data, signature, KM_SLEEP, req, DCA_DSA_VRFY);
4529 break;
4530 default:
4531 cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type "
4532 "0x%llx\n", (unsigned long long)mechanism->cm_type);
4533 error = CRYPTO_MECHANISM_INVALID;
4534 }
4535
4536 DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error);
4537
4538 return (error);
4539 }
4540
4541 /* ARGSUSED */
4542 static int
4543 dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4544 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4545 crypto_req_handle_t req)
4546 {
4547 int error = CRYPTO_MECHANISM_INVALID;
4548 dca_t *softc;
4549
4550 softc = DCA_SOFTC_FROM_CTX(ctx);
4551 DBG(softc, DENTRY, "dca_verify_recover_init: started\n");
4552
4553 if (ctx_template != NULL)
4554 return (CRYPTO_ARGUMENTS_BAD);
4555
4556 /* check mechanism */
4557 switch (mechanism->cm_type) {
4558 case RSA_PKCS_MECH_INFO_TYPE:
4559 case RSA_X_509_MECH_INFO_TYPE:
4560 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4561 break;
4562 default:
4563 cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type"
4564 " 0x%llx\n", (unsigned long long)mechanism->cm_type);
4565 }
4566
4567 DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error);
4568
4569 if (error == CRYPTO_SUCCESS)
4570 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4571 &softc->dca_ctx_list_lock);
4572
4573 return (error);
4574 }
4575
4576 static int
4577 dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
4578 crypto_data_t *data, crypto_req_handle_t req)
4579 {
4580 int error = CRYPTO_MECHANISM_INVALID;
4581 dca_t *softc;
4582
4583 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4584 return (CRYPTO_OPERATION_NOT_INITIALIZED);
4585
4586 softc = DCA_SOFTC_FROM_CTX(ctx);
4587 DBG(softc, DENTRY, "dca_verify_recover: started\n");
4588
4589 /* check mechanism */
4590 switch (DCA_MECH_FROM_CTX(ctx)) {
4591 case RSA_PKCS_MECH_INFO_TYPE:
4592 case RSA_X_509_MECH_INFO_TYPE:
4593 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR);
4594 break;
4595 default:
4596 cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type "
4597 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4598 }
4599
4600 DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error);
4601
4602 return (error);
4603 }
4604
4605 static int
4606 dca_verify_recover_atomic(crypto_provider_handle_t provider,
4607 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4608 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4609 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4610 {
4611 int error = CRYPTO_MECHANISM_INVALID;
4612 dca_t *softc = (dca_t *)provider;
4613
4614 DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n");
4615
4616 if (ctx_template != NULL)
4617 return (CRYPTO_ARGUMENTS_BAD);
4618
4619 /* check mechanism */
4620 switch (mechanism->cm_type) {
4621 case RSA_PKCS_MECH_INFO_TYPE:
4622 case RSA_X_509_MECH_INFO_TYPE:
4623 error = dca_rsaatomic(provider, session_id, mechanism, key,
4624 signature, data, KM_SLEEP, req, DCA_RSA_VRFYR);
4625 break;
4626 default:
4627 cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech "
4628 "type 0x%llx\n", (unsigned long long)mechanism->cm_type);
4629 error = CRYPTO_MECHANISM_INVALID;
4630 }
4631
4632 DBG(softc, DENTRY,
4633 "dca_verify_recover_atomic: done, err = 0x%x", error);
4634
4635 return (error);
4636 }
4637
4638 /*
4639 * Random number entry points.
4640 */
4641
4642 /* ARGSUSED */
4643 static int
4644 dca_generate_random(crypto_provider_handle_t provider,
4645 crypto_session_id_t session_id,
4646 uchar_t *buf, size_t len, crypto_req_handle_t req)
4647 {
4648 int error = CRYPTO_FAILED;
4649 dca_t *softc = (dca_t *)provider;
4650
4651 DBG(softc, DENTRY, "dca_generate_random: started");
4652
4653 error = dca_rng(softc, buf, len, req);
4654
4655 DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error);
4656
4657 return (error);
4658 }
4659
4660 /*
4661 * Context management entry points.
4662 */
4663
4664 int
4665 dca_free_context(crypto_ctx_t *ctx)
4666 {
4667 int error = CRYPTO_SUCCESS;
4668 dca_t *softc;
4669
4670 softc = DCA_SOFTC_FROM_CTX(ctx);
4671 DBG(softc, DENTRY, "dca_free_context: entered");
4672
4673 if (ctx->cc_provider_private == NULL)
4674 return (error);
4675
4676 dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock);
4677
4678 error = dca_free_context_low(ctx);
4679
4680 DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error);
4681
4682 return (error);
4683 }
4684
4685 static int
4686 dca_free_context_low(crypto_ctx_t *ctx)
4687 {
4688 int error = CRYPTO_SUCCESS;
4689
4690 /* check mechanism */
4691 switch (DCA_MECH_FROM_CTX(ctx)) {
4692 case DES_CBC_MECH_INFO_TYPE:
4693 case DES3_CBC_MECH_INFO_TYPE:
4694 dca_3desctxfree(ctx);
4695 break;
4696 case RSA_PKCS_MECH_INFO_TYPE:
4697 case RSA_X_509_MECH_INFO_TYPE:
4698 dca_rsactxfree(ctx);
4699 break;
4700 case DSA_MECH_INFO_TYPE:
4701 dca_dsactxfree(ctx);
4702 break;
4703 default:
4704 /* Should never reach here */
4705 cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type "
4706 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4707 error = CRYPTO_MECHANISM_INVALID;
4708 }
4709
4710 return (error);
4711 }
4712
4713
4714 /* Free any unfreed private context. It is called in detach. */
4715 static void
4716 dca_free_context_list(dca_t *dca)
4717 {
4718 dca_listnode_t *node;
4719 crypto_ctx_t ctx;
4720
4721 (void) memset(&ctx, 0, sizeof (ctx));
4722 ctx.cc_provider = dca;
4723
4724 while ((node = dca_delist2(&dca->dca_ctx_list,
4725 &dca->dca_ctx_list_lock)) != NULL) {
4726 ctx.cc_provider_private = node;
4727 (void) dca_free_context_low(&ctx);
4728 }
4729 }
4730
4731 static int
4732 ext_info_sym(crypto_provider_handle_t prov,
4733 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
4734 {
4735 return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM));
4736 }
4737
4738 static int
4739 ext_info_asym(crypto_provider_handle_t prov,
4740 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
4741 {
4742 int rv;
4743
4744 rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM);
4745 /* The asymmetric cipher slot supports random */
4746 ext_info->ei_flags |= CRYPTO_EXTF_RNG;
4747
4748 return (rv);
4749 }
4750
4751 /* ARGSUSED */
4752 static int
4753 ext_info_base(crypto_provider_handle_t prov,
4754 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id)
4755 {
4756 dca_t *dca = (dca_t *)prov;
4757 int len;
4758
4759 /* Label */
4760 (void) sprintf((char *)ext_info->ei_label, "%s/%d %s",
4761 ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id);
4762 len = strlen((char *)ext_info->ei_label);
4763 (void) memset(ext_info->ei_label + len, ' ',
4764 CRYPTO_EXT_SIZE_LABEL - len);
4765
4766 /* Manufacturer ID */
4767 (void) sprintf((char *)ext_info->ei_manufacturerID, "%s",
4768 DCA_MANUFACTURER_ID);
4769 len = strlen((char *)ext_info->ei_manufacturerID);
4770 (void) memset(ext_info->ei_manufacturerID + len, ' ',
4771 CRYPTO_EXT_SIZE_MANUF - len);
4772
4773 /* Model */
4774 (void) sprintf((char *)ext_info->ei_model, dca->dca_model);
4775
4776 DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model);
4777
4778 len = strlen((char *)ext_info->ei_model);
4779 (void) memset(ext_info->ei_model + len, ' ',
4780 CRYPTO_EXT_SIZE_MODEL - len);
4781
4782 /* Serial Number. Blank for Deimos */
4783 (void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL);
4784
4785 ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED;
4786
4787 ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO;
4788 ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO;
4789 ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO;
4790 ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO;
4791 ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO;
4792 ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO;
4793 ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO;
4794 ext_info->ei_hardware_version.cv_major = 0;
4795 ext_info->ei_hardware_version.cv_minor = 0;
4796 ext_info->ei_firmware_version.cv_major = 0;
4797 ext_info->ei_firmware_version.cv_minor = 0;
4798
4799 /* Time. No need to be supplied for token without a clock */
4800 ext_info->ei_time[0] = '\000';
4801
4802 return (CRYPTO_SUCCESS);
4803 }
4804
4805 static void
4806 dca_fma_init(dca_t *dca)
4807 {
4808 ddi_iblock_cookie_t fm_ibc;
4809 int fm_capabilities = DDI_FM_EREPORT_CAPABLE |
4810 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE |
4811 DDI_FM_ERRCB_CAPABLE;
4812
4813 /* Read FMA capabilities from dca.conf file (if present) */
4814 dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip,
4815 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
4816 fm_capabilities);
4817
4818 DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities);
4819
4820 /* Only register with IO Fault Services if we have some capability */
4821 if (dca->fm_capabilities) {
4822 dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC;
4823 dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR;
4824
4825 /* Register capabilities with IO Fault Services */
4826 ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc);
4827 DBG(dca, DWARN, "fm_capable() = 0x%x",
4828 ddi_fm_capable(dca->dca_dip));
4829
4830 /*
4831 * Initialize pci ereport capabilities if ereport capable
4832 */
4833 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
4834 DDI_FM_ERRCB_CAP(dca->fm_capabilities))
4835 pci_ereport_setup(dca->dca_dip);
4836
4837 /*
4838 * Initialize callback mutex and register error callback if
4839 * error callback capable.
4840 */
4841 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4842 ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb,
4843 (void *)dca);
4844 }
4845 } else {
4846 /*
4847 * These fields have to be cleared of FMA if there are no
4848 * FMA capabilities at runtime.
4849 */
4850 dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC;
4851 dca_dmaattr.dma_attr_flags = 0;
4852 }
4853 }
4854
4855
4856 static void
4857 dca_fma_fini(dca_t *dca)
4858 {
4859 /* Only unregister FMA capabilities if we registered some */
4860 if (dca->fm_capabilities) {
4861
4862 /*
4863 * Release any resources allocated by pci_ereport_setup()
4864 */
4865 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
4866 DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4867 pci_ereport_teardown(dca->dca_dip);
4868 }
4869
4870 /*
4871 * Free callback mutex and un-register error callback if
4872 * error callback capable.
4873 */
4874 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4875 ddi_fm_handler_unregister(dca->dca_dip);
4876 }
4877
4878 /* Unregister from IO Fault Services */
4879 ddi_fm_fini(dca->dca_dip);
4880 DBG(dca, DWARN, "fm_capable() = 0x%x",
4881 ddi_fm_capable(dca->dca_dip));
4882 }
4883 }
4884
4885
4886 /*
4887 * The IO fault service error handling callback function
4888 */
4889 /*ARGSUSED*/
4890 static int
4891 dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
4892 {
4893 dca_t *dca = (dca_t *)impl_data;
4894
4895 pci_ereport_post(dip, err, NULL);
4896 if (err->fme_status == DDI_FM_FATAL) {
4897 dca_failure(dca, DDI_DATAPATH_FAULT,
4898 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
4899 "fault PCI in FMA callback.");
4900 }
4901 return (err->fme_status);
4902 }
4903
4904
4905 static int
4906 dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
4907 dca_fma_eclass_t eclass_index)
4908 {
4909 ddi_fm_error_t de;
4910 int version = 0;
4911
4912 ddi_fm_acc_err_get(handle, &de, version);
4913 if (de.fme_status != DDI_FM_OK) {
4914 dca_failure(dca, DDI_DATAPATH_FAULT,
4915 eclass_index, fm_ena_increment(de.fme_ena),
4916 CRYPTO_DEVICE_ERROR, "");
4917 return (DDI_FAILURE);
4918 }
4919
4920 return (DDI_SUCCESS);
4921 }
4922
4923 int
4924 dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle,
4925 dca_fma_eclass_t eclass_index)
4926 {
4927 ddi_fm_error_t de;
4928 int version = 0;
4929
4930 ddi_fm_dma_err_get(handle, &de, version);
4931 if (de.fme_status != DDI_FM_OK) {
4932 dca_failure(dca, DDI_DATAPATH_FAULT,
4933 eclass_index, fm_ena_increment(de.fme_ena),
4934 CRYPTO_DEVICE_ERROR, "");
4935 return (DDI_FAILURE);
4936 }
4937 return (DDI_SUCCESS);
4938 }
4939
4940 static uint64_t
4941 dca_ena(uint64_t ena)
4942 {
4943 if (ena == 0)
4944 ena = fm_ena_generate(0, FM_ENA_FMT1);
4945 else
4946 ena = fm_ena_increment(ena);
4947 return (ena);
4948 }
4949
4950 static char *
4951 dca_fma_eclass_string(char *model, dca_fma_eclass_t index)
4952 {
4953 if (strstr(model, "500"))
4954 return (dca_fma_eclass_sca500[index]);
4955 else
4956 return (dca_fma_eclass_sca1000[index]);
4957 }