Print this page
4896 Performance improvements for KCF AES modes
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/crypto/core/kcf_prov_lib.c
+++ new/usr/src/uts/common/crypto/core/kcf_prov_lib.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 +/*
26 + * Copyright 2015 by Saso Kiselkov. All rights reserved.
27 + */
25 28
26 29 #include <sys/strsun.h>
27 30 #include <sys/systm.h>
28 31 #include <sys/sysmacros.h>
29 32 #include <sys/kmem.h>
30 33 #include <sys/md5.h>
31 34 #include <sys/sha1.h>
32 35 #include <sys/sha2.h>
33 36 #include <modes/modes.h>
34 37 #include <sys/crypto/common.h>
35 38 #include <sys/crypto/impl.h>
36 39
37 40 /*
38 41 * Utility routine to apply the command, 'cmd', to the
39 42 * data in the uio structure.
40 43 */
41 44 int
42 45 crypto_uio_data(crypto_data_t *data, uchar_t *buf, int len, cmd_type_t cmd,
43 46 void *digest_ctx, void (*update)())
44 47 {
45 48 uio_t *uiop = data->cd_uio;
46 49 off_t offset = data->cd_offset;
47 50 size_t length = len;
48 51 uint_t vec_idx;
49 52 size_t cur_len;
50 53 uchar_t *datap;
51 54
52 55 ASSERT(data->cd_format == CRYPTO_DATA_UIO);
53 56 if (uiop->uio_segflg != UIO_SYSSPACE) {
54 57 return (CRYPTO_ARGUMENTS_BAD);
55 58 }
56 59
57 60 /*
58 61 * Jump to the first iovec containing data to be
59 62 * processed.
60 63 */
61 64 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
62 65 offset >= uiop->uio_iov[vec_idx].iov_len;
63 66 offset -= uiop->uio_iov[vec_idx++].iov_len)
64 67 ;
65 68
66 69 if (vec_idx == uiop->uio_iovcnt) {
67 70 /*
68 71 * The caller specified an offset that is larger than
69 72 * the total size of the buffers it provided.
70 73 */
71 74 return (CRYPTO_DATA_LEN_RANGE);
72 75 }
73 76
74 77 while (vec_idx < uiop->uio_iovcnt && length > 0) {
75 78 cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
76 79 offset, length);
77 80
78 81 datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base +
79 82 offset);
80 83 switch (cmd) {
81 84 case COPY_FROM_DATA:
82 85 bcopy(datap, buf, cur_len);
83 86 buf += cur_len;
84 87 break;
85 88 case COPY_TO_DATA:
86 89 bcopy(buf, datap, cur_len);
87 90 buf += cur_len;
88 91 break;
89 92 case COMPARE_TO_DATA:
90 93 if (bcmp(datap, buf, cur_len))
91 94 return (CRYPTO_SIGNATURE_INVALID);
92 95 buf += cur_len;
93 96 break;
94 97 case MD5_DIGEST_DATA:
95 98 case SHA1_DIGEST_DATA:
96 99 case SHA2_DIGEST_DATA:
97 100 case GHASH_DATA:
98 101 update(digest_ctx, datap, cur_len);
99 102 break;
100 103 }
101 104
102 105 length -= cur_len;
103 106 vec_idx++;
104 107 offset = 0;
105 108 }
106 109
107 110 if (vec_idx == uiop->uio_iovcnt && length > 0) {
108 111 /*
109 112 * The end of the specified iovec's was reached but
110 113 * the length requested could not be processed.
111 114 */
112 115 switch (cmd) {
113 116 case COPY_TO_DATA:
114 117 data->cd_length = len;
115 118 return (CRYPTO_BUFFER_TOO_SMALL);
116 119 default:
117 120 return (CRYPTO_DATA_LEN_RANGE);
118 121 }
119 122 }
120 123
121 124 return (CRYPTO_SUCCESS);
122 125 }
123 126
124 127 /*
125 128 * Utility routine to apply the command, 'cmd', to the
126 129 * data in the mblk structure.
127 130 */
128 131 int
129 132 crypto_mblk_data(crypto_data_t *data, uchar_t *buf, int len, cmd_type_t cmd,
130 133 void *digest_ctx, void (*update)())
131 134 {
132 135 off_t offset = data->cd_offset;
133 136 size_t length = len;
134 137 mblk_t *mp;
135 138 size_t cur_len;
136 139 uchar_t *datap;
137 140
138 141 ASSERT(data->cd_format == CRYPTO_DATA_MBLK);
139 142 /*
140 143 * Jump to the first mblk_t containing data to be processed.
141 144 */
142 145 for (mp = data->cd_mp; mp != NULL && offset >= MBLKL(mp);
143 146 offset -= MBLKL(mp), mp = mp->b_cont)
144 147 ;
145 148 if (mp == NULL) {
146 149 /*
147 150 * The caller specified an offset that is larger
148 151 * than the total size of the buffers it provided.
149 152 */
150 153 return (CRYPTO_DATA_LEN_RANGE);
151 154 }
152 155
153 156 /*
154 157 * Now do the processing on the mblk chain.
155 158 */
156 159 while (mp != NULL && length > 0) {
157 160 cur_len = MIN(MBLKL(mp) - offset, length);
158 161
159 162 datap = (uchar_t *)(mp->b_rptr + offset);
160 163 switch (cmd) {
161 164 case COPY_FROM_DATA:
162 165 bcopy(datap, buf, cur_len);
163 166 buf += cur_len;
164 167 break;
165 168 case COPY_TO_DATA:
166 169 bcopy(buf, datap, cur_len);
167 170 buf += cur_len;
168 171 break;
169 172 case COMPARE_TO_DATA:
170 173 if (bcmp(datap, buf, cur_len))
171 174 return (CRYPTO_SIGNATURE_INVALID);
172 175 buf += cur_len;
173 176 break;
174 177 case MD5_DIGEST_DATA:
175 178 case SHA1_DIGEST_DATA:
176 179 case SHA2_DIGEST_DATA:
177 180 case GHASH_DATA:
178 181 update(digest_ctx, datap, cur_len);
179 182 break;
180 183 }
181 184
182 185 length -= cur_len;
183 186 offset = 0;
184 187 mp = mp->b_cont;
185 188 }
186 189
187 190 if (mp == NULL && length > 0) {
188 191 /*
189 192 * The end of the mblk was reached but the length
190 193 * requested could not be processed.
191 194 */
192 195 switch (cmd) {
193 196 case COPY_TO_DATA:
194 197 data->cd_length = len;
195 198 return (CRYPTO_BUFFER_TOO_SMALL);
196 199 default:
197 200 return (CRYPTO_DATA_LEN_RANGE);
198 201 }
199 202 }
200 203
201 204 return (CRYPTO_SUCCESS);
202 205 }
203 206
204 207 /*
205 208 * Utility routine to copy a buffer to a crypto_data structure.
206 209 */
207 210 int
208 211 crypto_put_output_data(uchar_t *buf, crypto_data_t *output, int len)
209 212 {
210 213 switch (output->cd_format) {
211 214 case CRYPTO_DATA_RAW:
212 215 if (output->cd_raw.iov_len < len) {
213 216 output->cd_length = len;
214 217 return (CRYPTO_BUFFER_TOO_SMALL);
215 218 }
216 219 bcopy(buf, (uchar_t *)(output->cd_raw.iov_base +
217 220 output->cd_offset), len);
218 221 break;
219 222
220 223 case CRYPTO_DATA_UIO:
221 224 return (crypto_uio_data(output, buf, len,
222 225 COPY_TO_DATA, NULL, NULL));
223 226
224 227 case CRYPTO_DATA_MBLK:
225 228 return (crypto_mblk_data(output, buf, len,
226 229 COPY_TO_DATA, NULL, NULL));
227 230
228 231 default:
229 232 return (CRYPTO_ARGUMENTS_BAD);
230 233 }
231 234
232 235 return (CRYPTO_SUCCESS);
233 236 }
234 237
235 238 /*
236 239 * Utility routine to get data from a crypto_data structure.
237 240 *
238 241 * '*dptr' contains a pointer to a buffer on return. 'buf'
239 242 * is allocated by the caller and is ignored for CRYPTO_DATA_RAW case.
240 243 */
241 244 int
242 245 crypto_get_input_data(crypto_data_t *input, uchar_t **dptr, uchar_t *buf)
243 246 {
244 247 int rv;
245 248
246 249 switch (input->cd_format) {
247 250 case CRYPTO_DATA_RAW:
248 251 if (input->cd_raw.iov_len < input->cd_length)
249 252 return (CRYPTO_ARGUMENTS_BAD);
250 253 *dptr = (uchar_t *)(input->cd_raw.iov_base +
251 254 input->cd_offset);
252 255 break;
253 256
254 257 case CRYPTO_DATA_UIO:
255 258 if ((rv = crypto_uio_data(input, buf, input->cd_length,
256 259 COPY_FROM_DATA, NULL, NULL)) != CRYPTO_SUCCESS)
257 260 return (rv);
258 261 *dptr = buf;
259 262 break;
260 263
261 264 case CRYPTO_DATA_MBLK:
262 265 if ((rv = crypto_mblk_data(input, buf, input->cd_length,
263 266 COPY_FROM_DATA, NULL, NULL)) != CRYPTO_SUCCESS)
264 267 return (rv);
265 268 *dptr = buf;
266 269 break;
267 270
268 271 default:
269 272 return (CRYPTO_ARGUMENTS_BAD);
270 273 }
271 274
272 275 return (CRYPTO_SUCCESS);
273 276 }
274 277
275 278 int
276 279 crypto_copy_key_to_ctx(crypto_key_t *in_key, crypto_key_t **out_key,
277 280 size_t *out_size, int kmflag)
278 281 {
279 282 int i, count;
280 283 size_t len;
281 284 caddr_t attr_val;
282 285 crypto_object_attribute_t *k_attrs = NULL;
283 286 crypto_key_t *key;
284 287
285 288 ASSERT(in_key->ck_format == CRYPTO_KEY_ATTR_LIST);
286 289
287 290 count = in_key->ck_count;
288 291 /* figure out how much memory to allocate for everything */
289 292 len = sizeof (crypto_key_t) +
290 293 count * sizeof (crypto_object_attribute_t);
291 294 for (i = 0; i < count; i++) {
292 295 len += roundup(in_key->ck_attrs[i].oa_value_len,
293 296 sizeof (caddr_t));
294 297 }
295 298
296 299 /* one big allocation for everything */
297 300 key = kmem_alloc(len, kmflag);
298 301 if (key == NULL)
299 302 return (CRYPTO_HOST_MEMORY);
300 303 k_attrs = (crypto_object_attribute_t *)(void *)((caddr_t)key +
301 304 sizeof (crypto_key_t));
302 305
303 306 attr_val = (caddr_t)k_attrs +
304 307 count * sizeof (crypto_object_attribute_t);
305 308 for (i = 0; i < count; i++) {
306 309 k_attrs[i].oa_type = in_key->ck_attrs[i].oa_type;
307 310 bcopy(in_key->ck_attrs[i].oa_value, attr_val,
308 311 in_key->ck_attrs[i].oa_value_len);
309 312 k_attrs[i].oa_value = attr_val;
310 313 k_attrs[i].oa_value_len = in_key->ck_attrs[i].oa_value_len;
311 314 attr_val += roundup(k_attrs[i].oa_value_len, sizeof (caddr_t));
312 315 }
313 316
314 317 key->ck_format = CRYPTO_KEY_ATTR_LIST;
315 318 key->ck_count = count;
316 319 key->ck_attrs = k_attrs;
317 320 *out_key = key;
318 321 *out_size = len; /* save the size to be freed */
319 322
320 323 return (CRYPTO_SUCCESS);
321 324 }
322 325
323 326 int
324 327 crypto_digest_data(crypto_data_t *data, void *dctx, uchar_t *digest,
325 328 void (*update)(), void (*final)(), uchar_t flag)
326 329 {
327 330 int rv, dlen;
328 331 uchar_t *dptr;
329 332
330 333 ASSERT(flag & CRYPTO_DO_MD5 || flag & CRYPTO_DO_SHA1 ||
331 334 flag & CRYPTO_DO_SHA2);
332 335 if (data == NULL) {
333 336 ASSERT((flag & CRYPTO_DO_UPDATE) == 0);
334 337 goto dofinal;
335 338 }
336 339
337 340 dlen = data->cd_length;
338 341
339 342 if (flag & CRYPTO_DO_UPDATE) {
340 343
341 344 switch (data->cd_format) {
342 345 case CRYPTO_DATA_RAW:
343 346 dptr = (uchar_t *)(data->cd_raw.iov_base +
344 347 data->cd_offset);
345 348
346 349 update(dctx, dptr, dlen);
347 350
348 351 break;
349 352
350 353 case CRYPTO_DATA_UIO:
351 354 if (flag & CRYPTO_DO_MD5)
352 355 rv = crypto_uio_data(data, NULL, dlen,
353 356 MD5_DIGEST_DATA, dctx, update);
354 357
355 358 else if (flag & CRYPTO_DO_SHA1)
356 359 rv = crypto_uio_data(data, NULL, dlen,
357 360 SHA1_DIGEST_DATA, dctx, update);
358 361
359 362 else
360 363 rv = crypto_uio_data(data, NULL, dlen,
361 364 SHA2_DIGEST_DATA, dctx, update);
362 365
363 366 if (rv != CRYPTO_SUCCESS)
364 367 return (rv);
365 368
366 369 break;
367 370
368 371 case CRYPTO_DATA_MBLK:
369 372 if (flag & CRYPTO_DO_MD5)
370 373 rv = crypto_mblk_data(data, NULL, dlen,
371 374 MD5_DIGEST_DATA, dctx, update);
372 375
373 376 else if (flag & CRYPTO_DO_SHA1)
374 377 rv = crypto_mblk_data(data, NULL, dlen,
375 378 SHA1_DIGEST_DATA, dctx, update);
376 379
377 380 else
378 381 rv = crypto_mblk_data(data, NULL, dlen,
379 382 SHA2_DIGEST_DATA, dctx, update);
380 383
381 384 if (rv != CRYPTO_SUCCESS)
382 385 return (rv);
383 386
384 387 break;
385 388 }
386 389 }
387 390
388 391 dofinal:
↓ open down ↓ |
354 lines elided |
↑ open up ↑ |
389 392 if (flag & CRYPTO_DO_FINAL) {
390 393 final(digest, dctx);
391 394 }
392 395
393 396 return (CRYPTO_SUCCESS);
394 397 }
395 398
396 399 int
397 400 crypto_update_iov(void *ctx, crypto_data_t *input, crypto_data_t *output,
398 401 int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
399 - void (*copy_block)(uint8_t *, uint64_t *))
402 + void (*copy_block)(const uint8_t *, uint64_t *))
400 403 {
401 404 common_ctx_t *common_ctx = ctx;
402 405 int rv;
403 406
404 407 if (input->cd_miscdata != NULL) {
405 408 copy_block((uint8_t *)input->cd_miscdata,
406 409 &common_ctx->cc_iv[0]);
407 410 }
408 411
409 412 if (input->cd_raw.iov_len < input->cd_length)
410 413 return (CRYPTO_ARGUMENTS_BAD);
↓ open down ↓ |
1 lines elided |
↑ open up ↑ |
411 414
412 415 rv = (cipher)(ctx, input->cd_raw.iov_base + input->cd_offset,
413 416 input->cd_length, (input == output) ? NULL : output);
414 417
415 418 return (rv);
416 419 }
417 420
418 421 int
419 422 crypto_update_uio(void *ctx, crypto_data_t *input, crypto_data_t *output,
420 423 int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
421 - void (*copy_block)(uint8_t *, uint64_t *))
424 + void (*copy_block)(const uint8_t *, uint64_t *))
422 425 {
423 426 common_ctx_t *common_ctx = ctx;
424 427 uio_t *uiop = input->cd_uio;
425 428 off_t offset = input->cd_offset;
426 429 size_t length = input->cd_length;
427 430 uint_t vec_idx;
428 431 size_t cur_len;
429 432
430 433 if (input->cd_miscdata != NULL) {
431 434 copy_block((uint8_t *)input->cd_miscdata,
432 435 &common_ctx->cc_iv[0]);
433 436 }
434 437
435 438 if (input->cd_uio->uio_segflg != UIO_SYSSPACE) {
436 439 return (CRYPTO_ARGUMENTS_BAD);
437 440 }
438 441
439 442 /*
440 443 * Jump to the first iovec containing data to be
441 444 * processed.
442 445 */
443 446 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
444 447 offset >= uiop->uio_iov[vec_idx].iov_len;
445 448 offset -= uiop->uio_iov[vec_idx++].iov_len)
446 449 ;
447 450 if (vec_idx == uiop->uio_iovcnt) {
448 451 /*
449 452 * The caller specified an offset that is larger than the
450 453 * total size of the buffers it provided.
451 454 */
452 455 return (CRYPTO_DATA_LEN_RANGE);
453 456 }
454 457
455 458 /*
456 459 * Now process the iovecs.
457 460 */
458 461 while (vec_idx < uiop->uio_iovcnt && length > 0) {
459 462 cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
460 463 offset, length);
461 464
462 465 (cipher)(ctx, uiop->uio_iov[vec_idx].iov_base + offset,
463 466 cur_len, (input == output) ? NULL : output);
464 467
465 468 length -= cur_len;
466 469 vec_idx++;
467 470 offset = 0;
468 471 }
469 472
470 473 if (vec_idx == uiop->uio_iovcnt && length > 0) {
471 474 /*
472 475 * The end of the specified iovec's was reached but
473 476 * the length requested could not be processed, i.e.
474 477 * The caller requested to digest more data than it provided.
475 478 */
↓ open down ↓ |
44 lines elided |
↑ open up ↑ |
476 479
477 480 return (CRYPTO_DATA_LEN_RANGE);
478 481 }
479 482
480 483 return (CRYPTO_SUCCESS);
481 484 }
482 485
483 486 int
484 487 crypto_update_mp(void *ctx, crypto_data_t *input, crypto_data_t *output,
485 488 int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
486 - void (*copy_block)(uint8_t *, uint64_t *))
489 + void (*copy_block)(const uint8_t *, uint64_t *))
487 490 {
488 491 common_ctx_t *common_ctx = ctx;
489 492 off_t offset = input->cd_offset;
490 493 size_t length = input->cd_length;
491 494 mblk_t *mp;
492 495 size_t cur_len;
493 496
494 497 if (input->cd_miscdata != NULL) {
495 498 copy_block((uint8_t *)input->cd_miscdata,
496 499 &common_ctx->cc_iv[0]);
497 500 }
498 501
499 502 /*
500 503 * Jump to the first mblk_t containing data to be processed.
501 504 */
502 505 for (mp = input->cd_mp; mp != NULL && offset >= MBLKL(mp);
503 506 offset -= MBLKL(mp), mp = mp->b_cont)
504 507 ;
505 508 if (mp == NULL) {
506 509 /*
507 510 * The caller specified an offset that is larger than the
508 511 * total size of the buffers it provided.
509 512 */
510 513 return (CRYPTO_DATA_LEN_RANGE);
511 514 }
512 515
513 516 /*
514 517 * Now do the processing on the mblk chain.
515 518 */
516 519 while (mp != NULL && length > 0) {
517 520 cur_len = MIN(MBLKL(mp) - offset, length);
518 521 (cipher)(ctx, (char *)(mp->b_rptr + offset), cur_len,
519 522 (input == output) ? NULL : output);
520 523
521 524 length -= cur_len;
522 525 offset = 0;
523 526 mp = mp->b_cont;
524 527 }
525 528
526 529 if (mp == NULL && length > 0) {
527 530 /*
528 531 * The end of the mblk was reached but the length requested
529 532 * could not be processed, i.e. The caller requested
530 533 * to digest more data than it provided.
531 534 */
532 535 return (CRYPTO_DATA_LEN_RANGE);
533 536 }
534 537
535 538 return (CRYPTO_SUCCESS);
536 539 }
537 540
538 541 /*
539 542 * Utility routine to look up a attribute of type, 'type',
540 543 * in the key.
541 544 */
542 545 int
543 546 crypto_get_key_attr(crypto_key_t *key, crypto_attr_type_t type,
544 547 uchar_t **value, ssize_t *value_len)
545 548 {
546 549 int i;
547 550
548 551 ASSERT(key->ck_format == CRYPTO_KEY_ATTR_LIST);
549 552 for (i = 0; i < key->ck_count; i++) {
550 553 if (key->ck_attrs[i].oa_type == type) {
551 554 *value = (uchar_t *)key->ck_attrs[i].oa_value;
552 555 *value_len = key->ck_attrs[i].oa_value_len;
553 556 return (CRYPTO_SUCCESS);
554 557 }
555 558 }
556 559
557 560 return (CRYPTO_FAILED);
558 561 }
↓ open down ↓ |
62 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX