Print this page
4896 Performance improvements for KCF AES modes
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/common/crypto/modes/ecb.c
+++ new/usr/src/common/crypto/modes/ecb.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 +/*
26 + * Copyright 2015 by Saso Kiselkov. All rights reserved.
27 + */
25 28
26 29 #ifndef _KERNEL
27 30 #include <strings.h>
28 31 #include <limits.h>
29 32 #include <assert.h>
30 33 #include <security/cryptoki.h>
31 34 #endif
32 35
33 36 #include <sys/types.h>
37 +#define INLINE_CRYPTO_GET_PTRS
34 38 #include <modes/modes.h>
35 39 #include <sys/crypto/common.h>
36 40 #include <sys/crypto/impl.h>
37 41
42 +boolean_t ecb_fastpath_enabled = B_TRUE;
43 +
38 44 /*
39 45 * Algorithm independent ECB functions.
46 + * `cipher' is a single-block version of the cipher function to be performed
47 + * on each input block. `cipher_ecb' is an optional parameter, which if
48 + * passed and the input/output conditions allow it, will be invoked for the
49 + * entire input buffer once to accelerate the operation.
40 50 */
41 51 int
42 52 ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length,
43 53 crypto_data_t *out, size_t block_size,
44 - int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
54 + int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
55 + int (*cipher_ecb)(const void *ks, const uint8_t *pt, uint8_t *ct,
56 + uint64_t len))
45 57 {
46 58 size_t remainder = length;
47 59 size_t need;
48 60 uint8_t *datap = (uint8_t *)data;
49 61 uint8_t *blockp;
50 62 uint8_t *lastp;
51 63 void *iov_or_mp;
52 64 offset_t offset;
53 65 uint8_t *out_data_1;
54 66 uint8_t *out_data_2;
55 67 size_t out_data_1_len;
56 68
69 + /*
70 + * ECB encryption/decryption fastpath requirements:
71 + * - fastpath is enabled
72 + * - caller passed an accelerated ECB version of the cipher function
73 + * - input is block-aligned
74 + * - output is a single contiguous region or the user requested that
75 + * we overwrite their input buffer (input/output aliasing allowed)
76 + */
77 + if (ecb_fastpath_enabled && cipher_ecb != NULL &&
78 + ctx->ecb_remainder_len == 0 && length % block_size == 0 &&
79 + (out == NULL || CRYPTO_DATA_IS_SINGLE_BLOCK(out))) {
80 + if (out == NULL) {
81 + cipher_ecb(ctx->ecb_keysched, (uint8_t *)data,
82 + (uint8_t *)data, length);
83 + } else {
84 + cipher_ecb(ctx->ecb_keysched, (uint8_t *)data,
85 + CRYPTO_DATA_FIRST_BLOCK(out), length);
86 + out->cd_offset += length;
87 + }
88 + return (CRYPTO_SUCCESS);
89 + }
90 +
57 91 if (length + ctx->ecb_remainder_len < block_size) {
58 92 /* accumulate bytes here and return */
59 93 bcopy(datap,
60 94 (uint8_t *)ctx->ecb_remainder + ctx->ecb_remainder_len,
61 95 length);
62 96 ctx->ecb_remainder_len += length;
63 97 ctx->ecb_copy_to = datap;
64 98 return (CRYPTO_SUCCESS);
65 99 }
66 100
67 101 lastp = (uint8_t *)ctx->ecb_iv;
68 102 if (out != NULL)
69 103 crypto_init_ptrs(out, &iov_or_mp, &offset);
70 104
71 105 do {
72 106 /* Unprocessed data from last call. */
73 107 if (ctx->ecb_remainder_len > 0) {
74 108 need = block_size - ctx->ecb_remainder_len;
75 109
76 110 if (need > remainder)
77 111 return (CRYPTO_DATA_LEN_RANGE);
78 112
79 113 bcopy(datap, &((uint8_t *)ctx->ecb_remainder)
80 114 [ctx->ecb_remainder_len], need);
81 115
82 116 blockp = (uint8_t *)ctx->ecb_remainder;
83 117 } else {
84 118 blockp = datap;
85 119 }
86 120
87 121 if (out == NULL) {
88 122 cipher(ctx->ecb_keysched, blockp, blockp);
89 123
90 124 ctx->ecb_lastp = blockp;
91 125 lastp = blockp;
92 126
93 127 if (ctx->ecb_remainder_len > 0) {
94 128 bcopy(blockp, ctx->ecb_copy_to,
95 129 ctx->ecb_remainder_len);
96 130 bcopy(blockp + ctx->ecb_remainder_len, datap,
97 131 need);
98 132 }
99 133 } else {
100 134 cipher(ctx->ecb_keysched, blockp, lastp);
101 135 crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
102 136 &out_data_1_len, &out_data_2, block_size);
103 137
104 138 /* copy block to where it belongs */
105 139 bcopy(lastp, out_data_1, out_data_1_len);
106 140 if (out_data_2 != NULL) {
107 141 bcopy(lastp + out_data_1_len, out_data_2,
108 142 block_size - out_data_1_len);
109 143 }
110 144 /* update offset */
111 145 out->cd_offset += block_size;
112 146 }
113 147
114 148 /* Update pointer to next block of data to be processed. */
115 149 if (ctx->ecb_remainder_len != 0) {
116 150 datap += need;
117 151 ctx->ecb_remainder_len = 0;
118 152 } else {
119 153 datap += block_size;
120 154 }
121 155
122 156 remainder = (size_t)&data[length] - (size_t)datap;
123 157
124 158 /* Incomplete last block. */
125 159 if (remainder > 0 && remainder < block_size) {
126 160 bcopy(datap, ctx->ecb_remainder, remainder);
127 161 ctx->ecb_remainder_len = remainder;
128 162 ctx->ecb_copy_to = datap;
129 163 goto out;
130 164 }
131 165 ctx->ecb_copy_to = NULL;
132 166
133 167 } while (remainder > 0);
134 168
135 169 out:
136 170 return (CRYPTO_SUCCESS);
137 171 }
138 172
139 173 /* ARGSUSED */
140 174 void *
141 175 ecb_alloc_ctx(int kmflag)
142 176 {
143 177 ecb_ctx_t *ecb_ctx;
144 178
145 179 #ifdef _KERNEL
146 180 if ((ecb_ctx = kmem_zalloc(sizeof (ecb_ctx_t), kmflag)) == NULL)
147 181 #else
148 182 if ((ecb_ctx = calloc(1, sizeof (ecb_ctx_t))) == NULL)
149 183 #endif
150 184 return (NULL);
151 185
152 186 ecb_ctx->ecb_flags = ECB_MODE;
153 187 return (ecb_ctx);
154 188 }
|
↓ open down ↓ |
88 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX