Print this page
4896 Performance improvements for KCF AES modes
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/common/crypto/modes/ctr.c
+++ new/usr/src/common/crypto/modes/ctr.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 +/*
26 + * Copyright 2015 by Saso Kiselkov. All rights reserved.
27 + */
25 28
26 29 #ifndef _KERNEL
27 30 #include <strings.h>
28 31 #include <limits.h>
29 32 #include <assert.h>
30 33 #include <security/cryptoki.h>
31 34 #endif
32 35
33 36 #include <sys/types.h>
37 +#define INLINE_CRYPTO_GET_PTRS
34 38 #include <modes/modes.h>
35 39 #include <sys/crypto/common.h>
36 40 #include <sys/crypto/impl.h>
37 41 #include <sys/byteorder.h>
42 +#include <sys/cmn_err.h>
38 43
44 +boolean_t ctr_fastpath_enabled = B_TRUE;
45 +
39 46 /*
40 47 * Encrypt and decrypt multiple blocks of data in counter mode.
41 48 */
42 49 int
43 50 ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
44 51 crypto_data_t *out, size_t block_size,
45 52 int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
46 - void (*xor_block)(uint8_t *, uint8_t *))
53 + void (*xor_block)(const uint8_t *, uint8_t *),
54 + int (*cipher_ctr)(const void *ks, const uint8_t *pt, uint8_t *ct,
55 + uint64_t len, uint64_t counter[2]))
47 56 {
48 57 size_t remainder = length;
49 58 size_t need;
50 59 uint8_t *datap = (uint8_t *)data;
51 60 uint8_t *blockp;
52 61 uint8_t *lastp;
53 62 void *iov_or_mp;
54 63 offset_t offset;
55 64 uint8_t *out_data_1;
56 65 uint8_t *out_data_2;
57 66 size_t out_data_1_len;
58 67 uint64_t lower_counter, upper_counter;
59 68
69 + /*
70 + * CTR encryption/decryption fastpath requirements:
71 + * - fastpath is enabled
72 + * - algorithm-specific acceleration function is available
73 + * - input is block-aligned
74 + * - the counter value won't overflow the lower counter mask
75 + * - output is a single contiguous region and doesn't alias input
76 + */
77 + if (ctr_fastpath_enabled && cipher_ctr != NULL &&
78 + ctx->ctr_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
79 + ntohll(ctx->ctr_cb[1]) <= ctx->ctr_lower_mask -
80 + length / block_size && CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
81 + cipher_ctr(ctx->ctr_keysched, (uint8_t *)data,
82 + CRYPTO_DATA_FIRST_BLOCK(out), length, ctx->ctr_cb);
83 + out->cd_offset += length;
84 + return (CRYPTO_SUCCESS);
85 + }
86 +
60 87 if (length + ctx->ctr_remainder_len < block_size) {
61 88 /* accumulate bytes here and return */
62 89 bcopy(datap,
63 90 (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len,
64 91 length);
65 92 ctx->ctr_remainder_len += length;
66 93 ctx->ctr_copy_to = datap;
67 94 return (CRYPTO_SUCCESS);
68 95 }
69 96
70 97 lastp = (uint8_t *)ctx->ctr_cb;
71 98 if (out != NULL)
72 99 crypto_init_ptrs(out, &iov_or_mp, &offset);
73 100
74 101 do {
75 102 /* Unprocessed data from last call. */
76 103 if (ctx->ctr_remainder_len > 0) {
77 104 need = block_size - ctx->ctr_remainder_len;
78 105
79 106 if (need > remainder)
80 107 return (CRYPTO_DATA_LEN_RANGE);
81 108
82 109 bcopy(datap, &((uint8_t *)ctx->ctr_remainder)
83 110 [ctx->ctr_remainder_len], need);
84 111
85 112 blockp = (uint8_t *)ctx->ctr_remainder;
86 113 } else {
87 114 blockp = datap;
88 115 }
89 116
90 117 /* ctr_cb is the counter block */
91 118 cipher(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
92 119 (uint8_t *)ctx->ctr_tmp);
93 120
94 121 lastp = (uint8_t *)ctx->ctr_tmp;
95 122
96 123 /*
97 124 * Increment Counter.
98 125 */
99 126 lower_counter = ntohll(ctx->ctr_cb[1] & ctx->ctr_lower_mask);
100 127 lower_counter = htonll(lower_counter + 1);
101 128 lower_counter &= ctx->ctr_lower_mask;
102 129 ctx->ctr_cb[1] = (ctx->ctr_cb[1] & ~(ctx->ctr_lower_mask)) |
103 130 lower_counter;
104 131
105 132 /* wrap around */
106 133 if (lower_counter == 0) {
107 134 upper_counter =
108 135 ntohll(ctx->ctr_cb[0] & ctx->ctr_upper_mask);
109 136 upper_counter = htonll(upper_counter + 1);
110 137 upper_counter &= ctx->ctr_upper_mask;
111 138 ctx->ctr_cb[0] =
112 139 (ctx->ctr_cb[0] & ~(ctx->ctr_upper_mask)) |
113 140 upper_counter;
114 141 }
115 142
116 143 /*
117 144 * XOR encrypted counter block with the current clear block.
118 145 */
119 146 xor_block(blockp, lastp);
120 147
121 148 if (out == NULL) {
122 149 if (ctx->ctr_remainder_len > 0) {
123 150 bcopy(lastp, ctx->ctr_copy_to,
124 151 ctx->ctr_remainder_len);
125 152 bcopy(lastp + ctx->ctr_remainder_len, datap,
126 153 need);
127 154 }
128 155 } else {
129 156 crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
130 157 &out_data_1_len, &out_data_2, block_size);
131 158
132 159 /* copy block to where it belongs */
133 160 bcopy(lastp, out_data_1, out_data_1_len);
134 161 if (out_data_2 != NULL) {
135 162 bcopy(lastp + out_data_1_len, out_data_2,
136 163 block_size - out_data_1_len);
137 164 }
138 165 /* update offset */
139 166 out->cd_offset += block_size;
140 167 }
141 168
142 169 /* Update pointer to next block of data to be processed. */
143 170 if (ctx->ctr_remainder_len != 0) {
144 171 datap += need;
145 172 ctx->ctr_remainder_len = 0;
146 173 } else {
147 174 datap += block_size;
148 175 }
149 176
150 177 remainder = (size_t)&data[length] - (size_t)datap;
151 178
152 179 /* Incomplete last block. */
153 180 if (remainder > 0 && remainder < block_size) {
154 181 bcopy(datap, ctx->ctr_remainder, remainder);
155 182 ctx->ctr_remainder_len = remainder;
156 183 ctx->ctr_copy_to = datap;
157 184 goto out;
158 185 }
159 186 ctx->ctr_copy_to = NULL;
160 187
161 188 } while (remainder > 0);
162 189
163 190 out:
164 191 return (CRYPTO_SUCCESS);
165 192 }
166 193
167 194 int
168 195 ctr_mode_final(ctr_ctx_t *ctx, crypto_data_t *out,
169 196 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
170 197 {
171 198 uint8_t *lastp;
172 199 void *iov_or_mp;
173 200 offset_t offset;
174 201 uint8_t *out_data_1;
175 202 uint8_t *out_data_2;
176 203 size_t out_data_1_len;
177 204 uint8_t *p;
178 205 int i;
179 206
180 207 if (out->cd_length < ctx->ctr_remainder_len)
181 208 return (CRYPTO_DATA_LEN_RANGE);
182 209
183 210 encrypt_block(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
184 211 (uint8_t *)ctx->ctr_tmp);
185 212
186 213 lastp = (uint8_t *)ctx->ctr_tmp;
187 214 p = (uint8_t *)ctx->ctr_remainder;
188 215 for (i = 0; i < ctx->ctr_remainder_len; i++) {
189 216 p[i] ^= lastp[i];
190 217 }
191 218
192 219 crypto_init_ptrs(out, &iov_or_mp, &offset);
193 220 crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
194 221 &out_data_1_len, &out_data_2, ctx->ctr_remainder_len);
195 222
196 223 bcopy(p, out_data_1, out_data_1_len);
197 224 if (out_data_2 != NULL) {
|
↓ open down ↓ |
128 lines elided |
↑ open up ↑ |
198 225 bcopy((uint8_t *)p + out_data_1_len,
199 226 out_data_2, ctx->ctr_remainder_len - out_data_1_len);
200 227 }
201 228 out->cd_offset += ctx->ctr_remainder_len;
202 229 ctx->ctr_remainder_len = 0;
203 230 return (CRYPTO_SUCCESS);
204 231 }
205 232
206 233 int
207 234 ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
208 -void (*copy_block)(uint8_t *, uint8_t *))
235 + void (*copy_block)(const uint8_t *, uint8_t *))
209 236 {
210 237 uint64_t upper_mask = 0;
211 238 uint64_t lower_mask = 0;
212 239
213 240 if (count == 0 || count > 128) {
214 241 return (CRYPTO_MECHANISM_PARAM_INVALID);
215 242 }
216 243 /* upper 64 bits of the mask */
217 244 if (count >= 64) {
218 245 count -= 64;
219 246 upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1;
220 247 lower_mask = UINT64_MAX;
221 248 } else {
222 249 /* now the lower 63 bits */
223 250 lower_mask = (1ULL << count) - 1;
224 251 }
225 252 ctr_ctx->ctr_lower_mask = htonll(lower_mask);
226 253 ctr_ctx->ctr_upper_mask = htonll(upper_mask);
227 254
228 255 copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb);
229 256 ctr_ctx->ctr_lastp = (uint8_t *)&ctr_ctx->ctr_cb[0];
230 257 ctr_ctx->ctr_flags |= CTR_MODE;
231 258 return (CRYPTO_SUCCESS);
232 259 }
233 260
234 261 /* ARGSUSED */
235 262 void *
236 263 ctr_alloc_ctx(int kmflag)
237 264 {
238 265 ctr_ctx_t *ctr_ctx;
239 266
240 267 #ifdef _KERNEL
241 268 if ((ctr_ctx = kmem_zalloc(sizeof (ctr_ctx_t), kmflag)) == NULL)
242 269 #else
243 270 if ((ctr_ctx = calloc(1, sizeof (ctr_ctx_t))) == NULL)
244 271 #endif
245 272 return (NULL);
246 273
247 274 ctr_ctx->ctr_flags = CTR_MODE;
248 275 return (ctr_ctx);
249 276 }
|
↓ open down ↓ |
31 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX