1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009 Intel Corporation
24 * All Rights Reserved.
25 */
26 /*
27 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31 /*
32 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
33 * instructions. This file contains an accelerated
34 * Galois Field Multiplication implementation.
35 *
36 * PCLMULQDQ is used to accelerate the most time-consuming part of GHASH,
37 * carry-less multiplication. More information about PCLMULQDQ can be
38 * found at:
39 * http://software.intel.com/en-us/articles/
40 * carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/
41 *
42 */
43
44 /*
45 * ====================================================================
46 * OpenSolaris OS modifications
47 *
48 * This source originates as file galois_hash_asm.c from
49 * Intel Corporation dated September 21, 2009.
50 *
51 * This OpenSolaris version has these major changes from the original source:
52 *
53 * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
54 * /usr/include/sys/asm_linkage.h, lint(1B) guards, and a dummy C function
55 * definition for lint.
56 *
57 * 2. Formatted code, added comments, and added #includes and #defines.
58 *
59 * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
60 * calling kpreempt_disable() and kpreempt_enable().
61 * If the TS bit is not set, Save and restore %xmm registers at the beginning
62 * and end of function calls (%xmm* registers are not saved and restored by
63 * during kernel thread preemption).
64 *
65 * 4. Removed code to perform hashing. This is already done with C macro
66 * GHASH in gcm.c. For better performance, this removed code should be
67 * reintegrated in the future to replace the C GHASH macro.
68 *
69 * 5. Added code to byte swap 16-byte input and output.
70 *
71 * 6. Folded in comments from the original C source with embedded assembly
72 * (SB_w_shift_xor.c)
73 *
74 * 7. Renamed function and reordered parameters to match OpenSolaris:
75 * Intel interface:
76 * void galois_hash_asm(unsigned char *hk, unsigned char *s,
77 * unsigned char *d, int length)
78 * OpenSolaris OS interface:
79 * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
80 * ====================================================================
81 */
82
83
84 #if defined(lint) || defined(__lint)
85
86 #include <sys/types.h>
87
88 /* ARGSUSED */
89 void
90 gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) {
91 }
92
93 #else /* lint */
94
95 #include <sys/asm_linkage.h>
96 #include <sys/controlregs.h>
97 #ifdef _KERNEL
98 #include <sys/machprivregs.h>
99 #endif
100
101 #ifdef _KERNEL
102 /*
103 * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is,
104 * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it
105 * uses it to pass P2 to syscall.
106 * This also occurs with the STTS macro, but we don't care if
107 * P2 (%rsi) is modified just before function exit.
108 * The CLTS and STTS macros push and pop P1 (%rdi) already.
109 */
110 #ifdef __xpv
111 #define PROTECTED_CLTS \
112 push %rsi; \
113 CLTS; \
114 pop %rsi
115 #else
116 #define PROTECTED_CLTS \
117 CLTS
118 #endif /* __xpv */
119
120 /*
121 * If CR0_TS is not set, align stack (with push %rbp) and push
122 * %xmm0 - %xmm10 on stack, otherwise clear CR0_TS
123 */
124 #define CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg) \
125 push %rbp; \
126 mov %rsp, %rbp; \
127 movq %cr0, tmpreg; \
128 testq $CR0_TS, tmpreg; \
129 jnz 1f; \
130 and $-XMM_ALIGN, %rsp; \
131 sub $[XMM_SIZE * 11], %rsp; \
132 movaps %xmm0, 160(%rsp); \
133 movaps %xmm1, 144(%rsp); \
134 movaps %xmm2, 128(%rsp); \
135 movaps %xmm3, 112(%rsp); \
136 movaps %xmm4, 96(%rsp); \
137 movaps %xmm5, 80(%rsp); \
138 movaps %xmm6, 64(%rsp); \
139 movaps %xmm7, 48(%rsp); \
140 movaps %xmm8, 32(%rsp); \
141 movaps %xmm9, 16(%rsp); \
142 movaps %xmm10, (%rsp); \
143 jmp 2f; \
144 1: \
145 PROTECTED_CLTS; \
146 2:
147
148
149 /*
150 * If CR0_TS was not set above, pop %xmm0 - %xmm10 off stack,
151 * otherwise set CR0_TS.
152 */
153 #define SET_TS_OR_POP_XMM_REGISTERS(tmpreg) \
154 testq $CR0_TS, tmpreg; \
155 jnz 1f; \
156 movaps (%rsp), %xmm10; \
157 movaps 16(%rsp), %xmm9; \
158 movaps 32(%rsp), %xmm8; \
159 movaps 48(%rsp), %xmm7; \
160 movaps 64(%rsp), %xmm6; \
161 movaps 80(%rsp), %xmm5; \
162 movaps 96(%rsp), %xmm4; \
163 movaps 112(%rsp), %xmm3; \
164 movaps 128(%rsp), %xmm2; \
165 movaps 144(%rsp), %xmm1; \
166 movaps 160(%rsp), %xmm0; \
167 jmp 2f; \
168 1: \
169 STTS(tmpreg); \
170 2: \
171 mov %rbp, %rsp; \
172 pop %rbp
173
174
175 #else
176 #define PROTECTED_CLTS
177 #define CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg)
178 #define SET_TS_OR_POP_XMM_REGISTERS(tmpreg)
179 #endif /* _KERNEL */
180
181 /*
182 * Use this mask to byte-swap a 16-byte integer with the pshufb instruction
183 */
184
185 // static uint8_t byte_swap16_mask[] = {
186 // 15, 14, 13, 12, 11, 10, 9, 8, 7, 6 ,5, 4, 3, 2, 1, 0 };
187 .text
188 .align XMM_ALIGN
189 .Lbyte_swap16_mask:
190 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
191
192
193
194 /*
195 * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
196 *
197 * Perform a carry-less multiplication (that is, use XOR instead of the
198 * multiply operator) on P1 and P2 and place the result in P3.
199 *
200 * Byte swap the input and the output.
201 *
202 * Note: x_in, y, and res all point to a block of 20-byte numbers
203 * (an array of two 64-bit integers).
204 *
205 * Note2: For kernel code, caller is responsible for ensuring
206 * kpreempt_disable() has been called. This is because %xmm registers are
207 * not saved/restored. Clear and set the CR0.TS bit on entry and exit,
208 * respectively, if TS is set on entry. Otherwise, if TS is not set,
209 * save and restore %xmm registers on the stack.
210 *
211 * Note3: Original Intel definition:
212 * void galois_hash_asm(unsigned char *hk, unsigned char *s,
213 * unsigned char *d, int length)
214 *
215 * Note4: Register/parameter mapping:
216 * Intel:
217 * Parameter 1: %rcx (copied to %xmm0) hk or x_in
218 * Parameter 2: %rdx (copied to %xmm1) s or y
219 * Parameter 3: %rdi (result) d or res
220 * OpenSolaris:
221 * Parameter 1: %rdi (copied to %xmm0) x_in
222 * Parameter 2: %rsi (copied to %xmm1) y
223 * Parameter 3: %rdx (result) res
224 */
225
226 ENTRY_NP(gcm_mul_pclmulqdq)
227 CLEAR_TS_OR_PUSH_XMM_REGISTERS(%r10)
228
229 //
230 // Copy Parameters
231 //
232 movdqu (%rdi), %xmm0 // P1
233 movdqu (%rsi), %xmm1 // P2
234
235 //
236 // Byte swap 16-byte input
237 //
238 lea .Lbyte_swap16_mask(%rip), %rax
239 movaps (%rax), %xmm10
240 pshufb %xmm10, %xmm0
241 pshufb %xmm10, %xmm1
242
243
244 //
245 // Multiply with the hash key
246 //
247 movdqu %xmm0, %xmm3
248 pclmulqdq $0, %xmm1, %xmm3 // xmm3 holds a0*b0
249
250 movdqu %xmm0, %xmm4
251 pclmulqdq $16, %xmm1, %xmm4 // xmm4 holds a0*b1
252
253 movdqu %xmm0, %xmm5
254 pclmulqdq $1, %xmm1, %xmm5 // xmm5 holds a1*b0
255 movdqu %xmm0, %xmm6
256 pclmulqdq $17, %xmm1, %xmm6 // xmm6 holds a1*b1
257
258 pxor %xmm5, %xmm4 // xmm4 holds a0*b1 + a1*b0
259
260 movdqu %xmm4, %xmm5 // move the contents of xmm4 to xmm5
261 psrldq $8, %xmm4 // shift by xmm4 64 bits to the right
262 pslldq $8, %xmm5 // shift by xmm5 64 bits to the left
263 pxor %xmm5, %xmm3
264 pxor %xmm4, %xmm6 // Register pair <xmm6:xmm3> holds the result
265 // of the carry-less multiplication of
266 // xmm0 by xmm1.
267
268 // We shift the result of the multiplication by one bit position
269 // to the left to cope for the fact that the bits are reversed.
270 movdqu %xmm3, %xmm7
271 movdqu %xmm6, %xmm8
272 pslld $1, %xmm3
273 pslld $1, %xmm6
274 psrld $31, %xmm7
275 psrld $31, %xmm8
276 movdqu %xmm7, %xmm9
277 pslldq $4, %xmm8
278 pslldq $4, %xmm7
279 psrldq $12, %xmm9
280 por %xmm7, %xmm3
281 por %xmm8, %xmm6
282 por %xmm9, %xmm6
283
284 //
285 // First phase of the reduction
286 //
287 // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
288 // independently.
289 movdqu %xmm3, %xmm7
290 movdqu %xmm3, %xmm8
291 movdqu %xmm3, %xmm9
292 pslld $31, %xmm7 // packed right shift shifting << 31
293 pslld $30, %xmm8 // packed right shift shifting << 30
294 pslld $25, %xmm9 // packed right shift shifting << 25
295 pxor %xmm8, %xmm7 // xor the shifted versions
296 pxor %xmm9, %xmm7
297 movdqu %xmm7, %xmm8
298 pslldq $12, %xmm7
299 psrldq $4, %xmm8
300 pxor %xmm7, %xmm3 // first phase of the reduction complete
301
302 //
303 // Second phase of the reduction
304 //
305 // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
306 // shift operations.
307 movdqu %xmm3, %xmm2
308 movdqu %xmm3, %xmm4 // packed left shifting >> 1
309 movdqu %xmm3, %xmm5
310 psrld $1, %xmm2
311 psrld $2, %xmm4 // packed left shifting >> 2
312 psrld $7, %xmm5 // packed left shifting >> 7
313 pxor %xmm4, %xmm2 // xor the shifted versions
314 pxor %xmm5, %xmm2
315 pxor %xmm8, %xmm2
316 pxor %xmm2, %xmm3
317 pxor %xmm3, %xmm6 // the result is in xmm6
318
319 //
320 // Byte swap 16-byte result
321 //
322 pshufb %xmm10, %xmm6 // %xmm10 has the swap mask
323
324 //
325 // Store the result
326 //
327 movdqu %xmm6, (%rdx) // P3
328
329
330 //
331 // Cleanup and Return
332 //
333 SET_TS_OR_POP_XMM_REGISTERS(%r10)
334 ret
335 SET_SIZE(gcm_mul_pclmulqdq)
336
337 #endif /* lint || __lint */