1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Fletcher Checksums
28 * ------------------
29 *
30 * ZFS's 2nd and 4th order Fletcher checksums are defined by the following
31 * recurrence relations:
32 *
33 * a = a + f
34 * i i-1 i-1
35 *
36 * b = b + a
37 * i i-1 i
38 *
39 * c = c + b (fletcher-4 only)
40 * i i-1 i
41 *
42 * d = d + c (fletcher-4 only)
43 * i i-1 i
44 *
45 * Where
46 * a_0 = b_0 = c_0 = d_0 = 0
47 * and
48 * f_0 .. f_(n-1) are the input data.
49 *
50 * Using standard techniques, these translate into the following series:
51 *
52 * __n_ __n_
53 * \ | \ |
54 * a = > f b = > i * f
55 * n /___| n - i n /___| n - i
56 * i = 1 i = 1
57 *
58 *
59 * __n_ __n_
60 * \ | i*(i+1) \ | i*(i+1)*(i+2)
61 * c = > ------- f d = > ------------- f
62 * n /___| 2 n - i n /___| 6 n - i
63 * i = 1 i = 1
64 *
65 * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
66 * Since the additions are done mod (2^64), errors in the high bits may not
67 * be noticed. For this reason, fletcher-2 is deprecated.
68 *
69 * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
70 * A conservative estimate of how big the buffer can get before we overflow
71 * can be estimated using f_i = 0xffffffff for all i:
72 *
73 * % bc
74 * f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4
75 * 2264
76 * quit
77 * %
78 *
79 * So blocks of up to 2k will not overflow. Our largest block size is
80 * 128k, which has 32k 4-byte words, so we can compute the largest possible
81 * accumulators, then divide by 2^64 to figure the max amount of overflow:
82 *
83 * % bc
84 * a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
85 * a/2^64;b/2^64;c/2^64;d/2^64
86 * 0
87 * 0
88 * 1365
89 * 11186858
90 * quit
91 * %
92 *
93 * So a and b cannot overflow. To make sure each bit of input has some
94 * effect on the contents of c and d, we can look at what the factors of
95 * the coefficients in the equations for c_n and d_n are. The number of 2s
96 * in the factors determines the lowest set bit in the multiplier. Running
97 * through the cases for n*(n+1)/2 reveals that the highest power of 2 is
98 * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15. So while some data may overflow
99 * the 64-bit accumulators, every bit of every f_i effects every accumulator,
100 * even for 128k blocks.
101 *
102 * If we wanted to make a stronger version of fletcher4 (fletcher4c?),
103 * we could do our calculations mod (2^32 - 1) by adding in the carries
104 * periodically, and store the number of carries in the top 32-bits.
105 *
106 * --------------------
107 * Checksum Performance
108 * --------------------
109 *
110 * There are two interesting components to checksum performance: cached and
111 * uncached performance. With cached data, fletcher-2 is about four times
112 * faster than fletcher-4. With uncached data, the performance difference is
113 * negligible, since the cost of a cache fill dominates the processing time.
114 * Even though fletcher-4 is slower than fletcher-2, it is still a pretty
115 * efficient pass over the data.
116 *
117 * In normal operation, the data which is being checksummed is in a buffer
118 * which has been filled either by:
119 *
120 * 1. a compression step, which will be mostly cached, or
121 * 2. a bcopy() or copyin(), which will be uncached (because the
122 * copy is cache-bypassing).
123 *
124 * For both cached and uncached data, both fletcher checksums are much faster
125 * than sha-256, and slower than 'off', which doesn't touch the data at all.
126 */
127
128 /*
129 * TODO: vectorize these functions
130 * All of these functions are written so that each iteration of the loop
131 * depends on the value of the previous iteration. Also, in the fletcher_4
132 * functions, each statement of the loop body depends on the previous
133 * statement. These dependencies prevent the compiler from vectorizing the
134 * code to take advantage of SIMD extensions (unless GCC is far smarter than I
135 * think). It would be easy to rewrite the loops to be amenable to
136 * autovectorization.
137 */
138
139 #include <sys/types.h>
140 #include <sys/sysmacros.h>
141 #include <sys/byteorder.h>
142 #include <sys/zio.h>
143 #include <sys/spa.h>
144
145 void
146 fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
147 {
148 const uint64_t *ip = buf;
149 const uint64_t *ipend = ip + (size / sizeof (uint64_t));
150 uint64_t a0, b0, a1, b1;
151
152 for (a0 = b0 = a1 = b1 = 0; ip < ipend; ip += 2) {
153 a0 += ip[0];
154 a1 += ip[1];
155 b0 += a0;
156 b1 += a1;
157 }
158
159 ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
160 }
161
162 void
163 fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
164 {
165 const uint64_t *ip = buf;
166 const uint64_t *ipend = ip + (size / sizeof (uint64_t));
167 uint64_t a0, b0, a1, b1;
168
169 for (a0 = b0 = a1 = b1 = 0; ip < ipend; ip += 2) {
170 a0 += BSWAP_64(ip[0]);
171 a1 += BSWAP_64(ip[1]);
172 b0 += a0;
173 b1 += a1;
174 }
175
176 ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
177 }
178
179 void
180 fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
181 {
182 const uint32_t *ip = buf;
183 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
184 uint64_t a, b, c, d;
185
186 for (a = b = c = d = 0; ip < ipend; ip++) {
187 a += ip[0];
188 b += a;
189 c += b;
190 d += c;
191 }
192
193 ZIO_SET_CHECKSUM(zcp, a, b, c, d);
194 }
195
196 void
197 fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
198 {
199 const uint32_t *ip = buf;
200 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
201 uint64_t a, b, c, d;
202
203 for (a = b = c = d = 0; ip < ipend; ip++) {
204 a += BSWAP_32(ip[0]);
205 b += a;
206 c += b;
207 d += c;
208 }
209
210 ZIO_SET_CHECKSUM(zcp, a, b, c, d);
211 }
212
213 void
214 fletcher_4_incremental_native(const void *buf, uint64_t size,
215 zio_cksum_t *zcp)
216 {
217 const uint32_t *ip = buf;
218 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
219 uint64_t a, b, c, d;
220
221 a = zcp->zc_word[0];
222 b = zcp->zc_word[1];
223 c = zcp->zc_word[2];
224 d = zcp->zc_word[3];
225
226 for (; ip < ipend; ip++) {
227 a += ip[0];
228 b += a;
229 c += b;
230 d += c;
231 }
232
233 ZIO_SET_CHECKSUM(zcp, a, b, c, d);
234 }
235
236 void
237 fletcher_4_incremental_byteswap(const void *buf, uint64_t size,
238 zio_cksum_t *zcp)
239 {
240 const uint32_t *ip = buf;
241 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
242 uint64_t a, b, c, d;
243
244 a = zcp->zc_word[0];
245 b = zcp->zc_word[1];
246 c = zcp->zc_word[2];
247 d = zcp->zc_word[3];
248
249 for (; ip < ipend; ip++) {
250 a += BSWAP_32(ip[0]);
251 b += a;
252 c += b;
253 d += c;
254 }
255
256 ZIO_SET_CHECKSUM(zcp, a, b, c, d);
257 }