6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /* Copyright (c) 1990 Mentat Inc. */
26
27 #include <sys/types.h>
28 #include <sys/inttypes.h>
29 #include <sys/systm.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <sys/debug.h>
33 #include <sys/ddi.h>
34 #include <sys/vtrace.h>
35 #include <inet/sctp_crc32.h>
36 #include <inet/ip.h>
37
38 #include <sys/multidata.h>
39 #include <sys/multidata_impl.h>
40
41 extern unsigned int ip_ocsum(ushort_t *address, int halfword_count,
42 unsigned int sum);
43
44 /*
45 * Checksum routine for Internet Protocol family headers.
46 * This routine is very heavily used in the network
47 * code and should be modified for each CPU to be as fast as possible.
48 */
49
50 #define mp_len(mp) ((mp)->b_wptr - (mp)->b_rptr)
51
52 /*
53 * Even/Odd checks. Usually it is performed on pointers but may be
54 * used on integers as well. uintptr_t is long enough to hold both
55 * integer and pointer.
56 */
57 #define is_odd(p) (((uintptr_t)(p) & 0x1) != 0)
58 #define is_even(p) (!is_odd(p))
59
60
61 #ifdef ZC_TEST
62 /*
63 * Disable the TCP s/w cksum.
64 * XXX - This is just a hack for testing purpose. Don't use it for
65 * anything else!
66 */
67 int noswcksum = 0;
68 #endif
69 /*
70 * Note: this does not ones-complement the result since it is used
71 * when computing partial checksums.
72 * For nonSTRUIO_IP mblks, assumes mp->b_rptr+offset is 16 bit aligned.
73 * For STRUIO_IP mblks, assumes mp->b_datap->db_struiobase is 16 bit aligned.
74 *
75 * Note: for STRUIO_IP special mblks some data may have been previously
76 * checksumed, this routine will handle additional data prefixed within
77 * an mblk or b_cont (chained) mblk(s). This routine will also handle
78 * suffixed b_cont mblk(s) and data suffixed within an mblk.
79 */
80 unsigned int
81 ip_cksum(mblk_t *mp, int offset, uint_t sum)
82 {
83 ushort_t *w;
84 ssize_t mlen;
85 int pmlen;
86 mblk_t *pmp;
87 dblk_t *dp = mp->b_datap;
88 ushort_t psum = 0;
89
90 #ifdef ZC_TEST
91 if (noswcksum)
92 return (0xffff);
93 #endif
94 ASSERT(dp);
95
96 if (mp->b_cont == NULL) {
97 /*
98 * May be fast-path, only one mblk.
99 */
100 w = (ushort_t *)(mp->b_rptr + offset);
101 if (dp->db_struioflag & STRUIO_IP) {
102 /*
103 * Checksum any data not already done by
104 * the caller and add in any partial checksum.
105 */
106 if ((offset > dp->db_cksumstart) ||
107 mp->b_wptr != (uchar_t *)(mp->b_rptr +
108 dp->db_cksumend)) {
109 /*
110 * Mblk data pointers aren't inclusive
111 * of uio data, so disregard checksum.
112 *
113 * not using all of data in dblk make sure
114 * not use to use the precalculated checksum
115 * in this case.
116 */
117 dp->db_struioflag &= ~STRUIO_IP;
118 goto norm;
119 }
120 ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend));
121 psum = *(ushort_t *)dp->db_struioun.data;
122 if ((mlen = dp->db_cksumstart - offset) < 0)
123 mlen = 0;
124 if (is_odd(mlen))
125 goto slow;
126 if (mlen && dp->db_cksumstart != dp->db_cksumstuff &&
127 dp->db_cksumend != dp->db_cksumstuff) {
128 /*
129 * There is prefix data to do and some uio
130 * data has already been checksumed and there
131 * is more uio data to do, so do the prefix
132 * data first, then do the remainder of the
133 * uio data.
134 */
135 sum = ip_ocsum(w, mlen >> 1, sum);
136 w = (ushort_t *)(mp->b_rptr +
137 dp->db_cksumstuff);
138 if (is_odd(w)) {
139 pmp = mp;
140 goto slow1;
141 }
142 mlen = dp->db_cksumend - dp->db_cksumstuff;
143 } else if (dp->db_cksumend != dp->db_cksumstuff) {
144 /*
145 * There may be uio data to do, if there is
146 * prefix data to do then add in all of the
147 * uio data (if any) to do, else just do any
148 * uio data.
149 */
150 if (mlen)
151 mlen += dp->db_cksumend
152 - dp->db_cksumstuff;
153 else {
154 w = (ushort_t *)(mp->b_rptr +
155 dp->db_cksumstuff);
156 if (is_odd(w))
157 goto slow;
158 mlen = dp->db_cksumend
159 - dp->db_cksumstuff;
160 }
161 } else if (mlen == 0)
162 return (psum);
163
164 if (is_odd(mlen))
165 goto slow;
166 sum += psum;
167 } else {
168 /*
169 * Checksum all data not already done by the caller.
170 */
171 norm:
172 mlen = mp->b_wptr - (uchar_t *)w;
173 if (is_odd(mlen))
174 goto slow;
175 }
176 ASSERT(is_even(w));
177 ASSERT(is_even(mlen));
178 return (ip_ocsum(w, mlen >> 1, sum));
179 }
180 if (dp->db_struioflag & STRUIO_IP)
181 psum = *(ushort_t *)dp->db_struioun.data;
182 slow:
183 pmp = 0;
184 slow1:
185 mlen = 0;
186 pmlen = 0;
187 for (; ; ) {
188 /*
189 * Each trip around loop adds in word(s) from one mbuf segment
190 * (except for when pmp == mp, then its two partial trips).
191 */
192 w = (ushort_t *)(mp->b_rptr + offset);
193 if (pmp) {
194 /*
195 * This is the second trip around for this mblk.
196 */
197 pmp = 0;
198 mlen = 0;
199 goto douio;
200 } else if (dp->db_struioflag & STRUIO_IP) {
201 /*
202 * Checksum any data not already done by the
203 * caller and add in any partial checksum.
204 */
205 if ((offset > dp->db_cksumstart) ||
206 mp->b_wptr != (uchar_t *)(mp->b_rptr +
207 dp->db_cksumend)) {
208 /*
209 * Mblk data pointers aren't inclusive
210 * of uio data, so disregard checksum.
211 *
212 * not using all of data in dblk make sure
213 * not use to use the precalculated checksum
214 * in this case.
215 */
216 dp->db_struioflag &= ~STRUIO_IP;
217 goto snorm;
218 }
219 ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend));
220 if ((mlen = dp->db_cksumstart - offset) < 0)
221 mlen = 0;
222 if (mlen && dp->db_cksumstart != dp->db_cksumstuff) {
223 /*
224 * There is prefix data too do and some
225 * uio data has already been checksumed,
226 * so do the prefix data only this trip.
227 */
228 pmp = mp;
229 } else {
230 /*
231 * Add in any partial cksum (if any) and
232 * do the remainder of the uio data.
233 */
234 int odd;
235 douio:
236 odd = is_odd(dp->db_cksumstuff -
237 dp->db_cksumstart);
238 if (pmlen == -1) {
239 /*
240 * Previous mlen was odd, so swap
241 * the partial checksum bytes.
242 */
243 sum += ((psum << 8) & 0xffff)
244 | (psum >> 8);
245 if (odd)
246 pmlen = 0;
247 } else {
248 sum += psum;
249 if (odd)
250 pmlen = -1;
251 }
252 if (dp->db_cksumend != dp->db_cksumstuff) {
253 /*
254 * If prefix data to do and then all
255 * the uio data nees to be checksumed,
256 * else just do any uio data.
257 */
258 if (mlen)
259 mlen += dp->db_cksumend
260 - dp->db_cksumstuff;
261 else {
262 w = (ushort_t *)(mp->b_rptr +
263 dp->db_cksumstuff);
264 mlen = dp->db_cksumend -
265 dp->db_cksumstuff;
266 }
267 }
268 }
269 } else {
270 /*
271 * Checksum all of the mblk data.
272 */
273 snorm:
274 mlen = mp->b_wptr - (uchar_t *)w;
275 }
276
277 mp = mp->b_cont;
278 if (mlen > 0 && pmlen == -1) {
279 /*
280 * There is a byte left from the last
281 * segment; add it into the checksum.
282 * Don't have to worry about a carry-
283 * out here because we make sure that
284 * high part of (32 bit) sum is small
285 * below.
286 */
287 #ifdef _LITTLE_ENDIAN
288 sum += *(uchar_t *)w << 8;
289 #else
290 sum += *(uchar_t *)w;
291 #endif
292 w = (ushort_t *)((char *)w + 1);
293 mlen--;
294 pmlen = 0;
295 }
296 if (mlen > 0) {
297 if (is_even(w)) {
298 sum = ip_ocsum(w, mlen>>1, sum);
299 w += mlen>>1;
300 /*
301 * If we had an odd number of bytes,
302 * then the last byte goes in the high
303 * part of the sum, and we take the
304 * first byte to the low part of the sum
305 * the next time around the loop.
306 */
307 if (is_odd(mlen)) {
308 #ifdef _LITTLE_ENDIAN
309 sum += *(uchar_t *)w;
310 #else
311 sum += *(uchar_t *)w << 8;
312 #endif
313 pmlen = -1;
314 }
315 } else {
316 ushort_t swsum;
317 #ifdef _LITTLE_ENDIAN
318 sum += *(uchar_t *)w;
319 #else
320 sum += *(uchar_t *)w << 8;
321 #endif
322 mlen--;
323 w = (ushort_t *)(1 + (uintptr_t)w);
324
325 /* Do a separate checksum and copy operation */
326 swsum = ip_ocsum(w, mlen>>1, 0);
327 sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
328 w += mlen>>1;
329 /*
330 * If we had an even number of bytes,
331 * then the last byte goes in the low
332 * part of the sum. Otherwise we had an
333 * odd number of bytes and we take the first
334 * byte to the low part of the sum the
335 * next time around the loop.
336 */
337 if (is_odd(mlen)) {
338 #ifdef _LITTLE_ENDIAN
339 sum += *(uchar_t *)w << 8;
340 #else
341 sum += *(uchar_t *)w;
342 #endif
343 }
344 else
345 pmlen = -1;
346 }
347 }
348 /*
349 * Locate the next block with some data.
350 * If there is a word split across a boundary we
351 * will wrap to the top with mlen == -1 and
352 * then add it in shifted appropriately.
353 */
354 offset = 0;
355 if (! pmp) {
356 for (; ; ) {
357 if (mp == 0) {
358 goto done;
359 }
360 if (mp_len(mp))
361 break;
362 mp = mp->b_cont;
363 }
364 dp = mp->b_datap;
365 if (dp->db_struioflag & STRUIO_IP)
366 psum = *(ushort_t *)dp->db_struioun.data;
367 } else
368 mp = pmp;
369 }
370 done:
371 /*
372 * Add together high and low parts of sum
373 * and carry to get cksum.
374 * Have to be careful to not drop the last
375 * carry here.
376 */
377 sum = (sum & 0xFFFF) + (sum >> 16);
378 sum = (sum & 0xFFFF) + (sum >> 16);
379 TRACE_3(TR_FAC_IP, TR_IP_CKSUM_END,
380 "ip_cksum_end:(%S) type %d (%X)", "ip_cksum", 1, sum);
381 return (sum);
382 }
383
384 uint32_t
385 sctp_cksum(mblk_t *mp, int offset)
386 {
387 uint32_t crc32;
388 uchar_t *p = NULL;
389
390 crc32 = 0xFFFFFFFF;
391 p = mp->b_rptr + offset;
392 crc32 = sctp_crc32(crc32, p, mp->b_wptr - p);
393 for (mp = mp->b_cont; mp != NULL; mp = mp->b_cont) {
394 crc32 = sctp_crc32(crc32, mp->b_rptr, MBLKL(mp));
395 }
396
397 /* Complement the result */
398 crc32 = ~crc32;
399
400 return (crc32);
401 }
451 * There is a byte left from the last
452 * segment; add it into the checksum.
453 * Don't have to worry about a carry-
454 * out here because we make sure that
455 * high part of (32 bit) sum is small
456 * below.
457 */
458 #ifdef _LITTLE_ENDIAN
459 sum += *(uchar_t *)w << 8;
460 #else
461 sum += *(uchar_t *)w;
462 #endif
463 w = (ushort_t *)((char *)w + 1);
464 mlen--;
465 byteleft = B_FALSE;
466 }
467
468 if (mlen == 0)
469 continue;
470
471 if (is_even(w)) {
472 sum = ip_ocsum(w, mlen >> 1, sum);
473 w += mlen >> 1;
474 /*
475 * If we had an odd number of bytes,
476 * then the last byte goes in the high
477 * part of the sum, and we take the
478 * first byte to the low part of the sum
479 * the next time around the loop.
480 */
481 if (is_odd(mlen)) {
482 #ifdef _LITTLE_ENDIAN
483 sum += *(uchar_t *)w;
484 #else
485 sum += *(uchar_t *)w << 8;
486 #endif
487 byteleft = B_TRUE;
488 }
489 } else {
490 ushort_t swsum;
491 #ifdef _LITTLE_ENDIAN
492 sum += *(uchar_t *)w;
493 #else
494 sum += *(uchar_t *)w << 8;
495 #endif
496 mlen--;
497 w = (ushort_t *)(1 + (uintptr_t)w);
498
499 /* Do a separate checksum and copy operation */
500 swsum = ip_ocsum(w, mlen >> 1, 0);
501 sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
502 w += mlen >> 1;
503 /*
504 * If we had an even number of bytes,
505 * then the last byte goes in the low
506 * part of the sum. Otherwise we had an
507 * odd number of bytes and we take the first
508 * byte to the low part of the sum the
509 * next time around the loop.
510 */
511 if (is_odd(mlen)) {
512 #ifdef _LITTLE_ENDIAN
513 sum += *(uchar_t *)w << 8;
514 #else
515 sum += *(uchar_t *)w;
516 #endif
517 } else {
518 byteleft = B_TRUE;
519 }
520 }
521 }
522
523 /*
524 * Add together high and low parts of sum and carry to get cksum.
525 * Have to be careful to not drop the last carry here.
526 */
527 sum = (sum & 0xffff) + (sum >> 16);
528 sum = (sum & 0xffff) + (sum >> 16);
529
530 return (sum);
531 }
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /* Copyright (c) 1990 Mentat Inc. */
26 /*
27 * Copyright 2019 Joyent, Inc.
28 */
29
30 #include <sys/types.h>
31 #include <sys/inttypes.h>
32 #include <sys/systm.h>
33 #include <sys/stream.h>
34 #include <sys/strsun.h>
35 #include <sys/strsubr.h>
36 #include <sys/debug.h>
37 #include <sys/ddi.h>
38 #include <sys/vtrace.h>
39 #include <inet/sctp_crc32.h>
40 #include <inet/ip.h>
41
42 #include <sys/multidata.h>
43 #include <sys/multidata_impl.h>
44
45 extern unsigned int ip_ocsum(ushort_t *address, int halfword_count,
46 unsigned int sum);
47
48 /*
49 * Checksum routine for Internet Protocol family headers.
50 * This routine is very heavily used in the network
51 * code and should be modified for each CPU to be as fast as possible.
52 */
53
54 /*
55 * Even/Odd checks. Usually it is performed on pointers but may be
56 * used on integers as well. uintptr_t is long enough to hold both
57 * integer and pointer.
58 */
59 #define IS_ODD(p) (((uintptr_t)(p) & 0x1) != 0)
60 #define IS_EVEN(p) (((uintptr_t)(p) & 0x1) == 0)
61
62 /*
63 */
64 #define HAS_UIOSUM(mp) ((mp)->b_datap->db_struioflag & STRUIO_IP)
65
66 #ifdef _LITTLE_ENDIAN
67 #define FRAG(ptr) (*(ptr))
68 #else
69 #define FRAG(ptr) (*(ptr) << 8)
70 #endif
71
72 /*
73 * Give the compiler a hint to help optimize the code layout
74 */
75 #define UNLIKELY(exp) __builtin_expect((exp), 0)
76
77 #define FOLD(val) (((val) & 0xFFFF) + ((val) >> 16))
78
79 /*
80 * Note: this does not ones-complement the result since it is used
81 * when computing partial checksums. It assumes mp->b_rptr + offset is
82 * 16 bit aligned and a valid offset in mp.
83 */
84 unsigned int
85 ip_cksum(mblk_t *mp, int offset, uint_t initial_sum)
86 {
87 const uint_t sum_mask[2] = { 0, UINT_MAX };
88 uint64_t sum = initial_sum;
89 uint64_t total_len = 0;
90 uchar_t *w;
91 size_t mlen = MBLKL(mp);
92 uint_t msum, mask;
93
94 ASSERT3S(offset, >=, 0);
95
96 VERIFY(!HAS_UIOSUM(mp));
97 while (UNLIKELY(offset > mlen)) {
98 ASSERT3P(mp->b_cont, !=, NULL);
99 mp = mp->b_cont;
100 VERIFY(!HAS_UIOSUM(mp));
101 offset -= mlen;
102 mlen = MBLKL(mp);
103 }
104
105 /*
106 * Make sure we start with a folded sum. Since the initial sum
107 * is 32 bits, folding twice will always produce a sum <= 0xFFFF
108 */
109 sum = FOLD(sum);
110 sum = FOLD(sum);
111 ASSERT3U(sum, <=, 0xFFFF);
112
113 while (mp != NULL) {
114 w = mp->b_rptr + offset;
115 mlen = mp->b_wptr - w;
116 offset = 0;
117
118 ASSERT3P(w, <=, mp->b_wptr);
119 VERIFY(!HAS_UIOSUM(mp));
120
121 if (UNLIKELY(mlen == 0)) {
122 mp = mp->b_cont;
123 continue;
124 }
125
126 /*
127 * ip_ocsum() currently requires a 16-bit aligned address.
128 * For unaligned buffers, we first sum the odd byte (and
129 * fold if necessary) before calling ip_ocsum(). ip_ocsum()
130 * also takes its length in units of 16-bit words. If
131 * we have an odd length, we must also manually add it after
132 * computing the main sum (and again fold if necessary).
133 *
134 * Since ip_ocsum() _should_ be a private per-platform
135 * optimized ip cksum implementation (with ip_cksum() being
136 * the less-private wrapper around it), a nice future
137 * optimization could be to modify ip_ocsum() for each
138 * platform to take a 64-bit sum. This would allow us to
139 * only have to fold exactly once before we return --
140 * the amount of data we'd need to checksum to overflow 64
141 * bits far exceeds the possible size of any mblk chain we
142 * could ever have.
143 */
144 if (UNLIKELY(IS_ODD(w))) {
145 sum += FRAG(w);
146 w++;
147
148 --mlen;
149 total_len++;
150
151 if (UNLIKELY(mlen == 0)) {
152 mp = mp->b_cont;
153 continue;
154 }
155 }
156
157 /*
158 * ip_ocsum() takes the length as the number of half words
159 * (i.e. uint16_ts). It returns a result that is already
160 * folded (<= 0xFFFF).
161 */
162 msum = ip_ocsum((ushort_t *)w, mlen / 2, 0);
163 ASSERT3U(msum, <=, 0xFFFF);
164
165 /*
166 * We mask the last byte based on the length of data.
167 * If the length is odd, we AND with UINT_MAX otherwise
168 * we AND with 0 (resulting in 0) and add the result to
169 * the mblk_t sum. This effectively gives us:
170 *
171 * if (IS_ODD(mlen))
172 * msum += FRAG(w + mlen - 1);
173 * else
174 * msum += 0;
175 *
176 * Without incurring a branch.
177 */
178 mask = sum_mask[IS_ODD(mlen)];
179 msum += FRAG(w + mlen - 1) & mask;
180
181 /*
182 * If the data we are checksumming has been split
183 * between two mblk_ts along a non-16 bit boundary, that is
184 * we have something like:
185 * mblk_t 1: aa bb cc
186 * mblk_t 2: dd ee ff
187 * the result must be the same as if we checksummed a
188 * single mblk_t with 'aa bb cc dd ee ff'. As can be seen
189 * from the example, this situation causes the grouping of
190 * the data in the second mblk_t to be offset by a byte.
191 * The fix is to byteswap the mblk_t sum before adding it
192 * to the final sum. Again, we AND the mblk_t sum with a mask
193 * so that either the non-swapped or byteswapped sum is zeroed
194 * out and the other one is preserved (depending on the
195 * total bytes checksummed so far) and added to the sum.
196 *
197 * Effectively,
198 *
199 * if (IS_ODD(total_len))
200 * sum += BSWAP_32(msum);
201 * else
202 * sum += msum;
203 */
204 mask = sum_mask[IS_ODD(total_len)];
205 sum += BSWAP_32(msum) & mask;
206 sum += msum & ~mask;
207
208 total_len += mlen;
209 mp = mp->b_cont;
210 }
211
212 /*
213 * To avoid unnecessary folding, we store the cumulative sum in
214 * a uint64_t. This means we can always checksum up to 2^56 bytes
215 * (2^(64-8)) without danger of overflowing. Since 2^56 is well past
216 * the petabyte range, and is far beyond the amount of data that
217 * could every be stored in a single mblk_t chain (for the forseeable
218 * future), this serves more as a sanity check than anything else.
219 */
220 VERIFY3U(total_len, <=, (uint64_t)1 << 56);
221
222 /*
223 * For a 64-bit sum, we have to fold at most 4 times to
224 * produce a sum <= 0xFFFF.
225 */
226 sum = FOLD(sum);
227 sum = FOLD(sum);
228 sum = FOLD(sum);
229 sum = FOLD(sum);
230
231 TRACE_3(TR_FAC_IP, TR_IP_CKSUM_END,
232 "ip_cksum_end:(%S) type %d (%X)", "ip_cksum", 1, sum);
233 return ((unsigned int)sum);
234 }
235
236 uint32_t
237 sctp_cksum(mblk_t *mp, int offset)
238 {
239 uint32_t crc32;
240 uchar_t *p = NULL;
241
242 crc32 = 0xFFFFFFFF;
243 p = mp->b_rptr + offset;
244 crc32 = sctp_crc32(crc32, p, mp->b_wptr - p);
245 for (mp = mp->b_cont; mp != NULL; mp = mp->b_cont) {
246 crc32 = sctp_crc32(crc32, mp->b_rptr, MBLKL(mp));
247 }
248
249 /* Complement the result */
250 crc32 = ~crc32;
251
252 return (crc32);
253 }
303 * There is a byte left from the last
304 * segment; add it into the checksum.
305 * Don't have to worry about a carry-
306 * out here because we make sure that
307 * high part of (32 bit) sum is small
308 * below.
309 */
310 #ifdef _LITTLE_ENDIAN
311 sum += *(uchar_t *)w << 8;
312 #else
313 sum += *(uchar_t *)w;
314 #endif
315 w = (ushort_t *)((char *)w + 1);
316 mlen--;
317 byteleft = B_FALSE;
318 }
319
320 if (mlen == 0)
321 continue;
322
323 if (IS_EVEN(w)) {
324 sum = ip_ocsum(w, mlen >> 1, sum);
325 w += mlen >> 1;
326 /*
327 * If we had an odd number of bytes,
328 * then the last byte goes in the high
329 * part of the sum, and we take the
330 * first byte to the low part of the sum
331 * the next time around the loop.
332 */
333 if (IS_ODD(mlen)) {
334 #ifdef _LITTLE_ENDIAN
335 sum += *(uchar_t *)w;
336 #else
337 sum += *(uchar_t *)w << 8;
338 #endif
339 byteleft = B_TRUE;
340 }
341 } else {
342 ushort_t swsum;
343 #ifdef _LITTLE_ENDIAN
344 sum += *(uchar_t *)w;
345 #else
346 sum += *(uchar_t *)w << 8;
347 #endif
348 mlen--;
349 w = (ushort_t *)(1 + (uintptr_t)w);
350
351 /* Do a separate checksum and copy operation */
352 swsum = ip_ocsum(w, mlen >> 1, 0);
353 sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
354 w += mlen >> 1;
355 /*
356 * If we had an even number of bytes,
357 * then the last byte goes in the low
358 * part of the sum. Otherwise we had an
359 * odd number of bytes and we take the first
360 * byte to the low part of the sum the
361 * next time around the loop.
362 */
363 if (IS_ODD(mlen)) {
364 #ifdef _LITTLE_ENDIAN
365 sum += *(uchar_t *)w << 8;
366 #else
367 sum += *(uchar_t *)w;
368 #endif
369 } else {
370 byteleft = B_TRUE;
371 }
372 }
373 }
374
375 /*
376 * Add together high and low parts of sum and carry to get cksum.
377 * Have to be careful to not drop the last carry here.
378 */
379 sum = (sum & 0xffff) + (sum >> 16);
380 sum = (sum & 0xffff) + (sum >> 16);
381
382 return (sum);
383 }
|