1 /*
2 * inffast.S is a hand tuned assembler version of:
3 *
4 * inffast.c -- fast decoding
5 * Copyright (C) 1995-2003 Mark Adler
6 * For conditions of distribution and use, see copyright notice in zlib.h
7 *
8 * Copyright (C) 2003 Chris Anderson <christop@charm.net>
9 * Please use the copyright conditions above.
10 *
11 * This version (Jan-23-2003) of inflate_fast was coded and tested under
12 * GNU/Linux on a pentium 3, using the gcc-3.2 compiler distribution. On that
13 * machine, I found that gzip style archives decompressed about 20% faster than
14 * the gcc-3.2 -O3 -fomit-frame-pointer compiled version. Your results will
15 * depend on how large of a buffer is used for z_stream.next_in & next_out
16 * (8K-32K worked best for my 256K cpu cache) and how much overhead there is in
17 * stream processing I/O and crc32/addler32. In my case, this routine used
18 * 70% of the cpu time and crc32 used 20%.
19 *
20 * I am confident that this version will work in the general case, but I have
21 * not tested a wide variety of datasets or a wide variety of platforms.
22 *
23 * Jan-24-2003 -- Added -DUSE_MMX define for slightly faster inflating.
24 * It should be a runtime flag instead of compile time flag...
25 *
26 * Jan-26-2003 -- Added runtime check for MMX support with cpuid instruction.
27 * With -DUSE_MMX, only MMX code is compiled. With -DNO_MMX, only non-MMX code
28 * is compiled. Without either option, runtime detection is enabled. Runtime
29 * detection should work on all modern cpus and the recomended algorithm (flip
30 * ID bit on eflags and then use the cpuid instruction) is used in many
31 * multimedia applications. Tested under win2k with gcc-2.95 and gas-2.12
32 * distributed with cygwin3. Compiling with gcc-2.95 -c inffast.S -o
33 * inffast.obj generates a COFF object which can then be linked with MSVC++
34 * compiled code. Tested under FreeBSD 4.7 with gcc-2.95.
35 *
36 * Jan-28-2003 -- Tested Athlon XP... MMX mode is slower than no MMX (and
37 * slower than compiler generated code). Adjusted cpuid check to use the MMX
38 * code only for Pentiums < P4 until I have more data on the P4. Speed
39 * improvment is only about 15% on the Athlon when compared with code generated
40 * with MSVC++. Not sure yet, but I think the P4 will also be slower using the
41 * MMX mode because many of it's x86 ALU instructions execute in .5 cycles and
42 * have less latency than MMX ops. Added code to buffer the last 11 bytes of
43 * the input stream since the MMX code grabs bits in chunks of 32, which
44 * differs from the inffast.c algorithm. I don't think there would have been
45 * read overruns where a page boundary was crossed (a segfault), but there
46 * could have been overruns when next_in ends on unaligned memory (unintialized
47 * memory read).
48 *
49 * Mar-13-2003 -- P4 MMX is slightly slower than P4 NO_MMX. I created a C
50 * version of the non-MMX code so that it doesn't depend on zstrm and zstate
51 * structure offsets which are hard coded in this file. This was last tested
52 * with zlib-1.2.0 which is currently in beta testing, newer versions of this
53 * and inffas86.c can be found at http://www.eetbeetee.com/zlib/ and
54 * http://www.charm.net/~christop/zlib/
55 */
56
57
58 /*
59 * if you have underscore linking problems (_inflate_fast undefined), try
60 * using -DGAS_COFF
61 */
62 #if ! defined( GAS_COFF ) && ! defined( GAS_ELF )
63
64 #if defined( WIN32 ) || defined( __CYGWIN__ )
65 #define GAS_COFF /* windows object format */
66 #else
67 #define GAS_ELF
68 #endif
69
70 #endif /* ! GAS_COFF && ! GAS_ELF */
71
72
73 #if defined( GAS_COFF )
74
75 /* coff externals have underscores */
76 #define inflate_fast _inflate_fast
77 #define inflate_fast_use_mmx _inflate_fast_use_mmx
78
79 #endif /* GAS_COFF */
80
81
82 .file "inffast.S"
83
84 .globl inflate_fast
85
86 .text
87 .align 4,0
88 .L_invalid_literal_length_code_msg:
89 .string "invalid literal/length code"
90
91 .align 4,0
92 .L_invalid_distance_code_msg:
93 .string "invalid distance code"
94
95 .align 4,0
96 .L_invalid_distance_too_far_msg:
97 .string "invalid distance too far back"
98
99 #if ! defined( NO_MMX )
100 .align 4,0
101 .L_mask: /* mask[N] = ( 1 << N ) - 1 */
102 .long 0
103 .long 1
104 .long 3
105 .long 7
106 .long 15
107 .long 31
108 .long 63
109 .long 127
110 .long 255
111 .long 511
112 .long 1023
113 .long 2047
114 .long 4095
115 .long 8191
116 .long 16383
117 .long 32767
118 .long 65535
119 .long 131071
120 .long 262143
121 .long 524287
122 .long 1048575
123 .long 2097151
124 .long 4194303
125 .long 8388607
126 .long 16777215
127 .long 33554431
128 .long 67108863
129 .long 134217727
130 .long 268435455
131 .long 536870911
132 .long 1073741823
133 .long 2147483647
134 .long 4294967295
135 #endif /* NO_MMX */
136
137 .text
138
139 /*
140 * struct z_stream offsets, in zlib.h
141 */
142 #define next_in_strm 0 /* strm->next_in */
143 #define avail_in_strm 4 /* strm->avail_in */
144 #define next_out_strm 12 /* strm->next_out */
145 #define avail_out_strm 16 /* strm->avail_out */
146 #define msg_strm 24 /* strm->msg */
147 #define state_strm 28 /* strm->state */
148
149 /*
150 * struct inflate_state offsets, in inflate.h
151 */
152 #define mode_state 0 /* state->mode */
153 #define wsize_state 32 /* state->wsize */
154 #define write_state 40 /* state->write */
155 #define window_state 44 /* state->window */
156 #define hold_state 48 /* state->hold */
157 #define bits_state 52 /* state->bits */
158 #define lencode_state 68 /* state->lencode */
159 #define distcode_state 72 /* state->distcode */
160 #define lenbits_state 76 /* state->lenbits */
161 #define distbits_state 80 /* state->distbits */
162
163 /*
164 * inflate_fast's activation record
165 */
166 #define local_var_size 64 /* how much local space for vars */
167 #define strm_sp 88 /* first arg: z_stream * (local_var_size + 24) */
168 #define start_sp 92 /* second arg: unsigned int (local_var_size + 28) */
169
170 /*
171 * offsets for local vars on stack
172 */
173 #define out 60 /* unsigned char* */
174 #define window 56 /* unsigned char* */
175 #define wsize 52 /* unsigned int */
176 #define write 48 /* unsigned int */
177 #define in 44 /* unsigned char* */
178 #define beg 40 /* unsigned char* */
179 #define buf 28 /* char[ 12 ] */
180 #define len 24 /* unsigned int */
181 #define last 20 /* unsigned char* */
182 #define end 16 /* unsigned char* */
183 #define dcode 12 /* code* */
184 #define lcode 8 /* code* */
185 #define dmask 4 /* unsigned int */
186 #define lmask 0 /* unsigned int */
187
188 /*
189 * typedef enum inflate_mode consts, in inflate.h
190 */
191 #define INFLATE_MODE_TYPE 11 /* state->mode flags enum-ed in inflate.h */
192 #define INFLATE_MODE_BAD 26
193
194
195 #if ! defined( USE_MMX ) && ! defined( NO_MMX )
196
197 #define RUN_TIME_MMX
198
199 #define CHECK_MMX 1
200 #define DO_USE_MMX 2
201 #define DONT_USE_MMX 3
202
203 .globl inflate_fast_use_mmx
204
205 .data
206
207 .align 4,0
208 inflate_fast_use_mmx: /* integer flag for run time control 1=check,2=mmx,3=no */
209 .long CHECK_MMX
210
211 #if defined( GAS_ELF )
212 /* elf info */
213 .type inflate_fast_use_mmx,@object
214 .size inflate_fast_use_mmx,4
215 #endif
216
217 #endif /* RUN_TIME_MMX */
218
219 #if defined( GAS_COFF )
220 /* coff info: scl 2 = extern, type 32 = function */
221 .def inflate_fast; .scl 2; .type 32; .endef
222 #endif
223
224 .text
225
226 .align 32,0x90
227 inflate_fast:
228 pushl %edi
229 pushl %esi
230 pushl %ebp
231 pushl %ebx
232 pushf /* save eflags (strm_sp, state_sp assumes this is 32 bits) */
233 subl $local_var_size, %esp
234 cld
235
236 #define strm_r %esi
237 #define state_r %edi
238
239 movl strm_sp(%esp), strm_r
240 movl state_strm(strm_r), state_r
241
242 /* in = strm->next_in;
243 * out = strm->next_out;
244 * last = in + strm->avail_in - 11;
245 * beg = out - (start - strm->avail_out);
246 * end = out + (strm->avail_out - 257);
247 */
248 movl avail_in_strm(strm_r), %edx
249 movl next_in_strm(strm_r), %eax
250
251 addl %eax, %edx /* avail_in += next_in */
252 subl $11, %edx /* avail_in -= 11 */
253
254 movl %eax, in(%esp)
255 movl %edx, last(%esp)
256
257 movl start_sp(%esp), %ebp
258 movl avail_out_strm(strm_r), %ecx
259 movl next_out_strm(strm_r), %ebx
260
261 subl %ecx, %ebp /* start -= avail_out */
262 negl %ebp /* start = -start */
263 addl %ebx, %ebp /* start += next_out */
264
265 subl $257, %ecx /* avail_out -= 257 */
266 addl %ebx, %ecx /* avail_out += out */
267
268 movl %ebx, out(%esp)
269 movl %ebp, beg(%esp)
270 movl %ecx, end(%esp)
271
272 /* wsize = state->wsize;
273 * write = state->write;
274 * window = state->window;
275 * hold = state->hold;
276 * bits = state->bits;
277 * lcode = state->lencode;
278 * dcode = state->distcode;
279 * lmask = ( 1 << state->lenbits ) - 1;
280 * dmask = ( 1 << state->distbits ) - 1;
281 */
282
283 movl lencode_state(state_r), %eax
284 movl distcode_state(state_r), %ecx
285
286 movl %eax, lcode(%esp)
287 movl %ecx, dcode(%esp)
288
289 movl $1, %eax
290 movl lenbits_state(state_r), %ecx
291 shll %cl, %eax
292 decl %eax
293 movl %eax, lmask(%esp)
294
295 movl $1, %eax
296 movl distbits_state(state_r), %ecx
297 shll %cl, %eax
298 decl %eax
299 movl %eax, dmask(%esp)
300
301 movl wsize_state(state_r), %eax
302 movl write_state(state_r), %ecx
303 movl window_state(state_r), %edx
304
305 movl %eax, wsize(%esp)
306 movl %ecx, write(%esp)
307 movl %edx, window(%esp)
308
309 movl hold_state(state_r), %ebp
310 movl bits_state(state_r), %ebx
311
312 #undef strm_r
313 #undef state_r
314
315 #define in_r %esi
316 #define from_r %esi
317 #define out_r %edi
318
319 movl in(%esp), in_r
320 movl last(%esp), %ecx
321 cmpl in_r, %ecx
322 ja .L_align_long /* if in < last */
323
324 addl $11, %ecx /* ecx = &in[ avail_in ] */
325 subl in_r, %ecx /* ecx = avail_in */
326 movl $12, %eax
327 subl %ecx, %eax /* eax = 12 - avail_in */
328 leal buf(%esp), %edi
329 rep movsb /* memcpy( buf, in, avail_in ) */
330 movl %eax, %ecx
331 xorl %eax, %eax
332 rep stosb /* memset( &buf[ avail_in ], 0, 12 - avail_in ) */
333 leal buf(%esp), in_r /* in = buf */
334 movl in_r, last(%esp) /* last = in, do just one iteration */
335 jmp .L_is_aligned
336
337 /* align in_r on long boundary */
338 .L_align_long:
339 testl $3, in_r
340 jz .L_is_aligned
341 xorl %eax, %eax
342 movb (in_r), %al
343 incl in_r
344 movl %ebx, %ecx
345 addl $8, %ebx
346 shll %cl, %eax
347 orl %eax, %ebp
348 jmp .L_align_long
349
350 .L_is_aligned:
351 movl out(%esp), out_r
352
353 #if defined( NO_MMX )
354 jmp .L_do_loop
355 #endif
356
357 #if defined( USE_MMX )
358 jmp .L_init_mmx
359 #endif
360
361 /*** Runtime MMX check ***/
362
363 #if defined( RUN_TIME_MMX )
364 .L_check_mmx:
365 cmpl $DO_USE_MMX, inflate_fast_use_mmx
366 je .L_init_mmx
367 ja .L_do_loop /* > 2 */
368
369 pushl %eax
370 pushl %ebx
371 pushl %ecx
372 pushl %edx
373 pushf
374 movl (%esp), %eax /* copy eflags to eax */
375 xorl $0x200000, (%esp) /* try toggling ID bit of eflags (bit 21)
376 * to see if cpu supports cpuid...
377 * ID bit method not supported by NexGen but
378 * bios may load a cpuid instruction and
379 * cpuid may be disabled on Cyrix 5-6x86 */
380 popf
381 pushf
382 popl %edx /* copy new eflags to edx */
383 xorl %eax, %edx /* test if ID bit is flipped */
384 jz .L_dont_use_mmx /* not flipped if zero */
385 xorl %eax, %eax
386 cpuid
387 cmpl $0x756e6547, %ebx /* check for GenuineIntel in ebx,ecx,edx */
388 jne .L_dont_use_mmx
389 cmpl $0x6c65746e, %ecx
390 jne .L_dont_use_mmx
391 cmpl $0x49656e69, %edx
392 jne .L_dont_use_mmx
393 movl $1, %eax
394 cpuid /* get cpu features */
395 shrl $8, %eax
396 andl $15, %eax
397 cmpl $6, %eax /* check for Pentium family, is 0xf for P4 */
398 jne .L_dont_use_mmx
399 testl $0x800000, %edx /* test if MMX feature is set (bit 23) */
400 jnz .L_use_mmx
401 jmp .L_dont_use_mmx
402 .L_use_mmx:
403 movl $DO_USE_MMX, inflate_fast_use_mmx
404 jmp .L_check_mmx_pop
405 .L_dont_use_mmx:
406 movl $DONT_USE_MMX, inflate_fast_use_mmx
407 .L_check_mmx_pop:
408 popl %edx
409 popl %ecx
410 popl %ebx
411 popl %eax
412 jmp .L_check_mmx
413 #endif
414
415
416 /*** Non-MMX code ***/
417
418 #if defined ( NO_MMX ) || defined( RUN_TIME_MMX )
419
420 #define hold_r %ebp
421 #define bits_r %bl
422 #define bitslong_r %ebx
423
424 .align 32,0x90
425 .L_while_test:
426 /* while (in < last && out < end)
427 */
428 cmpl out_r, end(%esp)
429 jbe .L_break_loop /* if (out >= end) */
430
431 cmpl in_r, last(%esp)
432 jbe .L_break_loop
433
434 .L_do_loop:
435 /* regs: %esi = in, %ebp = hold, %bl = bits, %edi = out
436 *
437 * do {
438 * if (bits < 15) {
439 * hold |= *((unsigned short *)in)++ << bits;
440 * bits += 16
441 * }
442 * this = lcode[hold & lmask]
443 */
444 cmpb $15, bits_r
445 ja .L_get_length_code /* if (15 < bits) */
446
447 xorl %eax, %eax
448 lodsw /* al = *(ushort *)in++ */
449 movb bits_r, %cl /* cl = bits, needs it for shifting */
450 addb $16, bits_r /* bits += 16 */
451 shll %cl, %eax
452 orl %eax, hold_r /* hold |= *((ushort *)in)++ << bits */
453
454 .L_get_length_code:
455 movl lmask(%esp), %edx /* edx = lmask */
456 movl lcode(%esp), %ecx /* ecx = lcode */
457 andl hold_r, %edx /* edx &= hold */
458 movl (%ecx,%edx,4), %eax /* eax = lcode[hold & lmask] */
459
460 .L_dolen:
461 /* regs: %esi = in, %ebp = hold, %bl = bits, %edi = out
462 *
463 * dolen:
464 * bits -= this.bits;
465 * hold >>= this.bits
466 */
467 movb %ah, %cl /* cl = this.bits */
468 subb %ah, bits_r /* bits -= this.bits */
469 shrl %cl, hold_r /* hold >>= this.bits */
470
471 /* check if op is a literal
472 * if (op == 0) {
473 * PUP(out) = this.val;
474 * }
475 */
476 testb %al, %al
477 jnz .L_test_for_length_base /* if (op != 0) 45.7% */
478
479 shrl $16, %eax /* output this.val char */
480 stosb
481 jmp .L_while_test
482
483 .L_test_for_length_base:
484 /* regs: %esi = in, %ebp = hold, %bl = bits, %edi = out, %edx = len
485 *
486 * else if (op & 16) {
487 * len = this.val
488 * op &= 15
489 * if (op) {
490 * if (op > bits) {
491 * hold |= *((unsigned short *)in)++ << bits;
492 * bits += 16
493 * }
494 * len += hold & mask[op];
495 * bits -= op;
496 * hold >>= op;
497 * }
498 */
499 #define len_r %edx
500 movl %eax, len_r /* len = this */
501 shrl $16, len_r /* len = this.val */
502 movb %al, %cl
503
504 testb $16, %al
505 jz .L_test_for_second_level_length /* if ((op & 16) == 0) 8% */
506 andb $15, %cl /* op &= 15 */
507 jz .L_save_len /* if (!op) */
508 cmpb %cl, bits_r
509 jae .L_add_bits_to_len /* if (op <= bits) */
510
511 movb %cl, %ch /* stash op in ch, freeing cl */
512 xorl %eax, %eax
513 lodsw /* al = *(ushort *)in++ */
514 movb bits_r, %cl /* cl = bits, needs it for shifting */
515 addb $16, bits_r /* bits += 16 */
516 shll %cl, %eax
517 orl %eax, hold_r /* hold |= *((ushort *)in)++ << bits */
518 movb %ch, %cl /* move op back to ecx */
519
520 .L_add_bits_to_len:
521 movl $1, %eax
522 shll %cl, %eax
523 decl %eax
524 subb %cl, bits_r
525 andl hold_r, %eax /* eax &= hold */
526 shrl %cl, hold_r
527 addl %eax, len_r /* len += hold & mask[op] */
528
529 .L_save_len:
530 movl len_r, len(%esp) /* save len */
531 #undef len_r
532
533 .L_decode_distance:
534 /* regs: %esi = in, %ebp = hold, %bl = bits, %edi = out, %edx = dist
535 *
536 * if (bits < 15) {
537 * hold |= *((unsigned short *)in)++ << bits;
538 * bits += 16
539 * }
540 * this = dcode[hold & dmask];
541 * dodist:
542 * bits -= this.bits;
543 * hold >>= this.bits;
544 * op = this.op;
545 */
546
547 cmpb $15, bits_r
548 ja .L_get_distance_code /* if (15 < bits) */
549
550 xorl %eax, %eax
551 lodsw /* al = *(ushort *)in++ */
552 movb bits_r, %cl /* cl = bits, needs it for shifting */
553 addb $16, bits_r /* bits += 16 */
554 shll %cl, %eax
555 orl %eax, hold_r /* hold |= *((ushort *)in)++ << bits */
556
557 .L_get_distance_code:
558 movl dmask(%esp), %edx /* edx = dmask */
559 movl dcode(%esp), %ecx /* ecx = dcode */
560 andl hold_r, %edx /* edx &= hold */
561 movl (%ecx,%edx,4), %eax /* eax = dcode[hold & dmask] */
562
563 #define dist_r %edx
564 .L_dodist:
565 movl %eax, dist_r /* dist = this */
566 shrl $16, dist_r /* dist = this.val */
567 movb %ah, %cl
568 subb %ah, bits_r /* bits -= this.bits */
569 shrl %cl, hold_r /* hold >>= this.bits */
570
571 /* if (op & 16) {
572 * dist = this.val
573 * op &= 15
574 * if (op > bits) {
575 * hold |= *((unsigned short *)in)++ << bits;
576 * bits += 16
577 * }
578 * dist += hold & mask[op];
579 * bits -= op;
580 * hold >>= op;
581 */
582 movb %al, %cl /* cl = this.op */
583
584 testb $16, %al /* if ((op & 16) == 0) */
585 jz .L_test_for_second_level_dist
586 andb $15, %cl /* op &= 15 */
587 jz .L_check_dist_one
588 cmpb %cl, bits_r
589 jae .L_add_bits_to_dist /* if (op <= bits) 97.6% */
590
591 movb %cl, %ch /* stash op in ch, freeing cl */
592 xorl %eax, %eax
593 lodsw /* al = *(ushort *)in++ */
594 movb bits_r, %cl /* cl = bits, needs it for shifting */
595 addb $16, bits_r /* bits += 16 */
596 shll %cl, %eax
597 orl %eax, hold_r /* hold |= *((ushort *)in)++ << bits */
598 movb %ch, %cl /* move op back to ecx */
599
600 .L_add_bits_to_dist:
601 movl $1, %eax
602 shll %cl, %eax
603 decl %eax /* (1 << op) - 1 */
604 subb %cl, bits_r
605 andl hold_r, %eax /* eax &= hold */
606 shrl %cl, hold_r
607 addl %eax, dist_r /* dist += hold & ((1 << op) - 1) */
608 jmp .L_check_window
609
610 .L_check_window:
611 /* regs: %esi = from, %ebp = hold, %bl = bits, %edi = out, %edx = dist
612 * %ecx = nbytes
613 *
614 * nbytes = out - beg;
615 * if (dist <= nbytes) {
616 * from = out - dist;
617 * do {
618 * PUP(out) = PUP(from);
619 * } while (--len > 0) {
620 * }
621 */
622
623 movl in_r, in(%esp) /* save in so from can use it's reg */
624 movl out_r, %eax
625 subl beg(%esp), %eax /* nbytes = out - beg */
626
627 cmpl dist_r, %eax
628 jb .L_clip_window /* if (dist > nbytes) 4.2% */
629
630 movl len(%esp), %ecx
631 movl out_r, from_r
632 subl dist_r, from_r /* from = out - dist */
633
634 subl $3, %ecx
635 movb (from_r), %al
636 movb %al, (out_r)
637 movb 1(from_r), %al
638 movb 2(from_r), %dl
639 addl $3, from_r
640 movb %al, 1(out_r)
641 movb %dl, 2(out_r)
642 addl $3, out_r
643 rep movsb
644
645 movl in(%esp), in_r /* move in back to %esi, toss from */
646 jmp .L_while_test
647
648 .align 16,0x90
649 .L_check_dist_one:
650 cmpl $1, dist_r
651 jne .L_check_window
652 cmpl out_r, beg(%esp)
653 je .L_check_window
654
655 decl out_r
656 movl len(%esp), %ecx
657 movb (out_r), %al
658 subl $3, %ecx
659
660 movb %al, 1(out_r)
661 movb %al, 2(out_r)
662 movb %al, 3(out_r)
663 addl $4, out_r
664 rep stosb
665
666 jmp .L_while_test
667
668 .align 16,0x90
669 .L_test_for_second_level_length:
670 /* else if ((op & 64) == 0) {
671 * this = lcode[this.val + (hold & mask[op])];
672 * }
673 */
674 testb $64, %al
675 jnz .L_test_for_end_of_block /* if ((op & 64) != 0) */
676
677 movl $1, %eax
678 shll %cl, %eax
679 decl %eax
680 andl hold_r, %eax /* eax &= hold */
681 addl %edx, %eax /* eax += this.val */
682 movl lcode(%esp), %edx /* edx = lcode */
683 movl (%edx,%eax,4), %eax /* eax = lcode[val + (hold&mask[op])] */
684 jmp .L_dolen
685
686 .align 16,0x90
687 .L_test_for_second_level_dist:
688 /* else if ((op & 64) == 0) {
689 * this = dcode[this.val + (hold & mask[op])];
690 * }
691 */
692 testb $64, %al
693 jnz .L_invalid_distance_code /* if ((op & 64) != 0) */
694
695 movl $1, %eax
696 shll %cl, %eax
697 decl %eax
698 andl hold_r, %eax /* eax &= hold */
699 addl %edx, %eax /* eax += this.val */
700 movl dcode(%esp), %edx /* edx = dcode */
701 movl (%edx,%eax,4), %eax /* eax = dcode[val + (hold&mask[op])] */
702 jmp .L_dodist
703
704 .align 16,0x90
705 .L_clip_window:
706 /* regs: %esi = from, %ebp = hold, %bl = bits, %edi = out, %edx = dist
707 * %ecx = nbytes
708 *
709 * else {
710 * if (dist > wsize) {
711 * invalid distance
712 * }
713 * from = window;
714 * nbytes = dist - nbytes;
715 * if (write == 0) {
716 * from += wsize - nbytes;
717 */
718 #define nbytes_r %ecx
719 movl %eax, nbytes_r
720 movl wsize(%esp), %eax /* prepare for dist compare */
721 negl nbytes_r /* nbytes = -nbytes */
722 movl window(%esp), from_r /* from = window */
723
724 cmpl dist_r, %eax
725 jb .L_invalid_distance_too_far /* if (dist > wsize) */
726
727 addl dist_r, nbytes_r /* nbytes = dist - nbytes */
728 cmpl $0, write(%esp)
729 jne .L_wrap_around_window /* if (write != 0) */
730
731 subl nbytes_r, %eax
732 addl %eax, from_r /* from += wsize - nbytes */
733
734 /* regs: %esi = from, %ebp = hold, %bl = bits, %edi = out, %edx = dist
735 * %ecx = nbytes, %eax = len
736 *
737 * if (nbytes < len) {
738 * len -= nbytes;
739 * do {
740 * PUP(out) = PUP(from);
741 * } while (--nbytes);
742 * from = out - dist;
743 * }
744 * }
745 */
746 #define len_r %eax
747 movl len(%esp), len_r
748 cmpl nbytes_r, len_r
749 jbe .L_do_copy1 /* if (nbytes >= len) */
750
751 subl nbytes_r, len_r /* len -= nbytes */
752 rep movsb
753 movl out_r, from_r
754 subl dist_r, from_r /* from = out - dist */
755 jmp .L_do_copy1
756
757 cmpl nbytes_r, len_r
758 jbe .L_do_copy1 /* if (nbytes >= len) */
759
760 subl nbytes_r, len_r /* len -= nbytes */
761 rep movsb
762 movl out_r, from_r
763 subl dist_r, from_r /* from = out - dist */
764 jmp .L_do_copy1
765
766 .L_wrap_around_window:
767 /* regs: %esi = from, %ebp = hold, %bl = bits, %edi = out, %edx = dist
768 * %ecx = nbytes, %eax = write, %eax = len
769 *
770 * else if (write < nbytes) {
771 * from += wsize + write - nbytes;
772 * nbytes -= write;
773 * if (nbytes < len) {
774 * len -= nbytes;
775 * do {
776 * PUP(out) = PUP(from);
777 * } while (--nbytes);
778 * from = window;
779 * nbytes = write;
780 * if (nbytes < len) {
781 * len -= nbytes;
782 * do {
783 * PUP(out) = PUP(from);
784 * } while(--nbytes);
785 * from = out - dist;
786 * }
787 * }
788 * }
789 */
790 #define write_r %eax
791 movl write(%esp), write_r
792 cmpl write_r, nbytes_r
793 jbe .L_contiguous_in_window /* if (write >= nbytes) */
794
795 addl wsize(%esp), from_r
796 addl write_r, from_r
797 subl nbytes_r, from_r /* from += wsize + write - nbytes */
798 subl write_r, nbytes_r /* nbytes -= write */
799 #undef write_r
800
801 movl len(%esp), len_r
802 cmpl nbytes_r, len_r
803 jbe .L_do_copy1 /* if (nbytes >= len) */
804
805 subl nbytes_r, len_r /* len -= nbytes */
806 rep movsb
807 movl window(%esp), from_r /* from = window */
808 movl write(%esp), nbytes_r /* nbytes = write */
809 cmpl nbytes_r, len_r
810 jbe .L_do_copy1 /* if (nbytes >= len) */
811
812 subl nbytes_r, len_r /* len -= nbytes */
813 rep movsb
814 movl out_r, from_r
815 subl dist_r, from_r /* from = out - dist */
816 jmp .L_do_copy1
817
818 .L_contiguous_in_window:
819 /* regs: %esi = from, %ebp = hold, %bl = bits, %edi = out, %edx = dist
820 * %ecx = nbytes, %eax = write, %eax = len
821 *
822 * else {
823 * from += write - nbytes;
824 * if (nbytes < len) {
825 * len -= nbytes;
826 * do {
827 * PUP(out) = PUP(from);
828 * } while (--nbytes);
829 * from = out - dist;
830 * }
831 * }
832 */
833 #define write_r %eax
834 addl write_r, from_r
835 subl nbytes_r, from_r /* from += write - nbytes */
836 #undef write_r
837
838 movl len(%esp), len_r
839 cmpl nbytes_r, len_r
840 jbe .L_do_copy1 /* if (nbytes >= len) */
841
842 subl nbytes_r, len_r /* len -= nbytes */
843 rep movsb
844 movl out_r, from_r
845 subl dist_r, from_r /* from = out - dist */
846
847 .L_do_copy1:
848 /* regs: %esi = from, %esi = in, %ebp = hold, %bl = bits, %edi = out
849 * %eax = len
850 *
851 * while (len > 0) {
852 * PUP(out) = PUP(from);
853 * len--;
854 * }
855 * }
856 * } while (in < last && out < end);
857 */
858 #undef nbytes_r
859 #define in_r %esi
860 movl len_r, %ecx
861 rep movsb
862
863 movl in(%esp), in_r /* move in back to %esi, toss from */
864 jmp .L_while_test
865
866 #undef len_r
867 #undef dist_r
868
869 #endif /* NO_MMX || RUN_TIME_MMX */
870
871
872 /*** MMX code ***/
873
874 #if defined( USE_MMX ) || defined( RUN_TIME_MMX )
875
876 .align 32,0x90
877 .L_init_mmx:
878 emms
879
880 #undef bits_r
881 #undef bitslong_r
882 #define bitslong_r %ebp
883 #define hold_mm %mm0
884 movd %ebp, hold_mm
885 movl %ebx, bitslong_r
886
887 #define used_mm %mm1
888 #define dmask2_mm %mm2
889 #define lmask2_mm %mm3
890 #define lmask_mm %mm4
891 #define dmask_mm %mm5
892 #define tmp_mm %mm6
893
894 movd lmask(%esp), lmask_mm
895 movq lmask_mm, lmask2_mm
896 movd dmask(%esp), dmask_mm
897 movq dmask_mm, dmask2_mm
898 pxor used_mm, used_mm
899 movl lcode(%esp), %ebx /* ebx = lcode */
900 jmp .L_do_loop_mmx
901
902 .align 32,0x90
903 .L_while_test_mmx:
904 /* while (in < last && out < end)
905 */
906 cmpl out_r, end(%esp)
907 jbe .L_break_loop /* if (out >= end) */
908
909 cmpl in_r, last(%esp)
910 jbe .L_break_loop
911
912 .L_do_loop_mmx:
913 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
914
915 cmpl $32, bitslong_r
916 ja .L_get_length_code_mmx /* if (32 < bits) */
917
918 movd bitslong_r, tmp_mm
919 movd (in_r), %mm7
920 addl $4, in_r
921 psllq tmp_mm, %mm7
922 addl $32, bitslong_r
923 por %mm7, hold_mm /* hold_mm |= *((uint *)in)++ << bits */
924
925 .L_get_length_code_mmx:
926 pand hold_mm, lmask_mm
927 movd lmask_mm, %eax
928 movq lmask2_mm, lmask_mm
929 movl (%ebx,%eax,4), %eax /* eax = lcode[hold & lmask] */
930
931 .L_dolen_mmx:
932 movzbl %ah, %ecx /* ecx = this.bits */
933 movd %ecx, used_mm
934 subl %ecx, bitslong_r /* bits -= this.bits */
935
936 testb %al, %al
937 jnz .L_test_for_length_base_mmx /* if (op != 0) 45.7% */
938
939 shrl $16, %eax /* output this.val char */
940 stosb
941 jmp .L_while_test_mmx
942
943 .L_test_for_length_base_mmx:
944 #define len_r %edx
945 movl %eax, len_r /* len = this */
946 shrl $16, len_r /* len = this.val */
947
948 testb $16, %al
949 jz .L_test_for_second_level_length_mmx /* if ((op & 16) == 0) 8% */
950 andl $15, %eax /* op &= 15 */
951 jz .L_decode_distance_mmx /* if (!op) */
952
953 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
954 movd %eax, used_mm
955 movd hold_mm, %ecx
956 subl %eax, bitslong_r
957 andl .L_mask(,%eax,4), %ecx
958 addl %ecx, len_r /* len += hold & mask[op] */
959
960 .L_decode_distance_mmx:
961 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
962
963 cmpl $32, bitslong_r
964 ja .L_get_dist_code_mmx /* if (32 < bits) */
965
966 movd bitslong_r, tmp_mm
967 movd (in_r), %mm7
968 addl $4, in_r
969 psllq tmp_mm, %mm7
970 addl $32, bitslong_r
971 por %mm7, hold_mm /* hold_mm |= *((uint *)in)++ << bits */
972
973 .L_get_dist_code_mmx:
974 movl dcode(%esp), %ebx /* ebx = dcode */
975 pand hold_mm, dmask_mm
976 movd dmask_mm, %eax
977 movq dmask2_mm, dmask_mm
978 movl (%ebx,%eax,4), %eax /* eax = dcode[hold & lmask] */
979
980 .L_dodist_mmx:
981 #define dist_r %ebx
982 movzbl %ah, %ecx /* ecx = this.bits */
983 movl %eax, dist_r
984 shrl $16, dist_r /* dist = this.val */
985 subl %ecx, bitslong_r /* bits -= this.bits */
986 movd %ecx, used_mm
987
988 testb $16, %al /* if ((op & 16) == 0) */
989 jz .L_test_for_second_level_dist_mmx
990 andl $15, %eax /* op &= 15 */
991 jz .L_check_dist_one_mmx
992
993 .L_add_bits_to_dist_mmx:
994 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
995 movd %eax, used_mm /* save bit length of current op */
996 movd hold_mm, %ecx /* get the next bits on input stream */
997 subl %eax, bitslong_r /* bits -= op bits */
998 andl .L_mask(,%eax,4), %ecx /* ecx = hold & mask[op] */
999 addl %ecx, dist_r /* dist += hold & mask[op] */
1000
1001 .L_check_window_mmx:
1002 movl in_r, in(%esp) /* save in so from can use it's reg */
1003 movl out_r, %eax
1004 subl beg(%esp), %eax /* nbytes = out - beg */
1005
1006 cmpl dist_r, %eax
1007 jb .L_clip_window_mmx /* if (dist > nbytes) 4.2% */
1008
1009 movl len_r, %ecx
1010 movl out_r, from_r
1011 subl dist_r, from_r /* from = out - dist */
1012
1013 subl $3, %ecx
1014 movb (from_r), %al
1015 movb %al, (out_r)
1016 movb 1(from_r), %al
1017 movb 2(from_r), %dl
1018 addl $3, from_r
1019 movb %al, 1(out_r)
1020 movb %dl, 2(out_r)
1021 addl $3, out_r
1022 rep movsb
1023
1024 movl in(%esp), in_r /* move in back to %esi, toss from */
1025 movl lcode(%esp), %ebx /* move lcode back to %ebx, toss dist */
1026 jmp .L_while_test_mmx
1027
1028 .align 16,0x90
1029 .L_check_dist_one_mmx:
1030 cmpl $1, dist_r
1031 jne .L_check_window_mmx
1032 cmpl out_r, beg(%esp)
1033 je .L_check_window_mmx
1034
1035 decl out_r
1036 movl len_r, %ecx
1037 movb (out_r), %al
1038 subl $3, %ecx
1039
1040 movb %al, 1(out_r)
1041 movb %al, 2(out_r)
1042 movb %al, 3(out_r)
1043 addl $4, out_r
1044 rep stosb
1045
1046 movl lcode(%esp), %ebx /* move lcode back to %ebx, toss dist */
1047 jmp .L_while_test_mmx
1048
1049 .align 16,0x90
1050 .L_test_for_second_level_length_mmx:
1051 testb $64, %al
1052 jnz .L_test_for_end_of_block /* if ((op & 64) != 0) */
1053
1054 andl $15, %eax
1055 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
1056 movd hold_mm, %ecx
1057 andl .L_mask(,%eax,4), %ecx
1058 addl len_r, %ecx
1059 movl (%ebx,%ecx,4), %eax /* eax = lcode[hold & lmask] */
1060 jmp .L_dolen_mmx
1061
1062 .align 16,0x90
1063 .L_test_for_second_level_dist_mmx:
1064 testb $64, %al
1065 jnz .L_invalid_distance_code /* if ((op & 64) != 0) */
1066
1067 andl $15, %eax
1068 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
1069 movd hold_mm, %ecx
1070 andl .L_mask(,%eax,4), %ecx
1071 movl dcode(%esp), %eax /* ecx = dcode */
1072 addl dist_r, %ecx
1073 movl (%eax,%ecx,4), %eax /* eax = lcode[hold & lmask] */
1074 jmp .L_dodist_mmx
1075
1076 .align 16,0x90
1077 .L_clip_window_mmx:
1078 #define nbytes_r %ecx
1079 movl %eax, nbytes_r
1080 movl wsize(%esp), %eax /* prepare for dist compare */
1081 negl nbytes_r /* nbytes = -nbytes */
1082 movl window(%esp), from_r /* from = window */
1083
1084 cmpl dist_r, %eax
1085 jb .L_invalid_distance_too_far /* if (dist > wsize) */
1086
1087 addl dist_r, nbytes_r /* nbytes = dist - nbytes */
1088 cmpl $0, write(%esp)
1089 jne .L_wrap_around_window_mmx /* if (write != 0) */
1090
1091 subl nbytes_r, %eax
1092 addl %eax, from_r /* from += wsize - nbytes */
1093
1094 cmpl nbytes_r, len_r
1095 jbe .L_do_copy1_mmx /* if (nbytes >= len) */
1096
1097 subl nbytes_r, len_r /* len -= nbytes */
1098 rep movsb
1099 movl out_r, from_r
1100 subl dist_r, from_r /* from = out - dist */
1101 jmp .L_do_copy1_mmx
1102
1103 cmpl nbytes_r, len_r
1104 jbe .L_do_copy1_mmx /* if (nbytes >= len) */
1105
1106 subl nbytes_r, len_r /* len -= nbytes */
1107 rep movsb
1108 movl out_r, from_r
1109 subl dist_r, from_r /* from = out - dist */
1110 jmp .L_do_copy1_mmx
1111
1112 .L_wrap_around_window_mmx:
1113 #define write_r %eax
1114 movl write(%esp), write_r
1115 cmpl write_r, nbytes_r
1116 jbe .L_contiguous_in_window_mmx /* if (write >= nbytes) */
1117
1118 addl wsize(%esp), from_r
1119 addl write_r, from_r
1120 subl nbytes_r, from_r /* from += wsize + write - nbytes */
1121 subl write_r, nbytes_r /* nbytes -= write */
1122 #undef write_r
1123
1124 cmpl nbytes_r, len_r
1125 jbe .L_do_copy1_mmx /* if (nbytes >= len) */
1126
1127 subl nbytes_r, len_r /* len -= nbytes */
1128 rep movsb
1129 movl window(%esp), from_r /* from = window */
1130 movl write(%esp), nbytes_r /* nbytes = write */
1131 cmpl nbytes_r, len_r
1132 jbe .L_do_copy1_mmx /* if (nbytes >= len) */
1133
1134 subl nbytes_r, len_r /* len -= nbytes */
1135 rep movsb
1136 movl out_r, from_r
1137 subl dist_r, from_r /* from = out - dist */
1138 jmp .L_do_copy1_mmx
1139
1140 .L_contiguous_in_window_mmx:
1141 #define write_r %eax
1142 addl write_r, from_r
1143 subl nbytes_r, from_r /* from += write - nbytes */
1144 #undef write_r
1145
1146 cmpl nbytes_r, len_r
1147 jbe .L_do_copy1_mmx /* if (nbytes >= len) */
1148
1149 subl nbytes_r, len_r /* len -= nbytes */
1150 rep movsb
1151 movl out_r, from_r
1152 subl dist_r, from_r /* from = out - dist */
1153
1154 .L_do_copy1_mmx:
1155 #undef nbytes_r
1156 #define in_r %esi
1157 movl len_r, %ecx
1158 rep movsb
1159
1160 movl in(%esp), in_r /* move in back to %esi, toss from */
1161 movl lcode(%esp), %ebx /* move lcode back to %ebx, toss dist */
1162 jmp .L_while_test_mmx
1163
1164 #undef hold_r
1165 #undef bitslong_r
1166
1167 #endif /* USE_MMX || RUN_TIME_MMX */
1168
1169
1170 /*** USE_MMX, NO_MMX, and RUNTIME_MMX from here on ***/
1171
1172 .L_invalid_distance_code:
1173 /* else {
1174 * strm->msg = "invalid distance code";
1175 * state->mode = BAD;
1176 * }
1177 */
1178 movl $.L_invalid_distance_code_msg, %ecx
1179 movl $INFLATE_MODE_BAD, %edx
1180 jmp .L_update_stream_state
1181
1182 .L_test_for_end_of_block:
1183 /* else if (op & 32) {
1184 * state->mode = TYPE;
1185 * break;
1186 * }
1187 */
1188 testb $32, %al
1189 jz .L_invalid_literal_length_code /* if ((op & 32) == 0) */
1190
1191 movl $0, %ecx
1192 movl $INFLATE_MODE_TYPE, %edx
1193 jmp .L_update_stream_state
1194
1195 .L_invalid_literal_length_code:
1196 /* else {
1197 * strm->msg = "invalid literal/length code";
1198 * state->mode = BAD;
1199 * }
1200 */
1201 movl $.L_invalid_literal_length_code_msg, %ecx
1202 movl $INFLATE_MODE_BAD, %edx
1203 jmp .L_update_stream_state
1204
1205 .L_invalid_distance_too_far:
1206 /* strm->msg = "invalid distance too far back";
1207 * state->mode = BAD;
1208 */
1209 movl in(%esp), in_r /* from_r has in's reg, put in back */
1210 movl $.L_invalid_distance_too_far_msg, %ecx
1211 movl $INFLATE_MODE_BAD, %edx
1212 jmp .L_update_stream_state
1213
1214 .L_update_stream_state:
1215 /* set strm->msg = %ecx, strm->state->mode = %edx */
1216 movl strm_sp(%esp), %eax
1217 testl %ecx, %ecx /* if (msg != NULL) */
1218 jz .L_skip_msg
1219 movl %ecx, msg_strm(%eax) /* strm->msg = msg */
1220 .L_skip_msg:
1221 movl state_strm(%eax), %eax /* state = strm->state */
1222 movl %edx, mode_state(%eax) /* state->mode = edx (BAD | TYPE) */
1223 jmp .L_break_loop
1224
1225 .align 32,0x90
1226 .L_break_loop:
1227
1228 /*
1229 * Regs:
1230 *
1231 * bits = %ebp when mmx, and in %ebx when non-mmx
1232 * hold = %hold_mm when mmx, and in %ebp when non-mmx
1233 * in = %esi
1234 * out = %edi
1235 */
1236
1237 #if defined( USE_MMX ) || defined( RUN_TIME_MMX )
1238
1239 #if defined( RUN_TIME_MMX )
1240
1241 cmpl $DO_USE_MMX, inflate_fast_use_mmx
1242 jne .L_update_next_in
1243
1244 #endif /* RUN_TIME_MMX */
1245
1246 movl %ebp, %ebx
1247
1248 .L_update_next_in:
1249
1250 #endif
1251
1252 #define strm_r %eax
1253 #define state_r %edx
1254
1255 /* len = bits >> 3;
1256 * in -= len;
1257 * bits -= len << 3;
1258 * hold &= (1U << bits) - 1;
1259 * state->hold = hold;
1260 * state->bits = bits;
1261 * strm->next_in = in;
1262 * strm->next_out = out;
1263 */
1264 movl strm_sp(%esp), strm_r
1265 movl %ebx, %ecx
1266 movl state_strm(strm_r), state_r
1267 shrl $3, %ecx
1268 subl %ecx, in_r
1269 shll $3, %ecx
1270 subl %ecx, %ebx
1271 movl out_r, next_out_strm(strm_r)
1272 movl %ebx, bits_state(state_r)
1273 movl %ebx, %ecx
1274
1275 leal buf(%esp), %ebx
1276 cmpl %ebx, last(%esp)
1277 jne .L_buf_not_used /* if buf != last */
1278
1279 subl %ebx, in_r /* in -= buf */
1280 movl next_in_strm(strm_r), %ebx
1281 movl %ebx, last(%esp) /* last = strm->next_in */
1282 addl %ebx, in_r /* in += strm->next_in */
1283 movl avail_in_strm(strm_r), %ebx
1284 subl $11, %ebx
1285 addl %ebx, last(%esp) /* last = &strm->next_in[ avail_in - 11 ] */
1286
1287 .L_buf_not_used:
1288 movl in_r, next_in_strm(strm_r)
1289
1290 movl $1, %ebx
1291 shll %cl, %ebx
1292 decl %ebx
1293
1294 #if defined( USE_MMX ) || defined( RUN_TIME_MMX )
1295
1296 #if defined( RUN_TIME_MMX )
1297
1298 cmpl $DO_USE_MMX, inflate_fast_use_mmx
1299 jne .L_update_hold
1300
1301 #endif /* RUN_TIME_MMX */
1302
1303 psrlq used_mm, hold_mm /* hold_mm >>= last bit length */
1304 movd hold_mm, %ebp
1305
1306 emms
1307
1308 .L_update_hold:
1309
1310 #endif /* USE_MMX || RUN_TIME_MMX */
1311
1312 andl %ebx, %ebp
1313 movl %ebp, hold_state(state_r)
1314
1315 #define last_r %ebx
1316
1317 /* strm->avail_in = in < last ? 11 + (last - in) : 11 - (in - last) */
1318 movl last(%esp), last_r
1319 cmpl in_r, last_r
1320 jbe .L_last_is_smaller /* if (in >= last) */
1321
1322 subl in_r, last_r /* last -= in */
1323 addl $11, last_r /* last += 11 */
1324 movl last_r, avail_in_strm(strm_r)
1325 jmp .L_fixup_out
1326 .L_last_is_smaller:
1327 subl last_r, in_r /* in -= last */
1328 negl in_r /* in = -in */
1329 addl $11, in_r /* in += 11 */
1330 movl in_r, avail_in_strm(strm_r)
1331
1332 #undef last_r
1333 #define end_r %ebx
1334
1335 .L_fixup_out:
1336 /* strm->avail_out = out < end ? 257 + (end - out) : 257 - (out - end)*/
1337 movl end(%esp), end_r
1338 cmpl out_r, end_r
1339 jbe .L_end_is_smaller /* if (out >= end) */
1340
1341 subl out_r, end_r /* end -= out */
1342 addl $257, end_r /* end += 257 */
1343 movl end_r, avail_out_strm(strm_r)
1344 jmp .L_done
1345 .L_end_is_smaller:
1346 subl end_r, out_r /* out -= end */
1347 negl out_r /* out = -out */
1348 addl $257, out_r /* out += 257 */
1349 movl out_r, avail_out_strm(strm_r)
1350
1351 #undef end_r
1352 #undef strm_r
1353 #undef state_r
1354
1355 .L_done:
1356 addl $local_var_size, %esp
1357 popf
1358 popl %ebx
1359 popl %ebp
1360 popl %esi
1361 popl %edi
1362 ret
1363
1364 #if defined( GAS_ELF )
1365 /* elf info */
1366 .type inflate_fast,@function
1367 .size inflate_fast,.-inflate_fast
1368 #endif