1 /* gzjoin -- command to join gzip files into one gzip file
2
3 Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved
4 version 1.2, 14 Aug 2012
5
6 This software is provided 'as-is', without any express or implied
7 warranty. In no event will the author be held liable for any damages
8 arising from the use of this software.
9
10 Permission is granted to anyone to use this software for any purpose,
11 including commercial applications, and to alter it and redistribute it
12 freely, subject to the following restrictions:
13
14 1. The origin of this software must not be misrepresented; you must not
15 claim that you wrote the original software. If you use this software
16 in a product, an acknowledgment in the product documentation would be
17 appreciated but is not required.
18 2. Altered source versions must be plainly marked as such, and must not be
19 misrepresented as being the original software.
20 3. This notice may not be removed or altered from any source distribution.
21
22 Mark Adler madler@alumni.caltech.edu
23 */
24
25 /*
26 * Change history:
27 *
28 * 1.0 11 Dec 2004 - First version
29 * 1.1 12 Jun 2005 - Changed ssize_t to long for portability
30 * 1.2 14 Aug 2012 - Clean up for z_const usage
31 */
32
33 /*
34 gzjoin takes one or more gzip files on the command line and writes out a
35 single gzip file that will uncompress to the concatenation of the
36 uncompressed data from the individual gzip files. gzjoin does this without
37 having to recompress any of the data and without having to calculate a new
38 crc32 for the concatenated uncompressed data. gzjoin does however have to
39 decompress all of the input data in order to find the bits in the compressed
40 data that need to be modified to concatenate the streams.
41
42 gzjoin does not do an integrity check on the input gzip files other than
43 checking the gzip header and decompressing the compressed data. They are
44 otherwise assumed to be complete and correct.
45
46 Each joint between gzip files removes at least 18 bytes of previous trailer
47 and subsequent header, and inserts an average of about three bytes to the
48 compressed data in order to connect the streams. The output gzip file
49 has a minimal ten-byte gzip header with no file name or modification time.
50
51 This program was written to illustrate the use of the Z_BLOCK option of
52 inflate() and the crc32_combine() function. gzjoin will not compile with
53 versions of zlib earlier than 1.2.3.
54 */
55
56 #include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */
57 #include <stdlib.h> /* exit(), malloc(), free() */
58 #include <fcntl.h> /* open() */
59 #include <unistd.h> /* close(), read(), lseek() */
60 #include "zlib.h"
61 /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
62
63 #define local static
64
65 /* exit with an error (return a value to allow use in an expression) */
66 local int bail(char *why1, char *why2)
67 {
68 fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2);
69 exit(1);
70 return 0;
71 }
72
73 /* -- simple buffered file input with access to the buffer -- */
74
75 #define CHUNK 32768 /* must be a power of two and fit in unsigned */
76
77 /* bin buffered input file type */
78 typedef struct {
79 char *name; /* name of file for error messages */
80 int fd; /* file descriptor */
81 unsigned left; /* bytes remaining at next */
82 unsigned char *next; /* next byte to read */
83 unsigned char *buf; /* allocated buffer of length CHUNK */
84 } bin;
85
86 /* close a buffered file and free allocated memory */
87 local void bclose(bin *in)
88 {
89 if (in != NULL) {
90 if (in->fd != -1)
91 close(in->fd);
92 if (in->buf != NULL)
93 free(in->buf);
94 free(in);
95 }
96 }
97
98 /* open a buffered file for input, return a pointer to type bin, or NULL on
99 failure */
100 local bin *bopen(char *name)
101 {
102 bin *in;
103
104 in = malloc(sizeof(bin));
105 if (in == NULL)
106 return NULL;
107 in->buf = malloc(CHUNK);
108 in->fd = open(name, O_RDONLY, 0);
109 if (in->buf == NULL || in->fd == -1) {
110 bclose(in);
111 return NULL;
112 }
113 in->left = 0;
114 in->next = in->buf;
115 in->name = name;
116 return in;
117 }
118
119 /* load buffer from file, return -1 on read error, 0 or 1 on success, with
120 1 indicating that end-of-file was reached */
121 local int bload(bin *in)
122 {
123 long len;
124
125 if (in == NULL)
126 return -1;
127 if (in->left != 0)
128 return 0;
129 in->next = in->buf;
130 do {
131 len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left);
132 if (len < 0)
133 return -1;
134 in->left += (unsigned)len;
135 } while (len != 0 && in->left < CHUNK);
136 return len == 0 ? 1 : 0;
137 }
138
139 /* get a byte from the file, bail if end of file */
140 #define bget(in) (in->left ? 0 : bload(in), \
141 in->left ? (in->left--, *(in->next)++) : \
142 bail("unexpected end of file on ", in->name))
143
144 /* get a four-byte little-endian unsigned integer from file */
145 local unsigned long bget4(bin *in)
146 {
147 unsigned long val;
148
149 val = bget(in);
150 val += (unsigned long)(bget(in)) << 8;
151 val += (unsigned long)(bget(in)) << 16;
152 val += (unsigned long)(bget(in)) << 24;
153 return val;
154 }
155
156 /* skip bytes in file */
157 local void bskip(bin *in, unsigned skip)
158 {
159 /* check pointer */
160 if (in == NULL)
161 return;
162
163 /* easy case -- skip bytes in buffer */
164 if (skip <= in->left) {
165 in->left -= skip;
166 in->next += skip;
167 return;
168 }
169
170 /* skip what's in buffer, discard buffer contents */
171 skip -= in->left;
172 in->left = 0;
173
174 /* seek past multiples of CHUNK bytes */
175 if (skip > CHUNK) {
176 unsigned left;
177
178 left = skip & (CHUNK - 1);
179 if (left == 0) {
180 /* exact number of chunks: seek all the way minus one byte to check
181 for end-of-file with a read */
182 lseek(in->fd, skip - 1, SEEK_CUR);
183 if (read(in->fd, in->buf, 1) != 1)
184 bail("unexpected end of file on ", in->name);
185 return;
186 }
187
188 /* skip the integral chunks, update skip with remainder */
189 lseek(in->fd, skip - left, SEEK_CUR);
190 skip = left;
191 }
192
193 /* read more input and skip remainder */
194 bload(in);
195 if (skip > in->left)
196 bail("unexpected end of file on ", in->name);
197 in->left -= skip;
198 in->next += skip;
199 }
200
201 /* -- end of buffered input functions -- */
202
203 /* skip the gzip header from file in */
204 local void gzhead(bin *in)
205 {
206 int flags;
207
208 /* verify gzip magic header and compression method */
209 if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8)
210 bail(in->name, " is not a valid gzip file");
211
212 /* get and verify flags */
213 flags = bget(in);
214 if ((flags & 0xe0) != 0)
215 bail("unknown reserved bits set in ", in->name);
216
217 /* skip modification time, extra flags, and os */
218 bskip(in, 6);
219
220 /* skip extra field if present */
221 if (flags & 4) {
222 unsigned len;
223
224 len = bget(in);
225 len += (unsigned)(bget(in)) << 8;
226 bskip(in, len);
227 }
228
229 /* skip file name if present */
230 if (flags & 8)
231 while (bget(in) != 0)
232 ;
233
234 /* skip comment if present */
235 if (flags & 16)
236 while (bget(in) != 0)
237 ;
238
239 /* skip header crc if present */
240 if (flags & 2)
241 bskip(in, 2);
242 }
243
244 /* write a four-byte little-endian unsigned integer to out */
245 local void put4(unsigned long val, FILE *out)
246 {
247 putc(val & 0xff, out);
248 putc((val >> 8) & 0xff, out);
249 putc((val >> 16) & 0xff, out);
250 putc((val >> 24) & 0xff, out);
251 }
252
253 /* Load up zlib stream from buffered input, bail if end of file */
254 local void zpull(z_streamp strm, bin *in)
255 {
256 if (in->left == 0)
257 bload(in);
258 if (in->left == 0)
259 bail("unexpected end of file on ", in->name);
260 strm->avail_in = in->left;
261 strm->next_in = in->next;
262 }
263
264 /* Write header for gzip file to out and initialize trailer. */
265 local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out)
266 {
267 fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
268 *crc = crc32(0L, Z_NULL, 0);
269 *tot = 0;
270 }
271
272 /* Copy the compressed data from name, zeroing the last block bit of the last
273 block if clr is true, and adding empty blocks as needed to get to a byte
274 boundary. If clr is false, then the last block becomes the last block of
275 the output, and the gzip trailer is written. crc and tot maintains the
276 crc and length (modulo 2^32) of the output for the trailer. The resulting
277 gzip file is written to out. gzinit() must be called before the first call
278 of gzcopy() to write the gzip header and to initialize crc and tot. */
279 local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot,
280 FILE *out)
281 {
282 int ret; /* return value from zlib functions */
283 int pos; /* where the "last block" bit is in byte */
284 int last; /* true if processing the last block */
285 bin *in; /* buffered input file */
286 unsigned char *start; /* start of compressed data in buffer */
287 unsigned char *junk; /* buffer for uncompressed data -- discarded */
288 z_off_t len; /* length of uncompressed data (support > 4 GB) */
289 z_stream strm; /* zlib inflate stream */
290
291 /* open gzip file and skip header */
292 in = bopen(name);
293 if (in == NULL)
294 bail("could not open ", name);
295 gzhead(in);
296
297 /* allocate buffer for uncompressed data and initialize raw inflate
298 stream */
299 junk = malloc(CHUNK);
300 strm.zalloc = Z_NULL;
301 strm.zfree = Z_NULL;
302 strm.opaque = Z_NULL;
303 strm.avail_in = 0;
304 strm.next_in = Z_NULL;
305 ret = inflateInit2(&strm, -15);
306 if (junk == NULL || ret != Z_OK)
307 bail("out of memory", "");
308
309 /* inflate and copy compressed data, clear last-block bit if requested */
310 len = 0;
311 zpull(&strm, in);
312 start = in->next;
313 last = start[0] & 1;
314 if (last && clr)
315 start[0] &= ~1;
316 strm.avail_out = 0;
317 for (;;) {
318 /* if input used and output done, write used input and get more */
319 if (strm.avail_in == 0 && strm.avail_out != 0) {
320 fwrite(start, 1, strm.next_in - start, out);
321 start = in->buf;
322 in->left = 0;
323 zpull(&strm, in);
324 }
325
326 /* decompress -- return early when end-of-block reached */
327 strm.avail_out = CHUNK;
328 strm.next_out = junk;
329 ret = inflate(&strm, Z_BLOCK);
330 switch (ret) {
331 case Z_MEM_ERROR:
332 bail("out of memory", "");
333 case Z_DATA_ERROR:
334 bail("invalid compressed data in ", in->name);
335 }
336
337 /* update length of uncompressed data */
338 len += CHUNK - strm.avail_out;
339
340 /* check for block boundary (only get this when block copied out) */
341 if (strm.data_type & 128) {
342 /* if that was the last block, then done */
343 if (last)
344 break;
345
346 /* number of unused bits in last byte */
347 pos = strm.data_type & 7;
348
349 /* find the next last-block bit */
350 if (pos != 0) {
351 /* next last-block bit is in last used byte */
352 pos = 0x100 >> pos;
353 last = strm.next_in[-1] & pos;
354 if (last && clr)
355 in->buf[strm.next_in - in->buf - 1] &= ~pos;
356 }
357 else {
358 /* next last-block bit is in next unused byte */
359 if (strm.avail_in == 0) {
360 /* don't have that byte yet -- get it */
361 fwrite(start, 1, strm.next_in - start, out);
362 start = in->buf;
363 in->left = 0;
364 zpull(&strm, in);
365 }
366 last = strm.next_in[0] & 1;
367 if (last && clr)
368 in->buf[strm.next_in - in->buf] &= ~1;
369 }
370 }
371 }
372
373 /* update buffer with unused input */
374 in->left = strm.avail_in;
375 in->next = in->buf + (strm.next_in - in->buf);
376
377 /* copy used input, write empty blocks to get to byte boundary */
378 pos = strm.data_type & 7;
379 fwrite(start, 1, in->next - start - 1, out);
380 last = in->next[-1];
381 if (pos == 0 || !clr)
382 /* already at byte boundary, or last file: write last byte */
383 putc(last, out);
384 else {
385 /* append empty blocks to last byte */
386 last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */
387 if (pos & 1) {
388 /* odd -- append an empty stored block */
389 putc(last, out);
390 if (pos == 1)
391 putc(0, out); /* two more bits in block header */
392 fwrite("\0\0\xff\xff", 1, 4, out);
393 }
394 else {
395 /* even -- append 1, 2, or 3 empty fixed blocks */
396 switch (pos) {
397 case 6:
398 putc(last | 8, out);
399 last = 0;
400 case 4:
401 putc(last | 0x20, out);
402 last = 0;
403 case 2:
404 putc(last | 0x80, out);
405 putc(0, out);
406 }
407 }
408 }
409
410 /* update crc and tot */
411 *crc = crc32_combine(*crc, bget4(in), len);
412 *tot += (unsigned long)len;
413
414 /* clean up */
415 inflateEnd(&strm);
416 free(junk);
417 bclose(in);
418
419 /* write trailer if this is the last gzip file */
420 if (!clr) {
421 put4(*crc, out);
422 put4(*tot, out);
423 }
424 }
425
426 /* join the gzip files on the command line, write result to stdout */
427 int main(int argc, char **argv)
428 {
429 unsigned long crc, tot; /* running crc and total uncompressed length */
430
431 /* skip command name */
432 argc--;
433 argv++;
434
435 /* show usage if no arguments */
436 if (argc == 0) {
437 fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n",
438 stderr);
439 return 0;
440 }
441
442 /* join gzip files on command line and write to stdout */
443 gzinit(&crc, &tot, stdout);
444 while (argc--)
445 gzcopy(*argv++, argc, &crc, &tot, stdout);
446
447 /* done */
448 return 0;
449 }