| /* gzjoin -- command to join gzip files into one gzip file |
| |
| Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved |
| version 1.2, 14 Aug 2012 |
| |
| This software is provided 'as-is', without any express or implied |
| warranty. In no event will the author be held liable for any damages |
| arising from the use of this software. |
| |
| Permission is granted to anyone to use this software for any purpose, |
| including commercial applications, and to alter it and redistribute it |
| freely, subject to the following restrictions: |
| |
| 1. The origin of this software must not be misrepresented; you must not |
| claim that you wrote the original software. If you use this software |
| in a product, an acknowledgment in the product documentation would be |
| appreciated but is not required. |
| 2. Altered source versions must be plainly marked as such, and must not be |
| misrepresented as being the original software. |
| 3. This notice may not be removed or altered from any source distribution. |
| |
| Mark Adler madler@alumni.caltech.edu |
| */ |
| |
| /* |
| * Change history: |
| * |
| * 1.0 11 Dec 2004 - First version |
| * 1.1 12 Jun 2005 - Changed ssize_t to long for portability |
| * 1.2 14 Aug 2012 - Clean up for z_const usage |
| */ |
| |
| /* |
| gzjoin takes one or more gzip files on the command line and writes out a |
| single gzip file that will uncompress to the concatenation of the |
| uncompressed data from the individual gzip files. gzjoin does this without |
| having to recompress any of the data and without having to calculate a new |
| crc32 for the concatenated uncompressed data. gzjoin does however have to |
| decompress all of the input data in order to find the bits in the compressed |
| data that need to be modified to concatenate the streams. |
| |
| gzjoin does not do an integrity check on the input gzip files other than |
| checking the gzip header and decompressing the compressed data. They are |
| otherwise assumed to be complete and correct. |
| |
| Each joint between gzip files removes at least 18 bytes of previous trailer |
| and subsequent header, and inserts an average of about three bytes to the |
| compressed data in order to connect the streams. The output gzip file |
| has a minimal ten-byte gzip header with no file name or modification time. |
| |
| This program was written to illustrate the use of the Z_BLOCK option of |
| inflate() and the crc32_combine() function. gzjoin will not compile with |
| versions of zlib earlier than 1.2.3. |
| */ |
| |
| #include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ |
| #include <stdlib.h> /* exit(), malloc(), free() */ |
| #include <fcntl.h> /* open() */ |
| #include <unistd.h> /* close(), read(), lseek() */ |
| #include "zlib.h" |
| /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */ |
| |
| #define local static |
| |
| /* exit with an error (return a value to allow use in an expression) */ |
| local int bail(char *why1, char *why2) |
| { |
| fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); |
| exit(1); |
| return 0; |
| } |
| |
| /* -- simple buffered file input with access to the buffer -- */ |
| |
| #define CHUNK 32768 /* must be a power of two and fit in unsigned */ |
| |
| /* bin buffered input file type */ |
| typedef struct { |
| char *name; /* name of file for error messages */ |
| int fd; /* file descriptor */ |
| unsigned left; /* bytes remaining at next */ |
| unsigned char *next; /* next byte to read */ |
| unsigned char *buf; /* allocated buffer of length CHUNK */ |
| } bin; |
| |
| /* close a buffered file and free allocated memory */ |
| local void bclose(bin *in) |
| { |
| if (in != NULL) { |
| if (in->fd != -1) |
| close(in->fd); |
| if (in->buf != NULL) |
| free(in->buf); |
| free(in); |
| } |
| } |
| |
| /* open a buffered file for input, return a pointer to type bin, or NULL on |
| failure */ |
| local bin *bopen(char *name) |
| { |
| bin *in; |
| |
| in = malloc(sizeof(bin)); |
| if (in == NULL) |
| return NULL; |
| in->buf = malloc(CHUNK); |
| in->fd = open(name, O_RDONLY, 0); |
| if (in->buf == NULL || in->fd == -1) { |
| bclose(in); |
| return NULL; |
| } |
| in->left = 0; |
| in->next = in->buf; |
| in->name = name; |
| return in; |
| } |
| |
| /* load buffer from file, return -1 on read error, 0 or 1 on success, with |
| 1 indicating that end-of-file was reached */ |
| local int bload(bin *in) |
| { |
| long len; |
| |
| if (in == NULL) |
| return -1; |
| if (in->left != 0) |
| return 0; |
| in->next = in->buf; |
| do { |
| len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); |
| if (len < 0) |
| return -1; |
| in->left += (unsigned)len; |
| } while (len != 0 && in->left < CHUNK); |
| return len == 0 ? 1 : 0; |
| } |
| |
| /* get a byte from the file, bail if end of file */ |
| #define bget(in) (in->left ? 0 : bload(in), \ |
| in->left ? (in->left--, *(in->next)++) : \ |
| bail("unexpected end of file on ", in->name)) |
| |
| /* get a four-byte little-endian unsigned integer from file */ |
| local unsigned long bget4(bin *in) |
| { |
| unsigned long val; |
| |
| val = bget(in); |
| val += (unsigned long)(bget(in)) << 8; |
| val += (unsigned long)(bget(in)) << 16; |
| val += (unsigned long)(bget(in)) << 24; |
| return val; |
| } |
| |
| /* skip bytes in file */ |
| local void bskip(bin *in, unsigned skip) |
| { |
| /* check pointer */ |
| if (in == NULL) |
| return; |
| |
| /* easy case -- skip bytes in buffer */ |
| if (skip <= in->left) { |
| in->left -= skip; |
| in->next += skip; |
| return; |
| } |
| |
| /* skip what's in buffer, discard buffer contents */ |
| skip -= in->left; |
| in->left = 0; |
| |
| /* seek past multiples of CHUNK bytes */ |
| if (skip > CHUNK) { |
| unsigned left; |
| |
| left = skip & (CHUNK - 1); |
| if (left == 0) { |
| /* exact number of chunks: seek all the way minus one byte to check |
| for end-of-file with a read */ |
| lseek(in->fd, skip - 1, SEEK_CUR); |
| if (read(in->fd, in->buf, 1) != 1) |
| bail("unexpected end of file on ", in->name); |
| return; |
| } |
| |
| /* skip the integral chunks, update skip with remainder */ |
| lseek(in->fd, skip - left, SEEK_CUR); |
| skip = left; |
| } |
| |
| /* read more input and skip remainder */ |
| bload(in); |
| if (skip > in->left) |
| bail("unexpected end of file on ", in->name); |
| in->left -= skip; |
| in->next += skip; |
| } |
| |
| /* -- end of buffered input functions -- */ |
| |
| /* skip the gzip header from file in */ |
| local void gzhead(bin *in) |
| { |
| int flags; |
| |
| /* verify gzip magic header and compression method */ |
| if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) |
| bail(in->name, " is not a valid gzip file"); |
| |
| /* get and verify flags */ |
| flags = bget(in); |
| if ((flags & 0xe0) != 0) |
| bail("unknown reserved bits set in ", in->name); |
| |
| /* skip modification time, extra flags, and os */ |
| bskip(in, 6); |
| |
| /* skip extra field if present */ |
| if (flags & 4) { |
| unsigned len; |
| |
| len = bget(in); |
| len += (unsigned)(bget(in)) << 8; |
| bskip(in, len); |
| } |
| |
| /* skip file name if present */ |
| if (flags & 8) |
| while (bget(in) != 0) |
| ; |
| |
| /* skip comment if present */ |
| if (flags & 16) |
| while (bget(in) != 0) |
| ; |
| |
| /* skip header crc if present */ |
| if (flags & 2) |
| bskip(in, 2); |
| } |
| |
| /* write a four-byte little-endian unsigned integer to out */ |
| local void put4(unsigned long val, FILE *out) |
| { |
| putc(val & 0xff, out); |
| putc((val >> 8) & 0xff, out); |
| putc((val >> 16) & 0xff, out); |
| putc((val >> 24) & 0xff, out); |
| } |
| |
| /* Load up zlib stream from buffered input, bail if end of file */ |
| local void zpull(z_streamp strm, bin *in) |
| { |
| if (in->left == 0) |
| bload(in); |
| if (in->left == 0) |
| bail("unexpected end of file on ", in->name); |
| strm->avail_in = in->left; |
| strm->next_in = in->next; |
| } |
| |
| /* Write header for gzip file to out and initialize trailer. */ |
| local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) |
| { |
| fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); |
| *crc = crc32(0L, Z_NULL, 0); |
| *tot = 0; |
| } |
| |
| /* Copy the compressed data from name, zeroing the last block bit of the last |
| block if clr is true, and adding empty blocks as needed to get to a byte |
| boundary. If clr is false, then the last block becomes the last block of |
| the output, and the gzip trailer is written. crc and tot maintains the |
| crc and length (modulo 2^32) of the output for the trailer. The resulting |
| gzip file is written to out. gzinit() must be called before the first call |
| of gzcopy() to write the gzip header and to initialize crc and tot. */ |
| local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, |
| FILE *out) |
| { |
| int ret; /* return value from zlib functions */ |
| int pos; /* where the "last block" bit is in byte */ |
| int last; /* true if processing the last block */ |
| bin *in; /* buffered input file */ |
| unsigned char *start; /* start of compressed data in buffer */ |
| unsigned char *junk; /* buffer for uncompressed data -- discarded */ |
| z_off_t len; /* length of uncompressed data (support > 4 GB) */ |
| z_stream strm; /* zlib inflate stream */ |
| |
| /* open gzip file and skip header */ |
| in = bopen(name); |
| if (in == NULL) |
| bail("could not open ", name); |
| gzhead(in); |
| |
| /* allocate buffer for uncompressed data and initialize raw inflate |
| stream */ |
| junk = malloc(CHUNK); |
| strm.zalloc = Z_NULL; |
| strm.zfree = Z_NULL; |
| strm.opaque = Z_NULL; |
| strm.avail_in = 0; |
| strm.next_in = Z_NULL; |
| ret = inflateInit2(&strm, -15); |
| if (junk == NULL || ret != Z_OK) |
| bail("out of memory", ""); |
| |
| /* inflate and copy compressed data, clear last-block bit if requested */ |
| len = 0; |
| zpull(&strm, in); |
| start = in->next; |
| last = start[0] & 1; |
| if (last && clr) |
| start[0] &= ~1; |
| strm.avail_out = 0; |
| for (;;) { |
| /* if input used and output done, write used input and get more */ |
| if (strm.avail_in == 0 && strm.avail_out != 0) { |
| fwrite(start, 1, strm.next_in - start, out); |
| start = in->buf; |
| in->left = 0; |
| zpull(&strm, in); |
| } |
| |
| /* decompress -- return early when end-of-block reached */ |
| strm.avail_out = CHUNK; |
| strm.next_out = junk; |
| ret = inflate(&strm, Z_BLOCK); |
| switch (ret) { |
| case Z_MEM_ERROR: |
| bail("out of memory", ""); |
| case Z_DATA_ERROR: |
| bail("invalid compressed data in ", in->name); |
| } |
| |
| /* update length of uncompressed data */ |
| len += CHUNK - strm.avail_out; |
| |
| /* check for block boundary (only get this when block copied out) */ |
| if (strm.data_type & 128) { |
| /* if that was the last block, then done */ |
| if (last) |
| break; |
| |
| /* number of unused bits in last byte */ |
| pos = strm.data_type & 7; |
| |
| /* find the next last-block bit */ |
| if (pos != 0) { |
| /* next last-block bit is in last used byte */ |
| pos = 0x100 >> pos; |
| last = strm.next_in[-1] & pos; |
| if (last && clr) |
| in->buf[strm.next_in - in->buf - 1] &= ~pos; |
| } |
| else { |
| /* next last-block bit is in next unused byte */ |
| if (strm.avail_in == 0) { |
| /* don't have that byte yet -- get it */ |
| fwrite(start, 1, strm.next_in - start, out); |
| start = in->buf; |
| in->left = 0; |
| zpull(&strm, in); |
| } |
| last = strm.next_in[0] & 1; |
| if (last && clr) |
| in->buf[strm.next_in - in->buf] &= ~1; |
| } |
| } |
| } |
| |
| /* update buffer with unused input */ |
| in->left = strm.avail_in; |
| in->next = in->buf + (strm.next_in - in->buf); |
| |
| /* copy used input, write empty blocks to get to byte boundary */ |
| pos = strm.data_type & 7; |
| fwrite(start, 1, in->next - start - 1, out); |
| last = in->next[-1]; |
| if (pos == 0 || !clr) |
| /* already at byte boundary, or last file: write last byte */ |
| putc(last, out); |
| else { |
| /* append empty blocks to last byte */ |
| last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ |
| if (pos & 1) { |
| /* odd -- append an empty stored block */ |
| putc(last, out); |
| if (pos == 1) |
| putc(0, out); /* two more bits in block header */ |
| fwrite("\0\0\xff\xff", 1, 4, out); |
| } |
| else { |
| /* even -- append 1, 2, or 3 empty fixed blocks */ |
| switch (pos) { |
| case 6: |
| putc(last | 8, out); |
| last = 0; |
| case 4: |
| putc(last | 0x20, out); |
| last = 0; |
| case 2: |
| putc(last | 0x80, out); |
| putc(0, out); |
| } |
| } |
| } |
| |
| /* update crc and tot */ |
| *crc = crc32_combine(*crc, bget4(in), len); |
| *tot += (unsigned long)len; |
| |
| /* clean up */ |
| inflateEnd(&strm); |
| free(junk); |
| bclose(in); |
| |
| /* write trailer if this is the last gzip file */ |
| if (!clr) { |
| put4(*crc, out); |
| put4(*tot, out); |
| } |
| } |
| |
| /* join the gzip files on the command line, write result to stdout */ |
| int main(int argc, char **argv) |
| { |
| unsigned long crc, tot; /* running crc and total uncompressed length */ |
| |
| /* skip command name */ |
| argc--; |
| argv++; |
| |
| /* show usage if no arguments */ |
| if (argc == 0) { |
| fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", |
| stderr); |
| return 0; |
| } |
| |
| /* join gzip files on command line and write to stdout */ |
| gzinit(&crc, &tot, stdout); |
| while (argc--) |
| gzcopy(*argv++, argc, &crc, &tot, stdout); |
| |
| /* done */ |
| return 0; |
| } |
