| /* gzappend -- command to append to a gzip file |
| |
| Copyright (C) 2003, 2012 Mark Adler, all rights reserved |
| version 1.2, 11 Oct 2012 |
| |
| This software is provided 'as-is', without any express or implied |
| warranty. In no event will the author be held liable for any damages |
| arising from the use of this software. |
| |
| Permission is granted to anyone to use this software for any purpose, |
| including commercial applications, and to alter it and redistribute it |
| freely, subject to the following restrictions: |
| |
| 1. The origin of this software must not be misrepresented; you must not |
| claim that you wrote the original software. If you use this software |
| in a product, an acknowledgment in the product documentation would be |
| appreciated but is not required. |
| 2. Altered source versions must be plainly marked as such, and must not be |
| misrepresented as being the original software. |
| 3. This notice may not be removed or altered from any source distribution. |
| |
| Mark Adler madler@alumni.caltech.edu |
| */ |
| |
| /* |
| * Change history: |
| * |
| * 1.0 19 Oct 2003 - First version |
| * 1.1 4 Nov 2003 - Expand and clarify some comments and notes |
| * - Add version and copyright to help |
| * - Send help to stdout instead of stderr |
| * - Add some preemptive typecasts |
| * - Add L to constants in lseek() calls |
| * - Remove some debugging information in error messages |
| * - Use new data_type definition for zlib 1.2.1 |
| * - Simplfy and unify file operations |
| * - Finish off gzip file in gztack() |
| * - Use deflatePrime() instead of adding empty blocks |
| * - Keep gzip file clean on appended file read errors |
| * - Use in-place rotate instead of auxiliary buffer |
| * (Why you ask? Because it was fun to write!) |
| * 1.2 11 Oct 2012 - Fix for proper z_const usage |
| * - Check for input buffer malloc failure |
| */ |
| |
| /* |
| gzappend takes a gzip file and appends to it, compressing files from the |
| command line or data from stdin. The gzip file is written to directly, to |
| avoid copying that file, in case it's large. Note that this results in the |
| unfriendly behavior that if gzappend fails, the gzip file is corrupted. |
| |
| This program was written to illustrate the use of the new Z_BLOCK option of |
| zlib 1.2.x's inflate() function. This option returns from inflate() at each |
| block boundary to facilitate locating and modifying the last block bit at |
| the start of the final deflate block. Also whether using Z_BLOCK or not, |
| another required feature of zlib 1.2.x is that inflate() now provides the |
| number of unusued bits in the last input byte used. gzappend will not work |
| with versions of zlib earlier than 1.2.1. |
| |
| gzappend first decompresses the gzip file internally, discarding all but |
| the last 32K of uncompressed data, and noting the location of the last block |
| bit and the number of unused bits in the last byte of the compressed data. |
| The gzip trailer containing the CRC-32 and length of the uncompressed data |
| is verified. This trailer will be later overwritten. |
| |
| Then the last block bit is cleared by seeking back in the file and rewriting |
| the byte that contains it. Seeking forward, the last byte of the compressed |
| data is saved along with the number of unused bits to initialize deflate. |
| |
| A deflate process is initialized, using the last 32K of the uncompressed |
| data from the gzip file to initialize the dictionary. If the total |
| uncompressed data was less than 32K, then all of it is used to initialize |
| the dictionary. The deflate output bit buffer is also initialized with the |
| last bits from the original deflate stream. From here on, the data to |
| append is simply compressed using deflate, and written to the gzip file. |
| When that is complete, the new CRC-32 and uncompressed length are written |
| as the trailer of the gzip file. |
| */ |
| |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <fcntl.h> |
| #include <unistd.h> |
| #include "zlib.h" |
| |
| #define local static |
| #define LGCHUNK 14 |
| #define CHUNK (1U << LGCHUNK) |
| #define DSIZE 32768U |
| |
| /* print an error message and terminate with extreme prejudice */ |
| local void bye(char *msg1, char *msg2) |
| { |
| fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2); |
| exit(1); |
| } |
| |
| /* return the greatest common divisor of a and b using Euclid's algorithm, |
| modified to be fast when one argument much greater than the other, and |
| coded to avoid unnecessary swapping */ |
| local unsigned gcd(unsigned a, unsigned b) |
| { |
| unsigned c; |
| |
| while (a && b) |
| if (a > b) { |
| c = b; |
| while (a - c >= c) |
| c <<= 1; |
| a -= c; |
| } |
| else { |
| c = a; |
| while (b - c >= c) |
| c <<= 1; |
| b -= c; |
| } |
| return a + b; |
| } |
| |
| /* rotate list[0..len-1] left by rot positions, in place */ |
| local void rotate(unsigned char *list, unsigned len, unsigned rot) |
| { |
| unsigned char tmp; |
| unsigned cycles; |
| unsigned char *start, *last, *to, *from; |
| |
| /* normalize rot and handle degenerate cases */ |
| if (len < 2) return; |
| if (rot >= len) rot %= len; |
| if (rot == 0) return; |
| |
| /* pointer to last entry in list */ |
| last = list + (len - 1); |
| |
| /* do simple left shift by one */ |
| if (rot == 1) { |
| tmp = *list; |
| memcpy(list, list + 1, len - 1); |
| *last = tmp; |
| return; |
| } |
| |
| /* do simple right shift by one */ |
| if (rot == len - 1) { |
| tmp = *last; |
| memmove(list + 1, list, len - 1); |
| *list = tmp; |
| return; |
| } |
| |
| /* otherwise do rotate as a set of cycles in place */ |
| cycles = gcd(len, rot); /* number of cycles */ |
| do { |
| start = from = list + cycles; /* start index is arbitrary */ |
| tmp = *from; /* save entry to be overwritten */ |
| for (;;) { |
| to = from; /* next step in cycle */ |
| from += rot; /* go right rot positions */ |
| if (from > last) from -= len; /* (pointer better not wrap) */ |
| if (from == start) break; /* all but one shifted */ |
| *to = *from; /* shift left */ |
| } |
| *to = tmp; /* complete the circle */ |
| } while (--cycles); |
| } |
| |
| /* structure for gzip file read operations */ |
| typedef struct { |
| int fd; /* file descriptor */ |
| int size; /* 1 << size is bytes in buf */ |
| unsigned left; /* bytes available at next */ |
| unsigned char *buf; /* buffer */ |
| z_const unsigned char *next; /* next byte in buffer */ |
| char *name; /* file name for error messages */ |
| } file; |
| |
| /* reload buffer */ |
| local int readin(file *in) |
| { |
| int len; |
| |
| len = read(in->fd, in->buf, 1 << in->size); |
| if (len == -1) bye("error reading ", in->name); |
| in->left = (unsigned)len; |
| in->next = in->buf; |
| return len; |
| } |
| |
| /* read from file in, exit if end-of-file */ |
| local int readmore(file *in) |
| { |
| if (readin(in) == 0) bye("unexpected end of ", in->name); |
| return 0; |
| } |
| |
| #define read1(in) (in->left == 0 ? readmore(in) : 0, \ |
| in->left--, *(in->next)++) |
| |
| /* skip over n bytes of in */ |
| local void skip(file *in, unsigned n) |
| { |
| unsigned bypass; |
| |
| if (n > in->left) { |
| n -= in->left; |
| bypass = n & ~((1U << in->size) - 1); |
| if (bypass) { |
| if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1) |
| bye("seeking ", in->name); |
| n -= bypass; |
| } |
| readmore(in); |
| if (n > in->left) |
| bye("unexpected end of ", in->name); |
| } |
| in->left -= n; |
| in->next += n; |
| } |
| |
| /* read a four-byte unsigned integer, little-endian, from in */ |
| unsigned long read4(file *in) |
| { |
| unsigned long val; |
| |
| val = read1(in); |
| val += (unsigned)read1(in) << 8; |
| val += (unsigned long)read1(in) << 16; |
| val += (unsigned long)read1(in) << 24; |
| return val; |
| } |
| |
| /* skip over gzip header */ |
| local void gzheader(file *in) |
| { |
| int flags; |
| unsigned n; |
| |
| if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file"); |
| if (read1(in) != 8) bye("unknown compression method in", in->name); |
| flags = read1(in); |
| if (flags & 0xe0) bye("unknown header flags set in", in->name); |
| skip(in, 6); |
| if (flags & 4) { |
| n = read1(in); |
| n += (unsigned)(read1(in)) << 8; |
| skip(in, n); |
| } |
| if (flags & 8) while (read1(in) != 0) ; |
| if (flags & 16) while (read1(in) != 0) ; |
| if (flags & 2) skip(in, 2); |
| } |
| |
| /* decompress gzip file "name", return strm with a deflate stream ready to |
| continue compression of the data in the gzip file, and return a file |
| descriptor pointing to where to write the compressed data -- the deflate |
| stream is initialized to compress using level "level" */ |
| local int gzscan(char *name, z_stream *strm, int level) |
| { |
| int ret, lastbit, left, full; |
| unsigned have; |
| unsigned long crc, tot; |
| unsigned char *window; |
| off_t lastoff, end; |
| file gz; |
| |
| /* open gzip file */ |
| gz.name = name; |
| gz.fd = open(name, O_RDWR, 0); |
| if (gz.fd == -1) bye("cannot open ", name); |
| gz.buf = malloc(CHUNK); |
| if (gz.buf == NULL) bye("out of memory", ""); |
| gz.size = LGCHUNK; |
| gz.left = 0; |
| |
| /* skip gzip header */ |
| gzheader(&gz); |
| |
| /* prepare to decompress */ |
| window = malloc(DSIZE); |
| if (window == NULL) bye("out of memory", ""); |
| strm->zalloc = Z_NULL; |
| strm->zfree = Z_NULL; |
| strm->opaque = Z_NULL; |
| ret = inflateInit2(strm, -15); |
| if (ret != Z_OK) bye("out of memory", " or library mismatch"); |
| |
| /* decompress the deflate stream, saving append information */ |
| lastbit = 0; |
| lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; |
| left = 0; |
| strm->avail_in = gz.left; |
| strm->next_in = gz.next; |
| crc = crc32(0L, Z_NULL, 0); |
| have = full = 0; |
| do { |
| /* if needed, get more input */ |
| if (strm->avail_in == 0) { |
| readmore(&gz); |
| strm->avail_in = gz.left; |
| strm->next_in = gz.next; |
| } |
| |
| /* set up output to next available section of sliding window */ |
| strm->avail_out = DSIZE - have; |
| strm->next_out = window + have; |
| |
| /* inflate and check for errors */ |
| ret = inflate(strm, Z_BLOCK); |
| if (ret == Z_STREAM_ERROR) bye("internal stream error!", ""); |
| if (ret == Z_MEM_ERROR) bye("out of memory", ""); |
| if (ret == Z_DATA_ERROR) |
| bye("invalid compressed data--format violated in", name); |
| |
| /* update crc and sliding window pointer */ |
| crc = crc32(crc, window + have, DSIZE - have - strm->avail_out); |
| if (strm->avail_out) |
| have = DSIZE - strm->avail_out; |
| else { |
| have = 0; |
| full = 1; |
| } |
| |
| /* process end of block */ |
| if (strm->data_type & 128) { |
| if (strm->data_type & 64) |
| left = strm->data_type & 0x1f; |
| else { |
| lastbit = strm->data_type & 0x1f; |
| lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in; |
| } |
| } |
| } while (ret != Z_STREAM_END); |
| inflateEnd(strm); |
| gz.left = strm->avail_in; |
| gz.next = strm->next_in; |
| |
| /* save the location of the end of the compressed data */ |
| end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; |
| |
| /* check gzip trailer and save total for deflate */ |
| if (crc != read4(&gz)) |
| bye("invalid compressed data--crc mismatch in ", name); |
| tot = strm->total_out; |
| if ((tot & 0xffffffffUL) != read4(&gz)) |
| bye("invalid compressed data--length mismatch in", name); |
| |
| /* if not at end of file, warn */ |
| if (gz.left || readin(&gz)) |
| fprintf(stderr, |
| "gzappend warning: junk at end of gzip file overwritten\n"); |
| |
| /* clear last block bit */ |
| lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET); |
| if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); |
| *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7))); |
| lseek(gz.fd, -1L, SEEK_CUR); |
| if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name); |
| |
| /* if window wrapped, build dictionary from window by rotating */ |
| if (full) { |
| rotate(window, DSIZE, have); |
| have = DSIZE; |
| } |
| |
| /* set up deflate stream with window, crc, total_in, and leftover bits */ |
| ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); |
| if (ret != Z_OK) bye("out of memory", ""); |
| deflateSetDictionary(strm, window, have); |
| strm->adler = crc; |
| strm->total_in = tot; |
| if (left) { |
| lseek(gz.fd, --end, SEEK_SET); |
| if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); |
| deflatePrime(strm, 8 - left, *gz.buf); |
| } |
| lseek(gz.fd, end, SEEK_SET); |
| |
| /* clean up and return */ |
| free(window); |
| free(gz.buf); |
| return gz.fd; |
| } |
| |
| /* append file "name" to gzip file gd using deflate stream strm -- if last |
| is true, then finish off the deflate stream at the end */ |
| local void gztack(char *name, int gd, z_stream *strm, int last) |
| { |
| int fd, len, ret; |
| unsigned left; |
| unsigned char *in, *out; |
| |
| /* open file to compress and append */ |
| fd = 0; |
| if (name != NULL) { |
| fd = open(name, O_RDONLY, 0); |
| if (fd == -1) |
| fprintf(stderr, "gzappend warning: %s not found, skipping ...\n", |
| name); |
| } |
| |
| /* allocate buffers */ |
| in = malloc(CHUNK); |
| out = malloc(CHUNK); |
| if (in == NULL || out == NULL) bye("out of memory", ""); |
| |
| /* compress input file and append to gzip file */ |
| do { |
| /* get more input */ |
| len = read(fd, in, CHUNK); |
| if (len == -1) { |
| fprintf(stderr, |
| "gzappend warning: error reading %s, skipping rest ...\n", |
| name); |
| len = 0; |
| } |
| strm->avail_in = (unsigned)len; |
| strm->next_in = in; |
| if (len) strm->adler = crc32(strm->adler, in, (unsigned)len); |
| |
| /* compress and write all available output */ |
| do { |
| strm->avail_out = CHUNK; |
| strm->next_out = out; |
| ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH); |
| left = CHUNK - strm->avail_out; |
| while (left) { |
| len = write(gd, out + CHUNK - strm->avail_out - left, left); |
| if (len == -1) bye("writing gzip file", ""); |
| left -= (unsigned)len; |
| } |
| } while (strm->avail_out == 0 && ret != Z_STREAM_END); |
| } while (len != 0); |
| |
| /* write trailer after last entry */ |
| if (last) { |
| deflateEnd(strm); |
| out[0] = (unsigned char)(strm->adler); |
| out[1] = (unsigned char)(strm->adler >> 8); |
| out[2] = (unsigned char)(strm->adler >> 16); |
| out[3] = (unsigned char)(strm->adler >> 24); |
| out[4] = (unsigned char)(strm->total_in); |
| out[5] = (unsigned char)(strm->total_in >> 8); |
| out[6] = (unsigned char)(strm->total_in >> 16); |
| out[7] = (unsigned char)(strm->total_in >> 24); |
| len = 8; |
| do { |
| ret = write(gd, out + 8 - len, len); |
| if (ret == -1) bye("writing gzip file", ""); |
| len -= ret; |
| } while (len); |
| close(gd); |
| } |
| |
| /* clean up and return */ |
| free(out); |
| free(in); |
| if (fd > 0) close(fd); |
| } |
| |
| /* process the compression level option if present, scan the gzip file, and |
| append the specified files, or append the data from stdin if no other file |
| names are provided on the command line -- the gzip file must be writable |
| and seekable */ |
| int main(int argc, char **argv) |
| { |
| int gd, level; |
| z_stream strm; |
| |
| /* ignore command name */ |
| argc--; argv++; |
| |
| /* provide usage if no arguments */ |
| if (*argv == NULL) { |
| printf( |
| "gzappend 1.2 (11 Oct 2012) Copyright (C) 2003, 2012 Mark Adler\n" |
| ); |
| printf( |
| "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n"); |
| return 0; |
| } |
| |
| /* set compression level */ |
| level = Z_DEFAULT_COMPRESSION; |
| if (argv[0][0] == '-') { |
| if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0) |
| bye("invalid compression level", ""); |
| level = argv[0][1] - '0'; |
| if (*++argv == NULL) bye("no gzip file name after options", ""); |
| } |
| |
| /* prepare to append to gzip file */ |
| gd = gzscan(*argv++, &strm, level); |
| |
| /* append files on command line, or from stdin if none */ |
| if (*argv == NULL) |
| gztack(NULL, gd, &strm, 1); |
| else |
| do { |
| gztack(*argv, gd, &strm, argv[1] == NULL); |
| } while (*++argv != NULL); |
| return 0; |
| } |
