From d177bb1fb75b129721219d9c40ff910840db9785 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 13:45:03 -0400 Subject: Move bundled zopfli sources into a new zopfli/ subdirectory. Separating source files by origin (and license) makes it easy for users who want to build with an external zopfli library to remove the bundled sources, ensuring they are not used. Unbundling libraries is mandatory for packaging in many popular Linux distributions, and can be useful for other applications as well. Additional changes will be needed to make it easy to select an external zopfli library. --- Makefile | 6 +- blocksplitter.c | 342 ------------------- blocksplitter.h | 77 ----- cache.c | 119 ------- cache.h | 66 ---- deflate.c | 866 ------------------------------------------------ deflate.h | 86 ----- gzip_container.c | 117 ------- gzip_container.h | 50 --- hash.c | 135 -------- hash.h | 70 ---- katajainen.c | 251 -------------- katajainen.h | 42 --- lz77.c | 482 --------------------------- lz77.h | 129 -------- squeeze.c | 546 ------------------------------ squeeze.h | 60 ---- tree.c | 101 ------ tree.h | 51 --- util.c | 213 ------------ util.h | 175 ---------- zlib_container.c | 79 ----- zlib_container.h | 50 --- zopfli.h | 97 ------ zopfli/blocksplitter.c | 342 +++++++++++++++++++ zopfli/blocksplitter.h | 77 +++++ zopfli/cache.c | 119 +++++++ zopfli/cache.h | 66 ++++ zopfli/deflate.c | 866 ++++++++++++++++++++++++++++++++++++++++++++++++ zopfli/deflate.h | 86 +++++ zopfli/gzip_container.c | 117 +++++++ zopfli/gzip_container.h | 50 +++ zopfli/hash.c | 135 ++++++++ zopfli/hash.h | 70 ++++ zopfli/katajainen.c | 251 ++++++++++++++ zopfli/katajainen.h | 42 +++ zopfli/lz77.c | 482 +++++++++++++++++++++++++++ zopfli/lz77.h | 129 ++++++++ zopfli/squeeze.c | 546 ++++++++++++++++++++++++++++++ zopfli/squeeze.h | 60 ++++ zopfli/tree.c | 101 ++++++ zopfli/tree.h | 51 +++ zopfli/util.c | 213 ++++++++++++ zopfli/util.h | 175 ++++++++++ zopfli/zlib_container.c | 79 +++++ zopfli/zlib_container.h | 50 +++ zopfli/zopfli.h | 97 ++++++ zopfli/zopfli_bin.c | 203 ++++++++++++ zopfli/zopfli_lib.c | 42 +++ zopfli_bin.c | 203 ------------ zopfli_lib.c | 42 --- 51 files changed, 4452 insertions(+), 4452 deletions(-) delete mode 100644 blocksplitter.c delete mode 100644 blocksplitter.h delete mode 100644 cache.c delete mode 100644 cache.h delete mode 100644 deflate.c delete mode 100644 deflate.h delete mode 100644 gzip_container.c delete mode 100644 gzip_container.h delete mode 100644 hash.c delete mode 100644 hash.h delete mode 100644 katajainen.c delete mode 100644 katajainen.h delete mode 100644 lz77.c delete mode 100644 lz77.h delete mode 100644 squeeze.c delete mode 100644 squeeze.h delete mode 100644 tree.c delete mode 100644 tree.h delete mode 100644 util.c delete mode 100644 util.h delete mode 100644 zlib_container.c delete mode 100644 zlib_container.h delete mode 100644 zopfli.h create mode 100644 zopfli/blocksplitter.c create mode 100644 zopfli/blocksplitter.h create mode 100644 zopfli/cache.c create mode 100644 zopfli/cache.h create mode 100644 zopfli/deflate.c create mode 100644 zopfli/deflate.h create mode 100644 zopfli/gzip_container.c create mode 100644 zopfli/gzip_container.h create mode 100644 zopfli/hash.c create mode 100644 zopfli/hash.h create mode 100644 zopfli/katajainen.c create mode 100644 zopfli/katajainen.h create mode 100644 zopfli/lz77.c create mode 100644 zopfli/lz77.h create mode 100644 zopfli/squeeze.c create mode 100644 zopfli/squeeze.h create mode 100644 zopfli/tree.c create mode 100644 zopfli/tree.h create mode 100644 zopfli/util.c create mode 100644 zopfli/util.h create mode 100644 zopfli/zlib_container.c create mode 100644 zopfli/zlib_container.h create mode 100644 zopfli/zopfli.h create mode 100644 zopfli/zopfli_bin.c create mode 100644 zopfli/zopfli_lib.c delete mode 100644 zopfli_bin.c delete mode 100644 zopfli_lib.c diff --git a/Makefile b/Makefile index cb2577a..2f644f2 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ -FILES=blocksplitter.c cache.c deflate.c gzip_container.c hash.c katajainen.c lz77.c squeeze.c tree.c util.c zlib_container.c zopfli_lib.c woff.c +FILES=zopfli/blocksplitter.c zopfli/cache.c zopfli/deflate.c zopfli/gzip_container.c zopfli/hash.c zopfli/katajainen.c zopfli/lz77.c zopfli/squeeze.c zopfli/tree.c zopfli/util.c zopfli/zlib_container.c zopfli/zopfli_lib.c woff.c all: sfnt2woff-zopfli woff2sfnt-zopfli sfnt2woff-zopfli: sfnt2woff.c $(FILES) Makefile - $(CC) $(LDFLAGS) $(FILES) $< -o $@ -lz -lm + $(CC) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm woff2sfnt-zopfli: woff2sfnt.c $(FILES) Makefile - $(CC) $(LDFLAGS) $(FILES) $< -o $@ -lz -lm + $(CC) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm clean: $(RM) -r *.o *.dSYM sfnt2woff-zopfli woff2sfnt-zopfli *.gch *.out diff --git a/blocksplitter.c b/blocksplitter.c deleted file mode 100644 index 68f5ff3..0000000 --- a/blocksplitter.c +++ /dev/null @@ -1,342 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "blocksplitter.h" - -#include -#include -#include - -#include "deflate.h" -#include "lz77.h" -#include "squeeze.h" -#include "tree.h" -#include "util.h" - -/* -The "f" for the FindMinimum function below. -i: the current parameter of f(i) -context: for your implementation -*/ -typedef double FindMinimumFun(size_t i, void* context); - -/* -Finds minimum of function f(i) where is is of type size_t, f(i) is of type -double, i is in range start-end (excluding end). -*/ -static size_t FindMinimum(FindMinimumFun f, void* context, - size_t start, size_t end) { - if (end - start < 1024) { - double best = ZOPFLI_LARGE_FLOAT; - size_t result = start; - size_t i; - for (i = start; i < end; i++) { - double v = f(i, context); - if (v < best) { - best = v; - result = i; - } - } - return result; - } else { - /* Try to find minimum faster by recursively checking multiple points. */ -#define NUM 9 /* Good value: 9. */ - size_t i; - size_t p[NUM]; - double vp[NUM]; - size_t besti; - double best; - double lastbest = ZOPFLI_LARGE_FLOAT; - size_t pos = start; - - for (;;) { - if (end - start <= NUM) break; - - for (i = 0; i < NUM; i++) { - p[i] = start + (i + 1) * ((end - start) / (NUM + 1)); - vp[i] = f(p[i], context); - } - besti = 0; - best = vp[0]; - for (i = 1; i < NUM; i++) { - if (vp[i] < best) { - best = vp[i]; - besti = i; - } - } - if (best > lastbest) break; - - start = besti == 0 ? start : p[besti - 1]; - end = besti == NUM - 1 ? end : p[besti + 1]; - - pos = p[besti]; - lastbest = best; - } - return pos; -#undef NUM - } -} - -/* -Returns estimated cost of a block in bits. It includes the size to encode the -tree and the size to encode all literal, length and distance symbols and their -extra bits. - -litlens: lz77 lit/lengths -dists: ll77 distances -lstart: start of block -lend: end of block (not inclusive) -*/ -static double EstimateCost(const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend) { - return ZopfliCalculateBlockSize(litlens, dists, lstart, lend, 2); -} - -typedef struct SplitCostContext { - const unsigned short* litlens; - const unsigned short* dists; - size_t llsize; - size_t start; - size_t end; -} SplitCostContext; - - -/* -Gets the cost which is the sum of the cost of the left and the right section -of the data. -type: FindMinimumFun -*/ -static double SplitCost(size_t i, void* context) { - SplitCostContext* c = (SplitCostContext*)context; - return EstimateCost(c->litlens, c->dists, c->start, i) + - EstimateCost(c->litlens, c->dists, i, c->end); -} - -static void AddSorted(size_t value, size_t** out, size_t* outsize) { - size_t i; - ZOPFLI_APPEND_DATA(value, out, outsize); - for (i = 0; i + 1 < *outsize; i++) { - if ((*out)[i] > value) { - size_t j; - for (j = *outsize - 1; j > i; j--) { - (*out)[j] = (*out)[j - 1]; - } - (*out)[i] = value; - break; - } - } -} - -/* -Prints the block split points as decimal and hex values in the terminal. -*/ -static void PrintBlockSplitPoints(const unsigned short* litlens, - const unsigned short* dists, - size_t llsize, const size_t* lz77splitpoints, - size_t nlz77points) { - size_t* splitpoints = 0; - size_t npoints = 0; - size_t i; - /* The input is given as lz77 indices, but we want to see the uncompressed - index values. */ - size_t pos = 0; - if (nlz77points > 0) { - for (i = 0; i < llsize; i++) { - size_t length = dists[i] == 0 ? 1 : litlens[i]; - if (lz77splitpoints[npoints] == i) { - ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints); - if (npoints == nlz77points) break; - } - pos += length; - } - } - assert(npoints == nlz77points); - - fprintf(stderr, "block split points: "); - for (i = 0; i < npoints; i++) { - fprintf(stderr, "%d ", (int)splitpoints[i]); - } - fprintf(stderr, "(hex:"); - for (i = 0; i < npoints; i++) { - fprintf(stderr, " %x", (int)splitpoints[i]); - } - fprintf(stderr, ")\n"); - - free(splitpoints); -} - -/* -Finds next block to try to split, the largest of the available ones. -The largest is chosen to make sure that if only a limited amount of blocks is -requested, their sizes are spread evenly. -llsize: the size of the LL77 data, which is the size of the done array here. -done: array indicating which blocks starting at that position are no longer - splittable (splitting them increases rather than decreases cost). -splitpoints: the splitpoints found so far. -npoints: the amount of splitpoints found so far. -lstart: output variable, giving start of block. -lend: output variable, giving end of block. -returns 1 if a block was found, 0 if no block found (all are done). -*/ -static int FindLargestSplittableBlock( - size_t llsize, const unsigned char* done, - const size_t* splitpoints, size_t npoints, - size_t* lstart, size_t* lend) { - size_t longest = 0; - int found = 0; - size_t i; - for (i = 0; i <= npoints; i++) { - size_t start = i == 0 ? 0 : splitpoints[i - 1]; - size_t end = i == npoints ? llsize - 1 : splitpoints[i]; - if (!done[start] && end - start > longest) { - *lstart = start; - *lend = end; - found = 1; - longest = end - start; - } - } - return found; -} - -void ZopfliBlockSplitLZ77(const ZopfliOptions* options, - const unsigned short* litlens, - const unsigned short* dists, - size_t llsize, size_t maxblocks, - size_t** splitpoints, size_t* npoints) { - size_t lstart, lend; - size_t i; - size_t llpos = 0; - size_t numblocks = 1; - unsigned char* done; - double splitcost, origcost; - - if (llsize < 10) return; /* This code fails on tiny files. */ - - done = (unsigned char*)malloc(llsize); - if (!done) exit(-1); /* Allocation failed. */ - for (i = 0; i < llsize; i++) done[i] = 0; - - lstart = 0; - lend = llsize; - for (;;) { - SplitCostContext c; - - if (maxblocks > 0 && numblocks >= maxblocks) { - break; - } - - c.litlens = litlens; - c.dists = dists; - c.llsize = llsize; - c.start = lstart; - c.end = lend; - assert(lstart < lend); - llpos = FindMinimum(SplitCost, &c, lstart + 1, lend); - - assert(llpos > lstart); - assert(llpos < lend); - - splitcost = EstimateCost(litlens, dists, lstart, llpos) + - EstimateCost(litlens, dists, llpos, lend); - origcost = EstimateCost(litlens, dists, lstart, lend); - - if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) { - done[lstart] = 1; - } else { - AddSorted(llpos, splitpoints, npoints); - numblocks++; - } - - if (!FindLargestSplittableBlock( - llsize, done, *splitpoints, *npoints, &lstart, &lend)) { - break; /* No further split will probably reduce compression. */ - } - - if (lend - lstart < 10) { - break; - } - } - - if (options->verbose) { - PrintBlockSplitPoints(litlens, dists, llsize, *splitpoints, *npoints); - } - - free(done); -} - -void ZopfliBlockSplit(const ZopfliOptions* options, - const unsigned char* in, size_t instart, size_t inend, - size_t maxblocks, size_t** splitpoints, size_t* npoints) { - size_t pos = 0; - size_t i; - ZopfliBlockState s; - size_t* lz77splitpoints = 0; - size_t nlz77points = 0; - ZopfliLZ77Store store; - - ZopfliInitLZ77Store(&store); - - s.options = options; - s.blockstart = instart; - s.blockend = inend; -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - s.lmc = 0; -#endif - - *npoints = 0; - *splitpoints = 0; - - /* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal - results in better blocks. */ - ZopfliLZ77Greedy(&s, in, instart, inend, &store); - - ZopfliBlockSplitLZ77(options, - store.litlens, store.dists, store.size, maxblocks, - &lz77splitpoints, &nlz77points); - - /* Convert LZ77 positions to positions in the uncompressed input. */ - pos = instart; - if (nlz77points > 0) { - for (i = 0; i < store.size; i++) { - size_t length = store.dists[i] == 0 ? 1 : store.litlens[i]; - if (lz77splitpoints[*npoints] == i) { - ZOPFLI_APPEND_DATA(pos, splitpoints, npoints); - if (*npoints == nlz77points) break; - } - pos += length; - } - } - assert(*npoints == nlz77points); - - free(lz77splitpoints); - ZopfliCleanLZ77Store(&store); -} - -void ZopfliBlockSplitSimple(const unsigned char* in, - size_t instart, size_t inend, - size_t blocksize, - size_t** splitpoints, size_t* npoints) { - size_t i = instart; - while (i < inend) { - ZOPFLI_APPEND_DATA(i, splitpoints, npoints); - i += blocksize; - } - (void)in; -} diff --git a/blocksplitter.h b/blocksplitter.h deleted file mode 100644 index 6791702..0000000 --- a/blocksplitter.h +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -Functions to choose good boundaries for block splitting. Deflate allows encoding -the data in multiple blocks, with a separate Huffman tree for each block. The -Huffman tree itself requires some bytes to encode, so by choosing certain -blocks, you can either hurt, or enhance compression. These functions choose good -ones that enhance it. -*/ - -#ifndef ZOPFLI_BLOCKSPLITTER_H_ -#define ZOPFLI_BLOCKSPLITTER_H_ - -#include - -#include "zopfli.h" - - -/* -Does blocksplitting on LZ77 data. -The output splitpoints are indices in the LZ77 data. -litlens: lz77 lit/lengths -dists: lz77 distances -llsize: size of litlens and dists -maxblocks: set a limit to the amount of blocks. Set to 0 to mean no limit. -*/ -void ZopfliBlockSplitLZ77(const ZopfliOptions* options, - const unsigned short* litlens, - const unsigned short* dists, - size_t llsize, size_t maxblocks, - size_t** splitpoints, size_t* npoints); - -/* -Does blocksplitting on uncompressed data. -The output splitpoints are indices in the uncompressed bytes. - -options: general program options. -in: uncompressed input data -instart: where to start splitting -inend: where to end splitting (not inclusive) -maxblocks: maximum amount of blocks to split into, or 0 for no limit -splitpoints: dynamic array to put the resulting split point coordinates into. - The coordinates are indices in the input array. -npoints: pointer to amount of splitpoints, for the dynamic array. The amount of - blocks is the amount of splitpoitns + 1. -*/ -void ZopfliBlockSplit(const ZopfliOptions* options, - const unsigned char* in, size_t instart, size_t inend, - size_t maxblocks, size_t** splitpoints, size_t* npoints); - -/* -Divides the input into equal blocks, does not even take LZ77 lengths into -account. -*/ -void ZopfliBlockSplitSimple(const unsigned char* in, - size_t instart, size_t inend, - size_t blocksize, - size_t** splitpoints, size_t* npoints); - -#endif /* ZOPFLI_BLOCKSPLITTER_H_ */ diff --git a/cache.c b/cache.c deleted file mode 100644 index 88a49ac..0000000 --- a/cache.c +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "cache.h" - -#include -#include -#include - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - -void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc) { - size_t i; - lmc->length = (unsigned short*)malloc(sizeof(unsigned short) * blocksize); - lmc->dist = (unsigned short*)malloc(sizeof(unsigned short) * blocksize); - /* Rather large amount of memory. */ - lmc->sublen = (unsigned char*)malloc(ZOPFLI_CACHE_LENGTH * 3 * blocksize); - - /* length > 0 and dist 0 is invalid combination, which indicates on purpose - that this cache value is not filled in yet. */ - for (i = 0; i < blocksize; i++) lmc->length[i] = 1; - for (i = 0; i < blocksize; i++) lmc->dist[i] = 0; - for (i = 0; i < ZOPFLI_CACHE_LENGTH * blocksize * 3; i++) lmc->sublen[i] = 0; -} - -void ZopfliCleanCache(ZopfliLongestMatchCache* lmc) { - free(lmc->length); - free(lmc->dist); - free(lmc->sublen); -} - -void ZopfliSublenToCache(const unsigned short* sublen, - size_t pos, size_t length, - ZopfliLongestMatchCache* lmc) { - size_t i; - size_t j = 0; - unsigned bestlength = 0; - unsigned char* cache; - -#if ZOPFLI_CACHE_LENGTH == 0 - return; -#endif - - cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3]; - if (length < 3) return; - for (i = 3; i <= length; i++) { - if (i == length || sublen[i] != sublen[i + 1]) { - cache[j * 3] = i - 3; - cache[j * 3 + 1] = sublen[i] % 256; - cache[j * 3 + 2] = (sublen[i] >> 8) % 256; - bestlength = i; - j++; - if (j >= ZOPFLI_CACHE_LENGTH) break; - } - } - if (j < ZOPFLI_CACHE_LENGTH) { - assert(bestlength == length); - cache[(ZOPFLI_CACHE_LENGTH - 1) * 3] = bestlength - 3; - } else { - assert(bestlength <= length); - } - assert(bestlength == ZopfliMaxCachedSublen(lmc, pos, length)); -} - -void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc, - size_t pos, size_t length, - unsigned short* sublen) { - size_t i, j; - unsigned maxlength = ZopfliMaxCachedSublen(lmc, pos, length); - unsigned prevlength = 0; - unsigned char* cache; -#if ZOPFLI_CACHE_LENGTH == 0 - return; -#endif - if (length < 3) return; - cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3]; - for (j = 0; j < ZOPFLI_CACHE_LENGTH; j++) { - unsigned length = cache[j * 3] + 3; - unsigned dist = cache[j * 3 + 1] + 256 * cache[j * 3 + 2]; - for (i = prevlength; i <= length; i++) { - sublen[i] = dist; - } - if (length == maxlength) break; - prevlength = length + 1; - } -} - -/* -Returns the length up to which could be stored in the cache. -*/ -unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc, - size_t pos, size_t length) { - unsigned char* cache; -#if ZOPFLI_CACHE_LENGTH == 0 - return 0; -#endif - cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3]; - (void)length; - if (cache[1] == 0 && cache[2] == 0) return 0; /* No sublen cached. */ - return cache[(ZOPFLI_CACHE_LENGTH - 1) * 3] + 3; -} - -#endif /* ZOPFLI_LONGEST_MATCH_CACHE */ diff --git a/cache.h b/cache.h deleted file mode 100644 index 5ca0c50..0000000 --- a/cache.h +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -The cache that speeds up ZopfliFindLongestMatch of lz77.c. -*/ - -#ifndef ZOPFLI_CACHE_H_ -#define ZOPFLI_CACHE_H_ - -#include "util.h" - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - -/* -Cache used by ZopfliFindLongestMatch to remember previously found length/dist -values. -This is needed because the squeeze runs will ask these values multiple times for -the same position. -Uses large amounts of memory, since it has to remember the distance belonging -to every possible shorter-than-the-best length (the so called "sublen" array). -*/ -typedef struct ZopfliLongestMatchCache { - unsigned short* length; - unsigned short* dist; - unsigned char* sublen; -} ZopfliLongestMatchCache; - -/* Initializes the ZopfliLongestMatchCache. */ -void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc); - -/* Frees up the memory of the ZopfliLongestMatchCache. */ -void ZopfliCleanCache(ZopfliLongestMatchCache* lmc); - -/* Stores sublen array in the cache. */ -void ZopfliSublenToCache(const unsigned short* sublen, - size_t pos, size_t length, - ZopfliLongestMatchCache* lmc); - -/* Extracts sublen array from the cache. */ -void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc, - size_t pos, size_t length, - unsigned short* sublen); -/* Returns the length up to which could be stored in the cache. */ -unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc, - size_t pos, size_t length); - -#endif /* ZOPFLI_LONGEST_MATCH_CACHE */ - -#endif /* ZOPFLI_CACHE_H_ */ diff --git a/deflate.c b/deflate.c deleted file mode 100644 index 4b0724b..0000000 --- a/deflate.c +++ /dev/null @@ -1,866 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "deflate.h" - -#include -#include -#include - -#include "blocksplitter.h" -#include "lz77.h" -#include "squeeze.h" -#include "tree.h" - -/* -bp = bitpointer, always in range [0, 7]. -The outsize is number of necessary bytes to encode the bits. -Given the value of bp and the amount of bytes, the amount of bits represented -is not simply bytesize * 8 + bp because even representing one bit requires a -whole byte. It is: (bp == 0) ? (bytesize * 8) : ((bytesize - 1) * 8 + bp) -*/ -static void AddBit(int bit, - unsigned char* bp, unsigned char** out, size_t* outsize) { - if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); - (*out)[*outsize - 1] |= bit << *bp; - *bp = (*bp + 1) & 7; -} - -static void AddBits(unsigned symbol, unsigned length, - unsigned char* bp, unsigned char** out, size_t* outsize) { - /* TODO(lode): make more efficient (add more bits at once). */ - unsigned i; - for (i = 0; i < length; i++) { - unsigned bit = (symbol >> i) & 1; - if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); - (*out)[*outsize - 1] |= bit << *bp; - *bp = (*bp + 1) & 7; - } -} - -/* -Adds bits, like AddBits, but the order is inverted. The deflate specification -uses both orders in one standard. -*/ -static void AddHuffmanBits(unsigned symbol, unsigned length, - unsigned char* bp, unsigned char** out, - size_t* outsize) { - /* TODO(lode): make more efficient (add more bits at once). */ - unsigned i; - for (i = 0; i < length; i++) { - unsigned bit = (symbol >> (length - i - 1)) & 1; - if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); - (*out)[*outsize - 1] |= bit << *bp; - *bp = (*bp + 1) & 7; - } -} - -/* -Ensures there are at least 2 distance codes to support buggy decoders. -Zlib 1.2.1 and below have a bug where it fails if there isn't at least 1 -distance code (with length > 0), even though it's valid according to the -deflate spec to have 0 distance codes. On top of that, some mobile phones -require at least two distance codes. To support these decoders too (but -potentially at the cost of a few bytes), add dummy code lengths of 1. -References to this bug can be found in the changelog of -Zlib 1.2.2 and here: http://www.jonof.id.au/forum/index.php?topic=515.0. - -d_lengths: the 32 lengths of the distance codes. -*/ -static void PatchDistanceCodesForBuggyDecoders(unsigned* d_lengths) { - int num_dist_codes = 0; /* Amount of non-zero distance codes */ - int i; - for (i = 0; i < 30 /* Ignore the two unused codes from the spec */; i++) { - if (d_lengths[i]) num_dist_codes++; - if (num_dist_codes >= 2) return; /* Two or more codes is fine. */ - } - - if (num_dist_codes == 0) { - d_lengths[0] = d_lengths[1] = 1; - } else if (num_dist_codes == 1) { - d_lengths[d_lengths[0] ? 1 : 0] = 1; - } -} - -/* -Encodes the Huffman tree and returns how many bits its encoding takes. If out -is a null pointer, only returns the size and runs faster. -*/ -static size_t EncodeTree(const unsigned* ll_lengths, - const unsigned* d_lengths, - int use_16, int use_17, int use_18, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - unsigned lld_total; /* Total amount of literal, length, distance codes. */ - /* Runlength encoded version of lengths of litlen and dist trees. */ - unsigned* rle = 0; - unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */ - size_t rle_size = 0; /* Size of rle array. */ - size_t rle_bits_size = 0; /* Should have same value as rle_size. */ - unsigned hlit = 29; /* 286 - 257 */ - unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/ - unsigned hclen; - unsigned hlit2; - size_t i, j; - size_t clcounts[19]; - unsigned clcl[19]; /* Code length code lengths. */ - unsigned clsymbols[19]; - /* The order in which code length code lengths are encoded as per deflate. */ - static const unsigned order[19] = { - 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 - }; - int size_only = !out; - size_t result_size = 0; - - for(i = 0; i < 19; i++) clcounts[i] = 0; - - /* Trim zeros. */ - while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--; - while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--; - hlit2 = hlit + 257; - - lld_total = hlit2 + hdist + 1; - - for (i = 0; i < lld_total; i++) { - /* This is an encoding of a huffman tree, so now the length is a symbol */ - unsigned char symbol = i < hlit2 ? ll_lengths[i] : d_lengths[i - hlit2]; - unsigned count = 1; - if(use_16 || (symbol == 0 && (use_17 || use_18))) { - for (j = i + 1; j < lld_total && symbol == - (j < hlit2 ? ll_lengths[j] : d_lengths[j - hlit2]); j++) { - count++; - } - } - i += count - 1; - - /* Repetitions of zeroes */ - if (symbol == 0 && count >= 3) { - if (use_18) { - while (count >= 11) { - unsigned count2 = count > 138 ? 138 : count; - if (!size_only) { - ZOPFLI_APPEND_DATA(18, &rle, &rle_size); - ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size); - } - clcounts[18]++; - count -= count2; - } - } - if (use_17) { - while (count >= 3) { - unsigned count2 = count > 10 ? 10 : count; - if (!size_only) { - ZOPFLI_APPEND_DATA(17, &rle, &rle_size); - ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); - } - clcounts[17]++; - count -= count2; - } - } - } - - /* Repetitions of any symbol */ - if (use_16 && count >= 4) { - count--; /* Since the first one is hardcoded. */ - clcounts[symbol]++; - if (!size_only) { - ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); - ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); - } - while (count >= 3) { - unsigned count2 = count > 6 ? 6 : count; - if (!size_only) { - ZOPFLI_APPEND_DATA(16, &rle, &rle_size); - ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); - } - clcounts[16]++; - count -= count2; - } - } - - /* No or insufficient repetition */ - clcounts[symbol] += count; - while (count > 0) { - if (!size_only) { - ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); - ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); - } - count--; - } - } - - ZopfliCalculateBitLengths(clcounts, 19, 7, clcl); - if (!size_only) ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols); - - hclen = 15; - /* Trim zeros. */ - while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--; - - if (!size_only) { - AddBits(hlit, 5, bp, out, outsize); - AddBits(hdist, 5, bp, out, outsize); - AddBits(hclen, 4, bp, out, outsize); - - for (i = 0; i < hclen + 4; i++) { - AddBits(clcl[order[i]], 3, bp, out, outsize); - } - - for (i = 0; i < rle_size; i++) { - unsigned symbol = clsymbols[rle[i]]; - AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize); - /* Extra bits. */ - if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize); - else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize); - else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize); - } - } - - result_size += 14; /* hlit, hdist, hclen bits */ - result_size += (hclen + 4) * 3; /* clcl bits */ - for(i = 0; i < 19; i++) { - result_size += clcl[i] * clcounts[i]; - } - /* Extra bits. */ - result_size += clcounts[16] * 2; - result_size += clcounts[17] * 3; - result_size += clcounts[18] * 7; - - /* Note: in case of "size_only" these are null pointers so no effect. */ - free(rle); - free(rle_bits); - - return result_size; -} - -static void AddDynamicTree(const unsigned* ll_lengths, - const unsigned* d_lengths, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - int i; - int best = 0; - size_t bestsize = 0; - - for(i = 0; i < 8; i++) { - size_t size = EncodeTree(ll_lengths, d_lengths, - i & 1, i & 2, i & 4, - 0, 0, 0); - if (bestsize == 0 || size < bestsize) { - bestsize = size; - best = i; - } - } - - EncodeTree(ll_lengths, d_lengths, - best & 1, best & 2, best & 4, - bp, out, outsize); -} - -/* -Gives the exact size of the tree, in bits, as it will be encoded in DEFLATE. -*/ -static size_t CalculateTreeSize(const unsigned* ll_lengths, - const unsigned* d_lengths) { - size_t result = 0; - int i; - - for(i = 0; i < 8; i++) { - size_t size = EncodeTree(ll_lengths, d_lengths, - i & 1, i & 2, i & 4, - 0, 0, 0); - if (result == 0 || size < result) result = size; - } - - return result; -} - -/* -Adds all lit/len and dist codes from the lists as huffman symbols. Does not add -end code 256. expected_data_size is the uncompressed block size, used for -assert, but you can set it to 0 to not do the assertion. -*/ -static void AddLZ77Data(const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend, - size_t expected_data_size, - const unsigned* ll_symbols, const unsigned* ll_lengths, - const unsigned* d_symbols, const unsigned* d_lengths, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - size_t testlength = 0; - size_t i; - - for (i = lstart; i < lend; i++) { - unsigned dist = dists[i]; - unsigned litlen = litlens[i]; - if (dist == 0) { - assert(litlen < 256); - assert(ll_lengths[litlen] > 0); - AddHuffmanBits(ll_symbols[litlen], ll_lengths[litlen], bp, out, outsize); - testlength++; - } else { - unsigned lls = ZopfliGetLengthSymbol(litlen); - unsigned ds = ZopfliGetDistSymbol(dist); - assert(litlen >= 3 && litlen <= 288); - assert(ll_lengths[lls] > 0); - assert(d_lengths[ds] > 0); - AddHuffmanBits(ll_symbols[lls], ll_lengths[lls], bp, out, outsize); - AddBits(ZopfliGetLengthExtraBitsValue(litlen), - ZopfliGetLengthExtraBits(litlen), - bp, out, outsize); - AddHuffmanBits(d_symbols[ds], d_lengths[ds], bp, out, outsize); - AddBits(ZopfliGetDistExtraBitsValue(dist), - ZopfliGetDistExtraBits(dist), - bp, out, outsize); - testlength += litlen; - } - } - assert(expected_data_size == 0 || testlength == expected_data_size); -} - -static void GetFixedTree(unsigned* ll_lengths, unsigned* d_lengths) { - size_t i; - for (i = 0; i < 144; i++) ll_lengths[i] = 8; - for (i = 144; i < 256; i++) ll_lengths[i] = 9; - for (i = 256; i < 280; i++) ll_lengths[i] = 7; - for (i = 280; i < 288; i++) ll_lengths[i] = 8; - for (i = 0; i < 32; i++) d_lengths[i] = 5; -} - -/* -Calculates size of the part after the header and tree of an LZ77 block, in bits. -*/ -static size_t CalculateBlockSymbolSize(const unsigned* ll_lengths, - const unsigned* d_lengths, - const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend) { - size_t result = 0; - size_t i; - for (i = lstart; i < lend; i++) { - if (dists[i] == 0) { - result += ll_lengths[litlens[i]]; - } else { - result += ll_lengths[ZopfliGetLengthSymbol(litlens[i])]; - result += d_lengths[ZopfliGetDistSymbol(dists[i])]; - result += ZopfliGetLengthExtraBits(litlens[i]); - result += ZopfliGetDistExtraBits(dists[i]); - } - } - result += ll_lengths[256]; /*end symbol*/ - return result; -} - -static size_t AbsDiff(size_t x, size_t y) { - if (x > y) - return x - y; - else - return y - x; -} - -/* -Change the population counts in a way that the consequent Hufmann tree -compression, especially its rle-part will be more likely to compress this data -more efficiently. length containts the size of the histogram. -*/ -void OptimizeHuffmanForRle(int length, size_t* counts) { - int i, k, stride; - size_t symbol, sum, limit; - int* good_for_rle; - - /* 1) We don't want to touch the trailing zeros. We may break the - rules of the format by adding more data in the distance codes. */ - for (; length >= 0; --length) { - if (length == 0) { - return; - } - if (counts[length - 1] != 0) { - /* Now counts[0..length - 1] does not have trailing zeros. */ - break; - } - } - /* 2) Let's mark all population counts that already can be encoded - with an rle code.*/ - good_for_rle = (int*)malloc(length * sizeof(int)); - for (i = 0; i < length; ++i) good_for_rle[i] = 0; - - /* Let's not spoil any of the existing good rle codes. - Mark any seq of 0's that is longer than 5 as a good_for_rle. - Mark any seq of non-0's that is longer than 7 as a good_for_rle.*/ - symbol = counts[0]; - stride = 0; - for (i = 0; i < length + 1; ++i) { - if (i == length || counts[i] != symbol) { - if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) { - for (k = 0; k < stride; ++k) { - good_for_rle[i - k - 1] = 1; - } - } - stride = 1; - if (i != length) { - symbol = counts[i]; - } - } else { - ++stride; - } - } - - /* 3) Let's replace those population counts that lead to more rle codes. */ - stride = 0; - limit = counts[0]; - sum = 0; - for (i = 0; i < length + 1; ++i) { - if (i == length || good_for_rle[i] - /* Heuristic for selecting the stride ranges to collapse. */ - || AbsDiff(counts[i], limit) >= 4) { - if (stride >= 4 || (stride >= 3 && sum == 0)) { - /* The stride must end, collapse what we have, if we have enough (4). */ - int count = (sum + stride / 2) / stride; - if (count < 1) count = 1; - if (sum == 0) { - /* Don't make an all zeros stride to be upgraded to ones. */ - count = 0; - } - for (k = 0; k < stride; ++k) { - /* We don't want to change value at counts[i], - that is already belonging to the next stride. Thus - 1. */ - counts[i - k - 1] = count; - } - } - stride = 0; - sum = 0; - if (i < length - 3) { - /* All interesting strides have a count of at least 4, - at least when non-zeros. */ - limit = (counts[i] + counts[i + 1] + - counts[i + 2] + counts[i + 3] + 2) / 4; - } else if (i < length) { - limit = counts[i]; - } else { - limit = 0; - } - } - ++stride; - if (i != length) { - sum += counts[i]; - } - } - - free(good_for_rle); -} - -/* -Calculates the bit lengths for the symbols for dynamic blocks. Chooses bit -lengths that give the smallest size of tree encoding + encoding of all the -symbols to have smallest output size. This are not necessarily the ideal Huffman -bit lengths. -*/ -static void GetDynamicLengths(const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend, - unsigned* ll_lengths, unsigned* d_lengths) { - size_t ll_counts[288]; - size_t d_counts[32]; - - ZopfliLZ77Counts(litlens, dists, lstart, lend, ll_counts, d_counts); - OptimizeHuffmanForRle(288, ll_counts); - OptimizeHuffmanForRle(32, d_counts); - ZopfliCalculateBitLengths(ll_counts, 288, 15, ll_lengths); - ZopfliCalculateBitLengths(d_counts, 32, 15, d_lengths); - PatchDistanceCodesForBuggyDecoders(d_lengths); -} - -double ZopfliCalculateBlockSize(const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend, int btype) { - unsigned ll_lengths[288]; - unsigned d_lengths[32]; - - double result = 3; /* bfinal and btype bits */ - - assert(btype == 1 || btype == 2); /* This is not for uncompressed blocks. */ - - if(btype == 1) { - GetFixedTree(ll_lengths, d_lengths); - } else { - GetDynamicLengths(litlens, dists, lstart, lend, ll_lengths, d_lengths); - result += CalculateTreeSize(ll_lengths, d_lengths); - } - - result += CalculateBlockSymbolSize( - ll_lengths, d_lengths, litlens, dists, lstart, lend); - - return result; -} - -/* -Adds a deflate block with the given LZ77 data to the output. -options: global program options -btype: the block type, must be 1 or 2 -final: whether to set the "final" bit on this block, must be the last block -litlens: literal/length array of the LZ77 data, in the same format as in - ZopfliLZ77Store. -dists: distance array of the LZ77 data, in the same format as in - ZopfliLZ77Store. -lstart: where to start in the LZ77 data -lend: where to end in the LZ77 data (not inclusive) -expected_data_size: the uncompressed block size, used for assert, but you can - set it to 0 to not do the assertion. -bp: output bit pointer -out: dynamic output array to append to -outsize: dynamic output array size -*/ -static void AddLZ77Block(const ZopfliOptions* options, int btype, int final, - const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend, - size_t expected_data_size, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - unsigned ll_lengths[288]; - unsigned d_lengths[32]; - unsigned ll_symbols[288]; - unsigned d_symbols[32]; - size_t detect_block_size = *outsize; - size_t compressed_size; - size_t uncompressed_size = 0; - size_t i; - - AddBit(final, bp, out, outsize); - AddBit(btype & 1, bp, out, outsize); - AddBit((btype & 2) >> 1, bp, out, outsize); - - if (btype == 1) { - /* Fixed block. */ - GetFixedTree(ll_lengths, d_lengths); - } else { - /* Dynamic block. */ - unsigned detect_tree_size; - assert(btype == 2); - - GetDynamicLengths(litlens, dists, lstart, lend, ll_lengths, d_lengths); - - detect_tree_size = *outsize; - AddDynamicTree(ll_lengths, d_lengths, bp, out, outsize); - if (options->verbose) { - fprintf(stderr, "treesize: %d\n", (int)(*outsize - detect_tree_size)); - } - } - - ZopfliLengthsToSymbols(ll_lengths, 288, 15, ll_symbols); - ZopfliLengthsToSymbols(d_lengths, 32, 15, d_symbols); - - detect_block_size = *outsize; - AddLZ77Data(litlens, dists, lstart, lend, expected_data_size, - ll_symbols, ll_lengths, d_symbols, d_lengths, - bp, out, outsize); - /* End symbol. */ - AddHuffmanBits(ll_symbols[256], ll_lengths[256], bp, out, outsize); - - for (i = lstart; i < lend; i++) { - uncompressed_size += dists[i] == 0 ? 1 : litlens[i]; - } - compressed_size = *outsize - detect_block_size; - if (options->verbose) { - fprintf(stderr, "compressed block size: %d (%dk) (unc: %d)\n", - (int)compressed_size, (int)(compressed_size / 1024), - (int)(uncompressed_size)); - } -} - -static void DeflateDynamicBlock(const ZopfliOptions* options, int final, - const unsigned char* in, - size_t instart, size_t inend, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - ZopfliBlockState s; - size_t blocksize = inend - instart; - ZopfliLZ77Store store; - int btype = 2; - - ZopfliInitLZ77Store(&store); - - s.options = options; - s.blockstart = instart; - s.blockend = inend; -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache)); - ZopfliInitCache(blocksize, s.lmc); -#endif - - ZopfliLZ77Optimal(&s, in, instart, inend, &store); - - /* For small block, encoding with fixed tree can be smaller. For large block, - don't bother doing this expensive test, dynamic tree will be better.*/ - if (store.size < 1000) { - double dyncost, fixedcost; - ZopfliLZ77Store fixedstore; - ZopfliInitLZ77Store(&fixedstore); - ZopfliLZ77OptimalFixed(&s, in, instart, inend, &fixedstore); - dyncost = ZopfliCalculateBlockSize(store.litlens, store.dists, - 0, store.size, 2); - fixedcost = ZopfliCalculateBlockSize(fixedstore.litlens, fixedstore.dists, - 0, fixedstore.size, 1); - if (fixedcost < dyncost) { - btype = 1; - ZopfliCleanLZ77Store(&store); - store = fixedstore; - } else { - ZopfliCleanLZ77Store(&fixedstore); - } - } - - AddLZ77Block(s.options, btype, final, - store.litlens, store.dists, 0, store.size, - blocksize, bp, out, outsize); - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - ZopfliCleanCache(s.lmc); - free(s.lmc); -#endif - ZopfliCleanLZ77Store(&store); -} - -static void DeflateFixedBlock(const ZopfliOptions* options, int final, - const unsigned char* in, - size_t instart, size_t inend, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - ZopfliBlockState s; - size_t blocksize = inend - instart; - ZopfliLZ77Store store; - - ZopfliInitLZ77Store(&store); - - s.options = options; - s.blockstart = instart; - s.blockend = inend; -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache)); - ZopfliInitCache(blocksize, s.lmc); -#endif - - ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store); - - AddLZ77Block(s.options, 1, final, store.litlens, store.dists, 0, store.size, - blocksize, bp, out, outsize); - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - ZopfliCleanCache(s.lmc); - free(s.lmc); -#endif - ZopfliCleanLZ77Store(&store); -} - -static void DeflateNonCompressedBlock(const ZopfliOptions* options, int final, - const unsigned char* in, size_t instart, - size_t inend, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - size_t i; - size_t blocksize = inend - instart; - unsigned short nlen = ~blocksize; - - (void)options; - assert(blocksize < 65536); /* Non compressed blocks are max this size. */ - - AddBit(final, bp, out, outsize); - /* BTYPE 00 */ - AddBit(0, bp, out, outsize); - AddBit(0, bp, out, outsize); - - /* Any bits of input up to the next byte boundary are ignored. */ - *bp = 0; - - ZOPFLI_APPEND_DATA(blocksize % 256, out, outsize); - ZOPFLI_APPEND_DATA((blocksize / 256) % 256, out, outsize); - ZOPFLI_APPEND_DATA(nlen % 256, out, outsize); - ZOPFLI_APPEND_DATA((nlen / 256) % 256, out, outsize); - - for (i = instart; i < inend; i++) { - ZOPFLI_APPEND_DATA(in[i], out, outsize); - } -} - -static void DeflateBlock(const ZopfliOptions* options, - int btype, int final, - const unsigned char* in, size_t instart, size_t inend, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - if (btype == 0) { - DeflateNonCompressedBlock( - options, final, in, instart, inend, bp, out, outsize); - } else if (btype == 1) { - DeflateFixedBlock(options, final, in, instart, inend, bp, out, outsize); - } else { - assert (btype == 2); - DeflateDynamicBlock(options, final, in, instart, inend, bp, out, outsize); - } -} - -/* -Does squeeze strategy where first block splitting is done, then each block is -squeezed. -Parameters: see description of the ZopfliDeflate function. -*/ -static void DeflateSplittingFirst(const ZopfliOptions* options, - int btype, int final, - const unsigned char* in, - size_t instart, size_t inend, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - size_t i; - size_t* splitpoints = 0; - size_t npoints = 0; - if (btype == 0) { - ZopfliBlockSplitSimple(in, instart, inend, 65535, &splitpoints, &npoints); - } else if (btype == 1) { - /* If all blocks are fixed tree, splitting into separate blocks only - increases the total size. Leave npoints at 0, this represents 1 block. */ - } else { - ZopfliBlockSplit(options, in, instart, inend, - options->blocksplittingmax, &splitpoints, &npoints); - } - - for (i = 0; i <= npoints; i++) { - size_t start = i == 0 ? instart : splitpoints[i - 1]; - size_t end = i == npoints ? inend : splitpoints[i]; - DeflateBlock(options, btype, i == npoints && final, in, start, end, - bp, out, outsize); - } - - free(splitpoints); -} - -/* -Does squeeze strategy where first the best possible lz77 is done, and then based -on that data, block splitting is done. -Parameters: see description of the ZopfliDeflate function. -*/ -static void DeflateSplittingLast(const ZopfliOptions* options, - int btype, int final, - const unsigned char* in, - size_t instart, size_t inend, - unsigned char* bp, - unsigned char** out, size_t* outsize) { - size_t i; - ZopfliBlockState s; - ZopfliLZ77Store store; - size_t* splitpoints = 0; - size_t npoints = 0; - - if (btype == 0) { - /* This function only supports LZ77 compression. DeflateSplittingFirst - supports the special case of noncompressed data. Punt it to that one. */ - DeflateSplittingFirst(options, btype, final, - in, instart, inend, - bp, out, outsize); - } - assert(btype == 1 || btype == 2); - - ZopfliInitLZ77Store(&store); - - s.options = options; - s.blockstart = instart; - s.blockend = inend; -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache)); - ZopfliInitCache(inend - instart, s.lmc); -#endif - - if (btype == 2) { - ZopfliLZ77Optimal(&s, in, instart, inend, &store); - } else { - assert (btype == 1); - ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store); - } - - if (btype == 1) { - /* If all blocks are fixed tree, splitting into separate blocks only - increases the total size. Leave npoints at 0, this represents 1 block. */ - } else { - ZopfliBlockSplitLZ77(options, store.litlens, store.dists, store.size, - options->blocksplittingmax, &splitpoints, &npoints); - } - - for (i = 0; i <= npoints; i++) { - size_t start = i == 0 ? 0 : splitpoints[i - 1]; - size_t end = i == npoints ? store.size : splitpoints[i]; - AddLZ77Block(options, btype, i == npoints && final, - store.litlens, store.dists, start, end, 0, - bp, out, outsize); - } - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - ZopfliCleanCache(s.lmc); - free(s.lmc); -#endif - - ZopfliCleanLZ77Store(&store); - free(splitpoints); -} - -/* -Deflate a part, to allow ZopfliDeflate() to use multiple master blocks if -needed. -It is possible to call this function multiple times in a row, shifting -instart and inend to next bytes of the data. If instart is larger than 0, then -previous bytes are used as the initial dictionary for LZ77. -This function will usually output multiple deflate blocks. If final is 1, then -the final bit will be set on the last block. -*/ -void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, - const unsigned char* in, size_t instart, size_t inend, - unsigned char* bp, unsigned char** out, - size_t* outsize) { - if (options->blocksplitting) { - if (options->blocksplittinglast) { - DeflateSplittingLast(options, btype, final, in, instart, inend, - bp, out, outsize); - } else { - DeflateSplittingFirst(options, btype, final, in, instart, inend, - bp, out, outsize); - } - } else { - DeflateBlock(options, btype, final, in, instart, inend, bp, out, outsize); - } -} - -void ZopfliDeflate(const ZopfliOptions* options, int btype, int final, - const unsigned char* in, size_t insize, - unsigned char* bp, unsigned char** out, size_t* outsize) { -#if ZOPFLI_MASTER_BLOCK_SIZE == 0 - ZopfliDeflatePart(options, btype, final, in, 0, insize, bp, out, outsize); -#else - size_t i = 0; - while (i < insize) { - int masterfinal = (i + ZOPFLI_MASTER_BLOCK_SIZE >= insize); - int final2 = final && masterfinal; - size_t size = masterfinal ? insize - i : ZOPFLI_MASTER_BLOCK_SIZE; - ZopfliDeflatePart(options, btype, final2, - in, i, i + size, bp, out, outsize); - i += size; - } -#endif - if (options->verbose) { - fprintf(stderr, - "Original Size: %d, Deflate: %d, Compression: %f%% Removed\n", - (int)insize, (int)*outsize, - 100.0 * (double)(insize - *outsize) / (double)insize); - } -} diff --git a/deflate.h b/deflate.h deleted file mode 100644 index 189c77a..0000000 --- a/deflate.h +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#ifndef ZOPFLI_DEFLATE_H_ -#define ZOPFLI_DEFLATE_H_ - -/* -Functions to compress according to the DEFLATE specification, using the -"squeeze" LZ77 compression backend. -*/ - -#include "zopfli.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* -Compresses according to the deflate specification and append the compressed -result to the output. -This function will usually output multiple deflate blocks. If final is 1, then -the final bit will be set on the last block. - -options: global program options -btype: the deflate block type. Use 2 for best compression. - -0: non compressed blocks (00) - -1: blocks with fixed tree (01) - -2: blocks with dynamic tree (10) -final: whether this is the last section of the input, sets the final bit to the - last deflate block. -in: the input bytes -insize: number of input bytes -bp: bit pointer for the output array. This must initially be 0, and for - consecutive calls must be reused (it can have values from 0-7). This is - because deflate appends blocks as bit-based data, rather than on byte - boundaries. -out: pointer to the dynamic output array to which the result is appended. Must - be freed after use. -outsize: pointer to the dynamic output array size. -*/ -void ZopfliDeflate(const ZopfliOptions* options, int btype, int final, - const unsigned char* in, size_t insize, - unsigned char* bp, unsigned char** out, size_t* outsize); - -/* -Like ZopfliDeflate, but allows to specify start and end byte with instart and -inend. Only that part is compressed, but earlier bytes are still used for the -back window. -*/ -void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, - const unsigned char* in, size_t instart, size_t inend, - unsigned char* bp, unsigned char** out, - size_t* outsize); - -/* -Calculates block size in bits. -litlens: lz77 lit/lengths -dists: ll77 distances -lstart: start of block -lend: end of block (not inclusive) -*/ -double ZopfliCalculateBlockSize(const unsigned short* litlens, - const unsigned short* dists, - size_t lstart, size_t lend, int btype); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* ZOPFLI_DEFLATE_H_ */ diff --git a/gzip_container.c b/gzip_container.c deleted file mode 100644 index 8a062f2..0000000 --- a/gzip_container.c +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2013 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "gzip_container.h" -#include "util.h" - -#include - -#include "deflate.h" - -/* Table of CRCs of all 8-bit messages. */ -static unsigned long crc_table[256]; - -/* Flag: has the table been computed? Initially false. */ -static int crc_table_computed = 0; - -/* Makes the table for a fast CRC. */ -static void MakeCRCTable() { - unsigned long c; - int n, k; - for (n = 0; n < 256; n++) { - c = (unsigned long) n; - for (k = 0; k < 8; k++) { - if (c & 1) { - c = 0xedb88320L ^ (c >> 1); - } else { - c = c >> 1; - } - } - crc_table[n] = c; - } - crc_table_computed = 1; -} - - -/* -Updates a running crc with the bytes buf[0..len-1] and returns -the updated crc. The crc should be initialized to zero. -*/ -static unsigned long UpdateCRC(unsigned long crc, - const unsigned char *buf, size_t len) { - unsigned long c = crc ^ 0xffffffffL; - unsigned n; - - if (!crc_table_computed) - MakeCRCTable(); - for (n = 0; n < len; n++) { - c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8); - } - return c ^ 0xffffffffL; -} - -/* Returns the CRC of the bytes buf[0..len-1]. */ -static unsigned long CRC(const unsigned char* buf, int len) { - return UpdateCRC(0L, buf, len); -} - -/* -Compresses the data according to the gzip specification. -*/ -void ZopfliGzipCompress(const ZopfliOptions* options, - const unsigned char* in, size_t insize, - unsigned char** out, size_t* outsize) { - unsigned long crcvalue = CRC(in, insize); - unsigned char bp = 0; - - ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */ - ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */ - ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */ - ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */ - /* MTIME */ - ZOPFLI_APPEND_DATA(0, out, outsize); - ZOPFLI_APPEND_DATA(0, out, outsize); - ZOPFLI_APPEND_DATA(0, out, outsize); - ZOPFLI_APPEND_DATA(0, out, outsize); - - ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */ - ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */ - - ZopfliDeflate(options, 2 /* Dynamic block */, 1, - in, insize, &bp, out, outsize); - - /* CRC */ - ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize); - ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize); - ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize); - ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize); - - /* ISIZE */ - ZOPFLI_APPEND_DATA(insize % 256, out, outsize); - ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize); - ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize); - ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize); - - if (options->verbose) { - fprintf(stderr, - "Original Size: %d, Gzip: %d, Compression: %f%% Removed\n", - (int)insize, (int)*outsize, - 100.0 * (double)(insize - *outsize) / (double)insize); - } -} diff --git a/gzip_container.h b/gzip_container.h deleted file mode 100644 index 8f5ed90..0000000 --- a/gzip_container.h +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2013 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#ifndef ZOPFLI_GZIP_H_ -#define ZOPFLI_GZIP_H_ - -/* -Functions to compress according to the Gzip specification. -*/ - -#include "zopfli.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* -Compresses according to the gzip specification and append the compressed -result to the output. - -options: global program options -out: pointer to the dynamic output array to which the result is appended. Must - be freed after use. -outsize: pointer to the dynamic output array size. -*/ -void ZopfliGzipCompress(const ZopfliOptions* options, - const unsigned char* in, size_t insize, - unsigned char** out, size_t* outsize); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* ZOPFLI_GZIP_H_ */ diff --git a/hash.c b/hash.c deleted file mode 100644 index a3b294f..0000000 --- a/hash.c +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "hash.h" - -#include -#include -#include - -#define HASH_SHIFT 5 -#define HASH_MASK 32767 - -void ZopfliInitHash(size_t window_size, ZopfliHash* h) { - size_t i; - - h->val = 0; - h->head = (int*)malloc(sizeof(*h->head) * 65536); - h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size); - h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size); - for (i = 0; i < 65536; i++) { - h->head[i] = -1; /* -1 indicates no head so far. */ - } - for (i = 0; i < window_size; i++) { - h->prev[i] = i; /* If prev[j] == j, then prev[j] is uninitialized. */ - h->hashval[i] = -1; - } - -#ifdef ZOPFLI_HASH_SAME - h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size); - for (i = 0; i < window_size; i++) { - h->same[i] = 0; - } -#endif - -#ifdef ZOPFLI_HASH_SAME_HASH - h->val2 = 0; - h->head2 = (int*)malloc(sizeof(*h->head2) * 65536); - h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size); - h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size); - for (i = 0; i < 65536; i++) { - h->head2[i] = -1; - } - for (i = 0; i < window_size; i++) { - h->prev2[i] = i; - h->hashval2[i] = -1; - } -#endif -} - -void ZopfliCleanHash(ZopfliHash* h) { - free(h->head); - free(h->prev); - free(h->hashval); - -#ifdef ZOPFLI_HASH_SAME_HASH - free(h->head2); - free(h->prev2); - free(h->hashval2); -#endif - -#ifdef ZOPFLI_HASH_SAME - free(h->same); -#endif -} - -/* -Update the sliding hash value with the given byte. All calls to this function -must be made on consecutive input characters. Since the hash value exists out -of multiple input bytes, a few warmups with this function are needed initially. -*/ -static void UpdateHashValue(ZopfliHash* h, unsigned char c) { - h->val = (((h->val) << HASH_SHIFT) ^ (c)) & HASH_MASK; -} - -void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end, - ZopfliHash* h) { - unsigned short hpos = pos & ZOPFLI_WINDOW_MASK; -#ifdef ZOPFLI_HASH_SAME - size_t amount = 0; -#endif - - UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ? - array[pos + ZOPFLI_MIN_MATCH - 1] : 0); - h->hashval[hpos] = h->val; - if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) { - h->prev[hpos] = h->head[h->val]; - } - else h->prev[hpos] = hpos; - h->head[h->val] = hpos; - -#ifdef ZOPFLI_HASH_SAME - /* Update "same". */ - if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) { - amount = h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] - 1; - } - while (pos + amount + 1 < end && - array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) { - amount++; - } - h->same[hpos] = amount; -#endif - -#ifdef ZOPFLI_HASH_SAME_HASH - h->val2 = ((h->same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h->val; - h->hashval2[hpos] = h->val2; - if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) { - h->prev2[hpos] = h->head2[h->val2]; - } - else h->prev2[hpos] = hpos; - h->head2[h->val2] = hpos; -#endif -} - -void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end, - ZopfliHash* h) { - (void)end; - UpdateHashValue(h, array[pos + 0]); - UpdateHashValue(h, array[pos + 1]); -} diff --git a/hash.h b/hash.h deleted file mode 100644 index 79c2479..0000000 --- a/hash.h +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -The hash for ZopfliFindLongestMatch of lz77.c. -*/ - -#ifndef ZOPFLI_HASH_H_ -#define ZOPFLI_HASH_H_ - -#include "util.h" - -typedef struct ZopfliHash { - int* head; /* Hash value to index of its most recent occurance. */ - unsigned short* prev; /* Index to index of prev. occurance of same hash. */ - int* hashval; /* Index to hash value at this index. */ - int val; /* Current hash value. */ - -#ifdef ZOPFLI_HASH_SAME_HASH - /* Fields with similar purpose as the above hash, but for the second hash with - a value that is calculated differently. */ - int* head2; /* Hash value to index of its most recent occurance. */ - unsigned short* prev2; /* Index to index of prev. occurance of same hash. */ - int* hashval2; /* Index to hash value at this index. */ - int val2; /* Current hash value. */ -#endif - -#ifdef ZOPFLI_HASH_SAME - unsigned short* same; /* Amount of repetitions of same byte after this .*/ -#endif -} ZopfliHash; - -/* Allocates and initializes all fields of ZopfliHash. */ -void ZopfliInitHash(size_t window_size, ZopfliHash* h); - -/* Frees all fields of ZopfliHash. */ -void ZopfliCleanHash(ZopfliHash* h); - -/* -Updates the hash values based on the current position in the array. All calls -to this must be made for consecutive bytes. -*/ -void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end, - ZopfliHash* h); - -/* -Prepopulates hash: -Fills in the initial values in the hash, before ZopfliUpdateHash can be used -correctly. -*/ -void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end, - ZopfliHash* h); - -#endif /* ZOPFLI_HASH_H_ */ diff --git a/katajainen.c b/katajainen.c deleted file mode 100644 index 783ea08..0000000 --- a/katajainen.c +++ /dev/null @@ -1,251 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -Bounded package merge algorithm, based on the paper -"A Fast and Space-Economical Algorithm for Length-Limited Coding -Jyrki Katajainen, Alistair Moffat, Andrew Turpin". -*/ - -#include "katajainen.h" -#include -#include - -typedef struct Node Node; - -/* -Nodes forming chains. Also used to represent leaves. -*/ -struct Node { - size_t weight; /* Total weight (symbol count) of this chain. */ - Node* tail; /* Previous node(s) of this chain, or 0 if none. */ - int count; /* Leaf symbol index, or number of leaves before this chain. */ - char inuse; /* Tracking for garbage collection. */ -}; - -/* -Memory pool for nodes. -*/ -typedef struct NodePool { - Node* nodes; /* The pool. */ - Node* next; /* Pointer to a possibly free node in the pool. */ - int size; /* Size of the memory pool. */ -} NodePool; - -/* -Initializes a chain node with the given values and marks it as in use. -*/ -static void InitNode(size_t weight, int count, Node* tail, Node* node) { - node->weight = weight; - node->count = count; - node->tail = tail; - node->inuse = 1; -} - -/* -Finds a free location in the memory pool. Performs garbage collection if needed. -lists: If given, used to mark in-use nodes during garbage collection. -maxbits: Size of lists. -pool: Memory pool to get free node from. -*/ -static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) { - for (;;) { - if (pool->next >= &pool->nodes[pool->size]) { - /* Garbage collection. */ - int i; - for (i = 0; i < pool->size; i++) { - pool->nodes[i].inuse = 0; - } - if (lists) { - for (i = 0; i < maxbits * 2; i++) { - Node* node; - for (node = lists[i / 2][i % 2]; node; node = node->tail) { - node->inuse = 1; - } - } - } - pool->next = &pool->nodes[0]; - } - if (!pool->next->inuse) break; /* Found one. */ - pool->next++; - } - return pool->next++; -} - - -/* -Performs a Boundary Package-Merge step. Puts a new chain in the given list. The -new chain is, depending on the weights, a leaf or a combination of two chains -from the previous list. -lists: The lists of chains. -maxbits: Number of lists. -leaves: The leaves, one per symbol. -numsymbols: Number of leaves. -pool: the node memory pool. -index: The index of the list in which a new chain or leaf is required. -final: Whether this is the last time this function is called. If it is then it - is no more needed to recursively call self. -*/ -static void BoundaryPM(Node* (*lists)[2], int maxbits, - Node* leaves, int numsymbols, NodePool* pool, int index, char final) { - Node* newchain; - Node* oldchain; - int lastcount = lists[index][1]->count; /* Count of last chain of list. */ - - if (index == 0 && lastcount >= numsymbols) return; - - newchain = GetFreeNode(lists, maxbits, pool); - oldchain = lists[index][1]; - - /* These are set up before the recursive calls below, so that there is a list - pointing to the new node, to let the garbage collection know it's in use. */ - lists[index][0] = oldchain; - lists[index][1] = newchain; - - if (index == 0) { - /* New leaf node in list 0. */ - InitNode(leaves[lastcount].weight, lastcount + 1, 0, newchain); - } else { - size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight; - if (lastcount < numsymbols && sum > leaves[lastcount].weight) { - /* New leaf inserted in list, so count is incremented. */ - InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail, - newchain); - } else { - InitNode(sum, lastcount, lists[index - 1][1], newchain); - if (!final) { - /* Two lookahead chains of previous list used up, create new ones. */ - BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0); - BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0); - } - } - } -} - -/* -Initializes each list with as lookahead chains the two leaves with lowest -weights. -*/ -static void InitLists( - NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) { - int i; - Node* node0 = GetFreeNode(0, maxbits, pool); - Node* node1 = GetFreeNode(0, maxbits, pool); - InitNode(leaves[0].weight, 1, 0, node0); - InitNode(leaves[1].weight, 2, 0, node1); - for (i = 0; i < maxbits; i++) { - lists[i][0] = node0; - lists[i][1] = node1; - } -} - -/* -Converts result of boundary package-merge to the bitlengths. The result in the -last chain of the last list contains the amount of active leaves in each list. -chain: Chain to extract the bit length from (last chain from last list). -*/ -static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) { - Node* node; - for (node = chain; node; node = node->tail) { - int i; - for (i = 0; i < node->count; i++) { - bitlengths[leaves[i].count]++; - } - } -} - -/* -Comparator for sorting the leaves. Has the function signature for qsort. -*/ -static int LeafComparator(const void* a, const void* b) { - return ((const Node*)a)->weight - ((const Node*)b)->weight; -} - -int ZopfliLengthLimitedCodeLengths( - const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) { - NodePool pool; - int i; - int numsymbols = 0; /* Amount of symbols with frequency > 0. */ - int numBoundaryPMRuns; - - /* Array of lists of chains. Each list requires only two lookahead chains at - a time, so each list is a array of two Node*'s. */ - Node* (*lists)[2]; - - /* One leaf per symbol. Only numsymbols leaves will be used. */ - Node* leaves = (Node*)malloc(n * sizeof(*leaves)); - - /* Initialize all bitlengths at 0. */ - for (i = 0; i < n; i++) { - bitlengths[i] = 0; - } - - /* Count used symbols and place them in the leaves. */ - for (i = 0; i < n; i++) { - if (frequencies[i]) { - leaves[numsymbols].weight = frequencies[i]; - leaves[numsymbols].count = i; /* Index of symbol this leaf represents. */ - numsymbols++; - } - } - - /* Check special cases and error conditions. */ - if ((1 << maxbits) < numsymbols) { - free(leaves); - return 1; /* Error, too few maxbits to represent symbols. */ - } - if (numsymbols == 0) { - free(leaves); - return 0; /* No symbols at all. OK. */ - } - if (numsymbols == 1) { - bitlengths[leaves[0].count] = 1; - free(leaves); - return 0; /* Only one symbol, give it bitlength 1, not 0. OK. */ - } - - /* Sort the leaves from lightest to heaviest. */ - qsort(leaves, numsymbols, sizeof(Node), LeafComparator); - - /* Initialize node memory pool. */ - pool.size = 2 * maxbits * (maxbits + 1); - pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes)); - pool.next = pool.nodes; - for (i = 0; i < pool.size; i++) { - pool.nodes[i].inuse = 0; - } - - lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists)); - InitLists(&pool, leaves, maxbits, lists); - - /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two - are already created in the initialization. Each BoundaryPM run creates one. */ - numBoundaryPMRuns = 2 * numsymbols - 4; - for (i = 0; i < numBoundaryPMRuns; i++) { - char final = i == numBoundaryPMRuns - 1; - BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final); - } - - ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths); - - free(lists); - free(leaves); - free(pool.nodes); - return 0; /* OK. */ -} diff --git a/katajainen.h b/katajainen.h deleted file mode 100644 index ee8a91e..0000000 --- a/katajainen.h +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#ifndef ZOPFLI_KATAJAINEN_H_ -#define ZOPFLI_KATAJAINEN_H_ - -#include - -/* -Outputs minimum-redundancy length-limited code bitlengths for symbols with the -given counts. The bitlengths are limited by maxbits. - -The output is tailored for DEFLATE: symbols that never occur, get a bit length -of 0, and if only a single symbol occurs at least once, its bitlength will be 1, -and not 0 as would theoretically be needed for a single symbol. - -frequencies: The amount of occurances of each symbol. -n: The amount of symbols. -maxbits: Maximum bit length, inclusive. -bitlengths: Output, the bitlengths for the symbol prefix codes. -return: 0 for OK, non-0 for error. -*/ -int ZopfliLengthLimitedCodeLengths( - const size_t* frequencies, int n, int maxbits, unsigned* bitlengths); - -#endif /* ZOPFLI_KATAJAINEN_H_ */ diff --git a/lz77.c b/lz77.c deleted file mode 100644 index 26186b4..0000000 --- a/lz77.c +++ /dev/null @@ -1,482 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "lz77.h" -#include "util.h" - -#include -#include -#include - -void ZopfliInitLZ77Store(ZopfliLZ77Store* store) { - store->size = 0; - store->litlens = 0; - store->dists = 0; -} - -void ZopfliCleanLZ77Store(ZopfliLZ77Store* store) { - free(store->litlens); - free(store->dists); -} - -void ZopfliCopyLZ77Store( - const ZopfliLZ77Store* source, ZopfliLZ77Store* dest) { - size_t i; - ZopfliCleanLZ77Store(dest); - dest->litlens = - (unsigned short*)malloc(sizeof(*dest->litlens) * source->size); - dest->dists = (unsigned short*)malloc(sizeof(*dest->dists) * source->size); - - if (!dest->litlens || !dest->dists) exit(-1); /* Allocation failed. */ - - dest->size = source->size; - for (i = 0; i < source->size; i++) { - dest->litlens[i] = source->litlens[i]; - dest->dists[i] = source->dists[i]; - } -} - -/* -Appends the length and distance to the LZ77 arrays of the ZopfliLZ77Store. -context must be a ZopfliLZ77Store*. -*/ -void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist, - ZopfliLZ77Store* store) { - size_t size2 = store->size; /* Needed for using ZOPFLI_APPEND_DATA twice. */ - ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size); - ZOPFLI_APPEND_DATA(dist, &store->dists, &size2); -} - -/* -Gets a score of the length given the distance. Typically, the score of the -length is the length itself, but if the distance is very long, decrease the -score of the length a bit to make up for the fact that long distances use large -amounts of extra bits. - -This is not an accurate score, it is a heuristic only for the greedy LZ77 -implementation. More accurate cost models are employed later. Making this -heuristic more accurate may hurt rather than improve compression. - -The two direct uses of this heuristic are: --avoid using a length of 3 in combination with a long distance. This only has - an effect if length == 3. --make a slightly better choice between the two options of the lazy matching. - -Indirectly, this affects: --the block split points if the default of block splitting first is used, in a - rather unpredictable way --the first zopfli run, so it affects the chance of the first run being closer - to the optimal output -*/ -static int GetLengthScore(int length, int distance) { - /* - At 1024, the distance uses 9+ extra bits and this seems to be the sweet spot - on tested files. - */ - return distance > 1024 ? length - 1 : length; -} - -void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos, - unsigned short dist, unsigned short length) { - - /* TODO(lode): make this only run in a debug compile, it's for assert only. */ - size_t i; - - assert(pos + length <= datasize); - for (i = 0; i < length; i++) { - if (data[pos - dist + i] != data[pos + i]) { - assert(data[pos - dist + i] == data[pos + i]); - break; - } - } -} - -/* -Finds how long the match of scan and match is. Can be used to find how many -bytes starting from scan, and from match, are equal. Returns the last byte -after scan, which is still equal to the correspondinb byte after match. -scan is the position to compare -match is the earlier position to compare. -end is the last possible byte, beyond which to stop looking. -safe_end is a few (8) bytes before end, for comparing multiple bytes at once. -*/ -static const unsigned char* GetMatch(const unsigned char* scan, - const unsigned char* match, - const unsigned char* end, - const unsigned char* safe_end) { - - if (sizeof(size_t) == 8) { - /* 8 checks at once per array bounds check (size_t is 64-bit). */ - while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) { - scan += 8; - match += 8; - } - } else if (sizeof(unsigned int) == 4) { - /* 4 checks at once per array bounds check (unsigned int is 32-bit). */ - while (scan < safe_end - && *((unsigned int*)scan) == *((unsigned int*)match)) { - scan += 4; - match += 4; - } - } else { - /* do 8 checks at once per array bounds check. */ - while (scan < safe_end && *scan == *match && *++scan == *++match - && *++scan == *++match && *++scan == *++match - && *++scan == *++match && *++scan == *++match - && *++scan == *++match && *++scan == *++match) { - scan++; match++; - } - } - - /* The remaining few bytes. */ - while (scan != end && *scan == *match) { - scan++; match++; - } - - return scan; -} - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE -/* -Gets distance, length and sublen values from the cache if possible. -Returns 1 if it got the values from the cache, 0 if not. -Updates the limit value to a smaller one if possible with more limited -information from the cache. -*/ -static int TryGetFromLongestMatchCache(ZopfliBlockState* s, - size_t pos, size_t* limit, - unsigned short* sublen, unsigned short* distance, unsigned short* length) { - /* The LMC cache starts at the beginning of the block rather than the - beginning of the whole array. */ - size_t lmcpos = pos - s->blockstart; - - /* Length > 0 and dist 0 is invalid combination, which indicates on purpose - that this cache value is not filled in yet. */ - unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || - s->lmc->dist[lmcpos] != 0); - unsigned char limit_ok_for_cache = cache_available && - (*limit == ZOPFLI_MAX_MATCH || s->lmc->length[lmcpos] <= *limit || - (sublen && ZopfliMaxCachedSublen(s->lmc, - lmcpos, s->lmc->length[lmcpos]) >= *limit)); - - if (s->lmc && limit_ok_for_cache && cache_available) { - if (!sublen || s->lmc->length[lmcpos] - <= ZopfliMaxCachedSublen(s->lmc, lmcpos, s->lmc->length[lmcpos])) { - *length = s->lmc->length[lmcpos]; - if (*length > *limit) *length = *limit; - if (sublen) { - ZopfliCacheToSublen(s->lmc, lmcpos, *length, sublen); - *distance = sublen[*length]; - if (*limit == ZOPFLI_MAX_MATCH && *length >= ZOPFLI_MIN_MATCH) { - assert(sublen[*length] == s->lmc->dist[lmcpos]); - } - } else { - *distance = s->lmc->dist[lmcpos]; - } - return 1; - } - /* Can't use much of the cache, since the "sublens" need to be calculated, - but at least we already know when to stop. */ - *limit = s->lmc->length[lmcpos]; - } - - return 0; -} - -/* -Stores the found sublen, distance and length in the longest match cache, if -possible. -*/ -static void StoreInLongestMatchCache(ZopfliBlockState* s, - size_t pos, size_t limit, - const unsigned short* sublen, - unsigned short distance, unsigned short length) { - /* The LMC cache starts at the beginning of the block rather than the - beginning of the whole array. */ - size_t lmcpos = pos - s->blockstart; - - /* Length > 0 and dist 0 is invalid combination, which indicates on purpose - that this cache value is not filled in yet. */ - unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || - s->lmc->dist[lmcpos] != 0); - - if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) { - assert(s->lmc->length[lmcpos] == 1 && s->lmc->dist[lmcpos] == 0); - s->lmc->dist[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : distance; - s->lmc->length[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : length; - assert(!(s->lmc->length[lmcpos] == 1 && s->lmc->dist[lmcpos] == 0)); - ZopfliSublenToCache(sublen, lmcpos, length, s->lmc); - } -} -#endif - -void ZopfliFindLongestMatch(ZopfliBlockState* s, const ZopfliHash* h, - const unsigned char* array, - size_t pos, size_t size, size_t limit, - unsigned short* sublen, unsigned short* distance, unsigned short* length) { - unsigned short hpos = pos & ZOPFLI_WINDOW_MASK, p, pp; - unsigned short bestdist = 0; - unsigned short bestlength = 1; - const unsigned char* scan; - const unsigned char* match; - const unsigned char* arrayend; - const unsigned char* arrayend_safe; -#if ZOPFLI_MAX_CHAIN_HITS < ZOPFLI_WINDOW_SIZE - int chain_counter = ZOPFLI_MAX_CHAIN_HITS; /* For quitting early. */ -#endif - - unsigned dist = 0; /* Not unsigned short on purpose. */ - - int* hhead = h->head; - unsigned short* hprev = h->prev; - int* hhashval = h->hashval; - int hval = h->val; - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - if (TryGetFromLongestMatchCache(s, pos, &limit, sublen, distance, length)) { - assert(pos + *length <= size); - return; - } -#endif - - assert(limit <= ZOPFLI_MAX_MATCH); - assert(limit >= ZOPFLI_MIN_MATCH); - assert(pos < size); - - if (size - pos < ZOPFLI_MIN_MATCH) { - /* The rest of the code assumes there are at least ZOPFLI_MIN_MATCH bytes to - try. */ - *length = 0; - *distance = 0; - return; - } - - if (pos + limit > size) { - limit = size - pos; - } - arrayend = &array[pos] + limit; - arrayend_safe = arrayend - 8; - - assert(hval < 65536); - - pp = hhead[hval]; /* During the whole loop, p == hprev[pp]. */ - p = hprev[pp]; - - assert(pp == hpos); - - dist = p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp); - - /* Go through all distances. */ - while (dist < ZOPFLI_WINDOW_SIZE) { - unsigned short currentlength = 0; - - assert(p < ZOPFLI_WINDOW_SIZE); - assert(p == hprev[pp]); - assert(hhashval[p] == hval); - - if (dist > 0) { - assert(pos < size); - assert(dist <= pos); - scan = &array[pos]; - match = &array[pos - dist]; - - /* Testing the byte at position bestlength first, goes slightly faster. */ - if (pos + bestlength >= size - || *(scan + bestlength) == *(match + bestlength)) { - -#ifdef ZOPFLI_HASH_SAME - unsigned short same0 = h->same[pos & ZOPFLI_WINDOW_MASK]; - if (same0 > 2 && *scan == *match) { - unsigned short same1 = h->same[(pos - dist) & ZOPFLI_WINDOW_MASK]; - unsigned short same = same0 < same1 ? same0 : same1; - if (same > limit) same = limit; - scan += same; - match += same; - } -#endif - scan = GetMatch(scan, match, arrayend, arrayend_safe); - currentlength = scan - &array[pos]; /* The found length. */ - } - - if (currentlength > bestlength) { - if (sublen) { - unsigned short j; - for (j = bestlength + 1; j <= currentlength; j++) { - sublen[j] = dist; - } - } - bestdist = dist; - bestlength = currentlength; - if (currentlength >= limit) break; - } - } - - -#ifdef ZOPFLI_HASH_SAME_HASH - /* Switch to the other hash once this will be more efficient. */ - if (hhead != h->head2 && bestlength >= h->same[hpos] && - h->val2 == h->hashval2[p]) { - /* Now use the hash that encodes the length and first byte. */ - hhead = h->head2; - hprev = h->prev2; - hhashval = h->hashval2; - hval = h->val2; - } -#endif - - pp = p; - p = hprev[p]; - if (p == pp) break; /* Uninited prev value. */ - - dist += p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp); - -#if ZOPFLI_MAX_CHAIN_HITS < ZOPFLI_WINDOW_SIZE - chain_counter--; - if (chain_counter <= 0) break; -#endif - } - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - StoreInLongestMatchCache(s, pos, limit, sublen, bestdist, bestlength); -#endif - - assert(bestlength <= limit); - - *distance = bestdist; - *length = bestlength; - assert(pos + *length <= size); -} - -void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, - size_t instart, size_t inend, - ZopfliLZ77Store* store) { - size_t i = 0, j; - unsigned short leng; - unsigned short dist; - int lengthscore; - size_t windowstart = instart > ZOPFLI_WINDOW_SIZE - ? instart - ZOPFLI_WINDOW_SIZE : 0; - unsigned short dummysublen[259]; - - ZopfliHash hash; - ZopfliHash* h = &hash; - -#ifdef ZOPFLI_LAZY_MATCHING - /* Lazy matching. */ - unsigned prev_length = 0; - unsigned prev_match = 0; - int prevlengthscore; - int match_available = 0; -#endif - - if (instart == inend) return; - - ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h); - ZopfliWarmupHash(in, windowstart, inend, h); - for (i = windowstart; i < instart; i++) { - ZopfliUpdateHash(in, i, inend, h); - } - - for (i = instart; i < inend; i++) { - ZopfliUpdateHash(in, i, inend, h); - - ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, dummysublen, - &dist, &leng); - lengthscore = GetLengthScore(leng, dist); - -#ifdef ZOPFLI_LAZY_MATCHING - /* Lazy matching. */ - prevlengthscore = GetLengthScore(prev_length, prev_match); - if (match_available) { - match_available = 0; - if (lengthscore > prevlengthscore + 1) { - ZopfliStoreLitLenDist(in[i - 1], 0, store); - if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { - match_available = 1; - prev_length = leng; - prev_match = dist; - continue; - } - } else { - /* Add previous to output. */ - leng = prev_length; - dist = prev_match; - lengthscore = prevlengthscore; - /* Add to output. */ - ZopfliVerifyLenDist(in, inend, i - 1, dist, leng); - ZopfliStoreLitLenDist(leng, dist, store); - for (j = 2; j < leng; j++) { - assert(i < inend); - i++; - ZopfliUpdateHash(in, i, inend, h); - } - continue; - } - } - else if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { - match_available = 1; - prev_length = leng; - prev_match = dist; - continue; - } - /* End of lazy matching. */ -#endif - - /* Add to output. */ - if (lengthscore >= ZOPFLI_MIN_MATCH) { - ZopfliVerifyLenDist(in, inend, i, dist, leng); - ZopfliStoreLitLenDist(leng, dist, store); - } else { - leng = 1; - ZopfliStoreLitLenDist(in[i], 0, store); - } - for (j = 1; j < leng; j++) { - assert(i < inend); - i++; - ZopfliUpdateHash(in, i, inend, h); - } - } - - ZopfliCleanHash(h); -} - -void ZopfliLZ77Counts(const unsigned short* litlens, - const unsigned short* dists, - size_t start, size_t end, - size_t* ll_count, size_t* d_count) { - size_t i; - - for (i = 0; i < 288; i++) { - ll_count[i] = 0; - } - for (i = 0; i < 32; i++) { - d_count[i] = 0; - } - - for (i = start; i < end; i++) { - if (dists[i] == 0) { - ll_count[litlens[i]]++; - } else { - ll_count[ZopfliGetLengthSymbol(litlens[i])]++; - d_count[ZopfliGetDistSymbol(dists[i])]++; - } - } - - ll_count[256] = 1; /* End symbol. */ -} diff --git a/lz77.h b/lz77.h deleted file mode 100644 index 55186a7..0000000 --- a/lz77.h +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -Functions for basic LZ77 compression and utilities for the "squeeze" LZ77 -compression. -*/ - -#ifndef ZOPFLI_LZ77_H_ -#define ZOPFLI_LZ77_H_ - -#include - -#include "cache.h" -#include "hash.h" -#include "zopfli.h" - -/* -Stores lit/length and dist pairs for LZ77. -Parameter litlens: Contains the literal symbols or length values. -Parameter dists: Contains the distances. A value is 0 to indicate that there is -no dist and the corresponding litlens value is a literal instead of a length. -Parameter size: The size of both the litlens and dists arrays. -The memory can best be managed by using ZopfliInitLZ77Store to initialize it, -ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values. - -*/ -typedef struct ZopfliLZ77Store { - unsigned short* litlens; /* Lit or len. */ - unsigned short* dists; /* If 0: indicates literal in corresponding litlens, - if > 0: length in corresponding litlens, this is the distance. */ - size_t size; -} ZopfliLZ77Store; - -void ZopfliInitLZ77Store(ZopfliLZ77Store* store); -void ZopfliCleanLZ77Store(ZopfliLZ77Store* store); -void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest); -void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist, - ZopfliLZ77Store* store); - -/* -Some state information for compressing a block. -This is currently a bit under-used (with mainly only the longest match cache), -but is kept for easy future expansion. -*/ -typedef struct ZopfliBlockState { - const ZopfliOptions* options; - -#ifdef ZOPFLI_LONGEST_MATCH_CACHE - /* Cache for length/distance pairs found so far. */ - ZopfliLongestMatchCache* lmc; -#endif - - /* The start (inclusive) and end (not inclusive) of the current block. */ - size_t blockstart; - size_t blockend; -} ZopfliBlockState; - -/* -Finds the longest match (length and corresponding distance) for LZ77 -compression. -Even when not using "sublen", it can be more efficient to provide an array, -because only then the caching is used. -array: the data -pos: position in the data to find the match for -size: size of the data -limit: limit length to maximum this value (default should be 258). This allows - finding a shorter dist for that length (= less extra bits). Must be - in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH]. -sublen: output array of 259 elements, or null. Has, for each length, the - smallest distance required to reach this length. Only 256 of its 259 values - are used, the first 3 are ignored (the shortest length is 3. It is purely - for convenience that the array is made 3 longer). -*/ -void ZopfliFindLongestMatch( - ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array, - size_t pos, size_t size, size_t limit, - unsigned short* sublen, unsigned short* distance, unsigned short* length); - -/* -Verifies if length and dist are indeed valid, only used for assertion. -*/ -void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos, - unsigned short dist, unsigned short length); - -/* -Counts the number of literal, length and distance symbols in the given lz77 -arrays. -litlens: lz77 lit/lengths -dists: ll77 distances -start: where to begin counting in litlens and dists -end: where to stop counting in litlens and dists (not inclusive) -ll_count: count of each lit/len symbol, must have size 288 (see deflate - standard) -d_count: count of each dist symbol, must have size 32 (see deflate standard) -*/ -void ZopfliLZ77Counts(const unsigned short* litlens, - const unsigned short* dists, - size_t start, size_t end, - size_t* ll_count, size_t* d_count); - -/* -Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than -with the slow but better "squeeze" implementation. -The result is placed in the ZopfliLZ77Store. -If instart is larger than 0, it uses values before instart as starting -dictionary. -*/ -void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, - size_t instart, size_t inend, - ZopfliLZ77Store* store); - -#endif /* ZOPFLI_LZ77_H_ */ diff --git a/squeeze.c b/squeeze.c deleted file mode 100644 index 09e7e2e..0000000 --- a/squeeze.c +++ /dev/null @@ -1,546 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "squeeze.h" - -#include -#include -#include - -#include "blocksplitter.h" -#include "deflate.h" -#include "tree.h" -#include "util.h" - -typedef struct SymbolStats { - /* The literal and length symbols. */ - size_t litlens[288]; - /* The 32 unique dist symbols, not the 32768 possible dists. */ - size_t dists[32]; - - double ll_symbols[288]; /* Length of each lit/len symbol in bits. */ - double d_symbols[32]; /* Length of each dist symbol in bits. */ -} SymbolStats; - -/* Sets everything to 0. */ -static void InitStats(SymbolStats* stats) { - memset(stats->litlens, 0, 288 * sizeof(stats->litlens[0])); - memset(stats->dists, 0, 32 * sizeof(stats->dists[0])); - - memset(stats->ll_symbols, 0, 288 * sizeof(stats->ll_symbols[0])); - memset(stats->d_symbols, 0, 32 * sizeof(stats->d_symbols[0])); -} - -static void CopyStats(SymbolStats* source, SymbolStats* dest) { - memcpy(dest->litlens, source->litlens, 288 * sizeof(dest->litlens[0])); - memcpy(dest->dists, source->dists, 32 * sizeof(dest->dists[0])); - - memcpy(dest->ll_symbols, source->ll_symbols, - 288 * sizeof(dest->ll_symbols[0])); - memcpy(dest->d_symbols, source->d_symbols, 32 * sizeof(dest->d_symbols[0])); -} - -/* Adds the bit lengths. */ -static void AddWeighedStatFreqs(const SymbolStats* stats1, double w1, - const SymbolStats* stats2, double w2, - SymbolStats* result) { - size_t i; - for (i = 0; i < 288; i++) { - result->litlens[i] = - (size_t) (stats1->litlens[i] * w1 + stats2->litlens[i] * w2); - } - for (i = 0; i < 32; i++) { - result->dists[i] = - (size_t) (stats1->dists[i] * w1 + stats2->dists[i] * w2); - } - result->litlens[256] = 1; /* End symbol. */ -} - -typedef struct RanState { - unsigned int m_w, m_z; -} RanState; - -static void InitRanState(RanState* state) { - state->m_w = 1; - state->m_z = 2; -} - -/* Get random number: "Multiply-With-Carry" generator of G. Marsaglia */ -static unsigned int Ran(RanState* state) { - state->m_z = 36969 * (state->m_z & 65535) + (state->m_z >> 16); - state->m_w = 18000 * (state->m_w & 65535) + (state->m_w >> 16); - return (state->m_z << 16) + state->m_w; /* 32-bit result. */ -} - -static void RandomizeFreqs(RanState* state, size_t* freqs, int n) { - int i; - for (i = 0; i < n; i++) { - if ((Ran(state) >> 4) % 3 == 0) freqs[i] = freqs[Ran(state) % n]; - } -} - -static void RandomizeStatFreqs(RanState* state, SymbolStats* stats) { - RandomizeFreqs(state, stats->litlens, 288); - RandomizeFreqs(state, stats->dists, 32); - stats->litlens[256] = 1; /* End symbol. */ -} - -static void ClearStatFreqs(SymbolStats* stats) { - size_t i; - for (i = 0; i < 288; i++) stats->litlens[i] = 0; - for (i = 0; i < 32; i++) stats->dists[i] = 0; -} - -/* -Function that calculates a cost based on a model for the given LZ77 symbol. -litlen: means literal symbol if dist is 0, length otherwise. -*/ -typedef double CostModelFun(unsigned litlen, unsigned dist, void* context); - -/* -Cost model which should exactly match fixed tree. -type: CostModelFun -*/ -static double GetCostFixed(unsigned litlen, unsigned dist, void* unused) { - (void)unused; - if (dist == 0) { - if (litlen <= 143) return 8; - else return 9; - } else { - int dbits = ZopfliGetDistExtraBits(dist); - int lbits = ZopfliGetLengthExtraBits(litlen); - int lsym = ZopfliGetLengthSymbol(litlen); - double cost = 0; - if (lsym <= 279) cost += 7; - else cost += 8; - cost += 5; /* Every dist symbol has length 5. */ - return cost + dbits + lbits; - } -} - -/* -Cost model based on symbol statistics. -type: CostModelFun -*/ -static double GetCostStat(unsigned litlen, unsigned dist, void* context) { - SymbolStats* stats = (SymbolStats*)context; - if (dist == 0) { - return stats->ll_symbols[litlen]; - } else { - int lsym = ZopfliGetLengthSymbol(litlen); - int lbits = ZopfliGetLengthExtraBits(litlen); - int dsym = ZopfliGetDistSymbol(dist); - int dbits = ZopfliGetDistExtraBits(dist); - return stats->ll_symbols[lsym] + lbits + stats->d_symbols[dsym] + dbits; - } -} - -/* -Finds the minimum possible cost this cost model can return for valid length and -distance symbols. -*/ -static double GetCostModelMinCost(CostModelFun* costmodel, void* costcontext) { - double mincost; - int bestlength = 0; /* length that has lowest cost in the cost model */ - int bestdist = 0; /* distance that has lowest cost in the cost model */ - int i; - /* - Table of distances that have a different distance symbol in the deflate - specification. Each value is the first distance that has a new symbol. Only - different symbols affect the cost model so only these need to be checked. - See RFC 1951 section 3.2.5. Compressed blocks (length and distance codes). - */ - static const int dsymbols[30] = { - 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, - 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 - }; - - mincost = ZOPFLI_LARGE_FLOAT; - for (i = 3; i < 259; i++) { - double c = costmodel(i, 1, costcontext); - if (c < mincost) { - bestlength = i; - mincost = c; - } - } - - mincost = ZOPFLI_LARGE_FLOAT; - for (i = 0; i < 30; i++) { - double c = costmodel(3, dsymbols[i], costcontext); - if (c < mincost) { - bestdist = dsymbols[i]; - mincost = c; - } - } - - return costmodel(bestlength, bestdist, costcontext); -} - -/* -Performs the forward pass for "squeeze". Gets the most optimal length to reach -every byte from a previous byte, using cost calculations. -s: the ZopfliBlockState -in: the input data array -instart: where to start -inend: where to stop (not inclusive) -costmodel: function to calculate the cost of some lit/len/dist pair. -costcontext: abstract context for the costmodel function -length_array: output array of size (inend - instart) which will receive the best - length to reach this byte from a previous byte. -returns the cost that was, according to the costmodel, needed to get to the end. -*/ -static double GetBestLengths(ZopfliBlockState *s, - const unsigned char* in, - size_t instart, size_t inend, - CostModelFun* costmodel, void* costcontext, - unsigned short* length_array) { - /* Best cost to get here so far. */ - size_t blocksize = inend - instart; - float* costs; - size_t i = 0, k; - unsigned short leng; - unsigned short dist; - unsigned short sublen[259]; - size_t windowstart = instart > ZOPFLI_WINDOW_SIZE - ? instart - ZOPFLI_WINDOW_SIZE : 0; - ZopfliHash hash; - ZopfliHash* h = &hash; - double result; - double mincost = GetCostModelMinCost(costmodel, costcontext); - - if (instart == inend) return 0; - - costs = (float*)malloc(sizeof(float) * (blocksize + 1)); - if (!costs) exit(-1); /* Allocation failed. */ - - ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h); - ZopfliWarmupHash(in, windowstart, inend, h); - for (i = windowstart; i < instart; i++) { - ZopfliUpdateHash(in, i, inend, h); - } - - for (i = 1; i < blocksize + 1; i++) costs[i] = ZOPFLI_LARGE_FLOAT; - costs[0] = 0; /* Because it's the start. */ - length_array[0] = 0; - - for (i = instart; i < inend; i++) { - size_t j = i - instart; /* Index in the costs array and length_array. */ - ZopfliUpdateHash(in, i, inend, h); - -#ifdef ZOPFLI_SHORTCUT_LONG_REPETITIONS - /* If we're in a long repetition of the same character and have more than - ZOPFLI_MAX_MATCH characters before and after our position. */ - if (h->same[i & ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH * 2 - && i > instart + ZOPFLI_MAX_MATCH + 1 - && i + ZOPFLI_MAX_MATCH * 2 + 1 < inend - && h->same[(i - ZOPFLI_MAX_MATCH) & ZOPFLI_WINDOW_MASK] - > ZOPFLI_MAX_MATCH) { - double symbolcost = costmodel(ZOPFLI_MAX_MATCH, 1, costcontext); - /* Set the length to reach each one to ZOPFLI_MAX_MATCH, and the cost to - the cost corresponding to that length. Doing this, we skip - ZOPFLI_MAX_MATCH values to avoid calling ZopfliFindLongestMatch. */ - for (k = 0; k < ZOPFLI_MAX_MATCH; k++) { - costs[j + ZOPFLI_MAX_MATCH] = costs[j] + symbolcost; - length_array[j + ZOPFLI_MAX_MATCH] = ZOPFLI_MAX_MATCH; - i++; - j++; - ZopfliUpdateHash(in, i, inend, h); - } - } -#endif - - ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, sublen, - &dist, &leng); - - /* Literal. */ - if (i + 1 <= inend) { - double newCost = costs[j] + costmodel(in[i], 0, costcontext); - assert(newCost >= 0); - if (newCost < costs[j + 1]) { - costs[j + 1] = newCost; - length_array[j + 1] = 1; - } - } - /* Lengths. */ - for (k = 3; k <= leng && i + k <= inend; k++) { - double newCost; - - /* Calling the cost model is expensive, avoid this if we are already at - the minimum possible cost that it can return. */ - if (costs[j + k] - costs[j] <= mincost) continue; - - newCost = costs[j] + costmodel(k, sublen[k], costcontext); - assert(newCost >= 0); - if (newCost < costs[j + k]) { - assert(k <= ZOPFLI_MAX_MATCH); - costs[j + k] = newCost; - length_array[j + k] = k; - } - } - } - - assert(costs[blocksize] >= 0); - result = costs[blocksize]; - - ZopfliCleanHash(h); - free(costs); - - return result; -} - -/* -Calculates the optimal path of lz77 lengths to use, from the calculated -length_array. The length_array must contain the optimal length to reach that -byte. The path will be filled with the lengths to use, so its data size will be -the amount of lz77 symbols. -*/ -static void TraceBackwards(size_t size, const unsigned short* length_array, - unsigned short** path, size_t* pathsize) { - size_t index = size; - if (size == 0) return; - for (;;) { - ZOPFLI_APPEND_DATA(length_array[index], path, pathsize); - assert(length_array[index] <= index); - assert(length_array[index] <= ZOPFLI_MAX_MATCH); - assert(length_array[index] != 0); - index -= length_array[index]; - if (index == 0) break; - } - - /* Mirror result. */ - for (index = 0; index < *pathsize / 2; index++) { - unsigned short temp = (*path)[index]; - (*path)[index] = (*path)[*pathsize - index - 1]; - (*path)[*pathsize - index - 1] = temp; - } -} - -static void FollowPath(ZopfliBlockState* s, - const unsigned char* in, size_t instart, size_t inend, - unsigned short* path, size_t pathsize, - ZopfliLZ77Store* store) { - size_t i, j, pos = 0; - size_t windowstart = instart > ZOPFLI_WINDOW_SIZE - ? instart - ZOPFLI_WINDOW_SIZE : 0; - - size_t total_length_test = 0; - - ZopfliHash hash; - ZopfliHash* h = &hash; - - if (instart == inend) return; - - ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h); - ZopfliWarmupHash(in, windowstart, inend, h); - for (i = windowstart; i < instart; i++) { - ZopfliUpdateHash(in, i, inend, h); - } - - pos = instart; - for (i = 0; i < pathsize; i++) { - unsigned short length = path[i]; - unsigned short dummy_length; - unsigned short dist; - assert(pos < inend); - - ZopfliUpdateHash(in, pos, inend, h); - - /* Add to output. */ - if (length >= ZOPFLI_MIN_MATCH) { - /* Get the distance by recalculating longest match. The found length - should match the length from the path. */ - ZopfliFindLongestMatch(s, h, in, pos, inend, length, 0, - &dist, &dummy_length); - assert(!(dummy_length != length && length > 2 && dummy_length > 2)); - ZopfliVerifyLenDist(in, inend, pos, dist, length); - ZopfliStoreLitLenDist(length, dist, store); - total_length_test += length; - } else { - length = 1; - ZopfliStoreLitLenDist(in[pos], 0, store); - total_length_test++; - } - - - assert(pos + length <= inend); - for (j = 1; j < length; j++) { - ZopfliUpdateHash(in, pos + j, inend, h); - } - - pos += length; - } - - ZopfliCleanHash(h); -} - -/* Calculates the entropy of the statistics */ -static void CalculateStatistics(SymbolStats* stats) { - ZopfliCalculateEntropy(stats->litlens, 288, stats->ll_symbols); - ZopfliCalculateEntropy(stats->dists, 32, stats->d_symbols); -} - -/* Appends the symbol statistics from the store. */ -static void GetStatistics(const ZopfliLZ77Store* store, SymbolStats* stats) { - size_t i; - for (i = 0; i < store->size; i++) { - if (store->dists[i] == 0) { - stats->litlens[store->litlens[i]]++; - } else { - stats->litlens[ZopfliGetLengthSymbol(store->litlens[i])]++; - stats->dists[ZopfliGetDistSymbol(store->dists[i])]++; - } - } - stats->litlens[256] = 1; /* End symbol. */ - - CalculateStatistics(stats); -} - -/* -Does a single run for ZopfliLZ77Optimal. For good compression, repeated runs -with updated statistics should be performed. - -s: the block state -in: the input data array -instart: where to start -inend: where to stop (not inclusive) -path: pointer to dynamically allocated memory to store the path -pathsize: pointer to the size of the dynamic path array -length_array: array if size (inend - instart) used to store lengths -costmodel: function to use as the cost model for this squeeze run -costcontext: abstract context for the costmodel function -store: place to output the LZ77 data -returns the cost that was, according to the costmodel, needed to get to the end. - This is not the actual cost. -*/ -static double LZ77OptimalRun(ZopfliBlockState* s, - const unsigned char* in, size_t instart, size_t inend, - unsigned short** path, size_t* pathsize, - unsigned short* length_array, CostModelFun* costmodel, - void* costcontext, ZopfliLZ77Store* store) { - double cost = GetBestLengths( - s, in, instart, inend, costmodel, costcontext, length_array); - free(*path); - *path = 0; - *pathsize = 0; - TraceBackwards(inend - instart, length_array, path, pathsize); - FollowPath(s, in, instart, inend, *path, *pathsize, store); - assert(cost < ZOPFLI_LARGE_FLOAT); - return cost; -} - -void ZopfliLZ77Optimal(ZopfliBlockState *s, - const unsigned char* in, size_t instart, size_t inend, - ZopfliLZ77Store* store) { - /* Dist to get to here with smallest cost. */ - size_t blocksize = inend - instart; - unsigned short* length_array = - (unsigned short*)malloc(sizeof(unsigned short) * (blocksize + 1)); - unsigned short* path = 0; - size_t pathsize = 0; - ZopfliLZ77Store currentstore; - SymbolStats stats, beststats, laststats; - int i; - double cost; - double bestcost = ZOPFLI_LARGE_FLOAT; - double lastcost = 0; - /* Try randomizing the costs a bit once the size stabilizes. */ - RanState ran_state; - int lastrandomstep = -1; - - if (!length_array) exit(-1); /* Allocation failed. */ - - InitRanState(&ran_state); - InitStats(&stats); - ZopfliInitLZ77Store(¤tstore); - - /* Do regular deflate, then loop multiple shortest path runs, each time using - the statistics of the previous run. */ - - /* Initial run. */ - ZopfliLZ77Greedy(s, in, instart, inend, ¤tstore); - GetStatistics(¤tstore, &stats); - - /* Repeat statistics with each time the cost model from the previous stat - run. */ - for (i = 0; i < s->options->numiterations; i++) { - ZopfliCleanLZ77Store(¤tstore); - ZopfliInitLZ77Store(¤tstore); - LZ77OptimalRun(s, in, instart, inend, &path, &pathsize, - length_array, GetCostStat, (void*)&stats, - ¤tstore); - cost = ZopfliCalculateBlockSize(currentstore.litlens, currentstore.dists, - 0, currentstore.size, 2); - if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { - fprintf(stderr, "Iteration %d: %d bit\n", i, (int) cost); - } - if (cost < bestcost) { - /* Copy to the output store. */ - ZopfliCopyLZ77Store(¤tstore, store); - CopyStats(&stats, &beststats); - bestcost = cost; - } - CopyStats(&stats, &laststats); - ClearStatFreqs(&stats); - GetStatistics(¤tstore, &stats); - if (lastrandomstep != -1) { - /* This makes it converge slower but better. Do it only once the - randomness kicks in so that if the user does few iterations, it gives a - better result sooner. */ - AddWeighedStatFreqs(&stats, 1.0, &laststats, 0.5, &stats); - CalculateStatistics(&stats); - } - if (i > 5 && cost == lastcost) { - CopyStats(&beststats, &stats); - RandomizeStatFreqs(&ran_state, &stats); - CalculateStatistics(&stats); - lastrandomstep = i; - } - lastcost = cost; - } - - free(length_array); - free(path); - ZopfliCleanLZ77Store(¤tstore); -} - -void ZopfliLZ77OptimalFixed(ZopfliBlockState *s, - const unsigned char* in, - size_t instart, size_t inend, - ZopfliLZ77Store* store) -{ - /* Dist to get to here with smallest cost. */ - size_t blocksize = inend - instart; - unsigned short* length_array = - (unsigned short*)malloc(sizeof(unsigned short) * (blocksize + 1)); - unsigned short* path = 0; - size_t pathsize = 0; - - if (!length_array) exit(-1); /* Allocation failed. */ - - s->blockstart = instart; - s->blockend = inend; - - /* Shortest path for fixed tree This one should give the shortest possible - result for fixed tree, no repeated runs are needed since the tree is known. */ - LZ77OptimalRun(s, in, instart, inend, &path, &pathsize, - length_array, GetCostFixed, 0, store); - - free(length_array); - free(path); -} diff --git a/squeeze.h b/squeeze.h deleted file mode 100644 index e850aaa..0000000 --- a/squeeze.h +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -The squeeze functions do enhanced LZ77 compression by optimal parsing with a -cost model, rather than greedily choosing the longest length or using a single -step of lazy matching like regular implementations. - -Since the cost model is based on the Huffman tree that can only be calculated -after the LZ77 data is generated, there is a chicken and egg problem, and -multiple runs are done with updated cost models to converge to a better -solution. -*/ - -#ifndef ZOPFLI_SQUEEZE_H_ -#define ZOPFLI_SQUEEZE_H_ - -#include "lz77.h" - -/* -Calculates lit/len and dist pairs for given data. -If instart is larger than 0, it uses values before instart as starting -dictionary. -*/ -void ZopfliLZ77Optimal(ZopfliBlockState *s, - const unsigned char* in, size_t instart, size_t inend, - ZopfliLZ77Store* store); - -/* -Does the same as ZopfliLZ77Optimal, but optimized for the fixed tree of the -deflate standard. -The fixed tree never gives the best compression. But this gives the best -possible LZ77 encoding possible with the fixed tree. -This does not create or output any fixed tree, only LZ77 data optimized for -using with a fixed tree. -If instart is larger than 0, it uses values before instart as starting -dictionary. -*/ -void ZopfliLZ77OptimalFixed(ZopfliBlockState *s, - const unsigned char* in, - size_t instart, size_t inend, - ZopfliLZ77Store* store); - -#endif /* ZOPFLI_SQUEEZE_H_ */ diff --git a/tree.c b/tree.c deleted file mode 100644 index c457511..0000000 --- a/tree.c +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "tree.h" - -#include -#include -#include -#include - -#include "katajainen.h" -#include "util.h" - -void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits, - unsigned* symbols) { - size_t* bl_count = (size_t*)malloc(sizeof(size_t) * (maxbits + 1)); - size_t* next_code = (size_t*)malloc(sizeof(size_t) * (maxbits + 1)); - unsigned bits, i; - unsigned code; - - for (i = 0; i < n; i++) { - symbols[i] = 0; - } - - /* 1) Count the number of codes for each code length. Let bl_count[N] be the - number of codes of length N, N >= 1. */ - for (bits = 0; bits <= maxbits; bits++) { - bl_count[bits] = 0; - } - for (i = 0; i < n; i++) { - assert(lengths[i] <= maxbits); - bl_count[lengths[i]]++; - } - /* 2) Find the numerical value of the smallest code for each code length. */ - code = 0; - bl_count[0] = 0; - for (bits = 1; bits <= maxbits; bits++) { - code = (code + bl_count[bits-1]) << 1; - next_code[bits] = code; - } - /* 3) Assign numerical values to all codes, using consecutive values for all - codes of the same length with the base values determined at step 2. */ - for (i = 0; i < n; i++) { - unsigned len = lengths[i]; - if (len != 0) { - symbols[i] = next_code[len]; - next_code[len]++; - } - } - - free(bl_count); - free(next_code); -} - -void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths) { - static const double kInvLog2 = 1.4426950408889; /* 1.0 / log(2.0) */ - unsigned sum = 0; - unsigned i; - double log2sum; - for (i = 0; i < n; ++i) { - sum += count[i]; - } - log2sum = (sum == 0 ? log(n) : log(sum)) * kInvLog2; - for (i = 0; i < n; ++i) { - /* When the count of the symbol is 0, but its cost is requested anyway, it - means the symbol will appear at least once anyway, so give it the cost as if - its count is 1.*/ - if (count[i] == 0) bitlengths[i] = log2sum; - else bitlengths[i] = log2sum - log(count[i]) * kInvLog2; - /* Depending on compiler and architecture, the above subtraction of two - floating point numbers may give a negative result very close to zero - instead of zero (e.g. -5.973954e-17 with gcc 4.1.2 on Ubuntu 11.4). Clamp - it to zero. These floating point imprecisions do not affect the cost model - significantly so this is ok. */ - if (bitlengths[i] < 0 && bitlengths[i] > -1e-5) bitlengths[i] = 0; - assert(bitlengths[i] >= 0); - } -} - -void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits, - unsigned* bitlengths) { - int error = ZopfliLengthLimitedCodeLengths(count, n, maxbits, bitlengths); - (void) error; - assert(!error); -} diff --git a/tree.h b/tree.h deleted file mode 100644 index 4d6f469..0000000 --- a/tree.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -Utilities for creating and using Huffman trees. -*/ - -#ifndef ZOPFLI_TREE_H_ -#define ZOPFLI_TREE_H_ - -#include - -/* -Calculates the bitlengths for the Huffman tree, based on the counts of each -symbol. -*/ -void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits, - unsigned *bitlengths); - -/* -Converts a series of Huffman tree bitlengths, to the bit values of the symbols. -*/ -void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits, - unsigned* symbols); - -/* -Calculates the entropy of each symbol, based on the counts of each symbol. The -result is similar to the result of ZopfliCalculateBitLengths, but with the -actual theoritical bit lengths according to the entropy. Since the resulting -values are fractional, they cannot be used to encode the tree specified by -DEFLATE. -*/ -void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths); - -#endif /* ZOPFLI_TREE_H_ */ diff --git a/util.c b/util.c deleted file mode 100644 index d207145..0000000 --- a/util.c +++ /dev/null @@ -1,213 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "util.h" - -#include "zopfli.h" - -#include -#include -#include - -int ZopfliGetDistExtraBits(int dist) { -#ifdef __GNUC__ - if (dist < 5) return 0; - return (31 ^ __builtin_clz(dist - 1)) - 1; /* log2(dist - 1) - 1 */ -#else - if (dist < 5) return 0; - else if (dist < 9) return 1; - else if (dist < 17) return 2; - else if (dist < 33) return 3; - else if (dist < 65) return 4; - else if (dist < 129) return 5; - else if (dist < 257) return 6; - else if (dist < 513) return 7; - else if (dist < 1025) return 8; - else if (dist < 2049) return 9; - else if (dist < 4097) return 10; - else if (dist < 8193) return 11; - else if (dist < 16385) return 12; - else return 13; -#endif -} - -int ZopfliGetDistExtraBitsValue(int dist) { -#ifdef __GNUC__ - if (dist < 5) { - return 0; - } else { - int l = 31 ^ __builtin_clz(dist - 1); /* log2(dist - 1) */ - return (dist - (1 + (1 << l))) & ((1 << (l - 1)) - 1); - } -#else - if (dist < 5) return 0; - else if (dist < 9) return (dist - 5) & 1; - else if (dist < 17) return (dist - 9) & 3; - else if (dist < 33) return (dist - 17) & 7; - else if (dist < 65) return (dist - 33) & 15; - else if (dist < 129) return (dist - 65) & 31; - else if (dist < 257) return (dist - 129) & 63; - else if (dist < 513) return (dist - 257) & 127; - else if (dist < 1025) return (dist - 513) & 255; - else if (dist < 2049) return (dist - 1025) & 511; - else if (dist < 4097) return (dist - 2049) & 1023; - else if (dist < 8193) return (dist - 4097) & 2047; - else if (dist < 16385) return (dist - 8193) & 4095; - else return (dist - 16385) & 8191; -#endif -} - -int ZopfliGetDistSymbol(int dist) { -#ifdef __GNUC__ - if (dist < 5) { - return dist - 1; - } else { - int l = (31 ^ __builtin_clz(dist - 1)); /* log2(dist - 1) */ - int r = ((dist - 1) >> (l - 1)) & 1; - return l * 2 + r; - } -#else - if (dist < 193) { - if (dist < 13) { /* dist 0..13. */ - if (dist < 5) return dist - 1; - else if (dist < 7) return 4; - else if (dist < 9) return 5; - else return 6; - } else { /* dist 13..193. */ - if (dist < 17) return 7; - else if (dist < 25) return 8; - else if (dist < 33) return 9; - else if (dist < 49) return 10; - else if (dist < 65) return 11; - else if (dist < 97) return 12; - else if (dist < 129) return 13; - else return 14; - } - } else { - if (dist < 2049) { /* dist 193..2049. */ - if (dist < 257) return 15; - else if (dist < 385) return 16; - else if (dist < 513) return 17; - else if (dist < 769) return 18; - else if (dist < 1025) return 19; - else if (dist < 1537) return 20; - else return 21; - } else { /* dist 2049..32768. */ - if (dist < 3073) return 22; - else if (dist < 4097) return 23; - else if (dist < 6145) return 24; - else if (dist < 8193) return 25; - else if (dist < 12289) return 26; - else if (dist < 16385) return 27; - else if (dist < 24577) return 28; - else return 29; - } - } -#endif -} - -int ZopfliGetLengthExtraBits(int l) { - static const int table[259] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0 - }; - return table[l]; -} - -int ZopfliGetLengthExtraBitsValue(int l) { - static const int table[259] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0, - 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, - 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0 - }; - return table[l]; -} - -/* -Returns symbol in range [257-285] (inclusive). -*/ -int ZopfliGetLengthSymbol(int l) { - static const int table[259] = { - 0, 0, 0, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 265, 266, 266, 267, 267, 268, 268, - 269, 269, 269, 269, 270, 270, 270, 270, - 271, 271, 271, 271, 272, 272, 272, 272, - 273, 273, 273, 273, 273, 273, 273, 273, - 274, 274, 274, 274, 274, 274, 274, 274, - 275, 275, 275, 275, 275, 275, 275, 275, - 276, 276, 276, 276, 276, 276, 276, 276, - 277, 277, 277, 277, 277, 277, 277, 277, - 277, 277, 277, 277, 277, 277, 277, 277, - 278, 278, 278, 278, 278, 278, 278, 278, - 278, 278, 278, 278, 278, 278, 278, 278, - 279, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 279, 279, - 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 280, - 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, - 282, 282, 282, 282, 282, 282, 282, 282, - 282, 282, 282, 282, 282, 282, 282, 282, - 282, 282, 282, 282, 282, 282, 282, 282, - 282, 282, 282, 282, 282, 282, 282, 282, - 283, 283, 283, 283, 283, 283, 283, 283, - 283, 283, 283, 283, 283, 283, 283, 283, - 283, 283, 283, 283, 283, 283, 283, 283, - 283, 283, 283, 283, 283, 283, 283, 283, - 284, 284, 284, 284, 284, 284, 284, 284, - 284, 284, 284, 284, 284, 284, 284, 284, - 284, 284, 284, 284, 284, 284, 284, 284, - 284, 284, 284, 284, 284, 284, 284, 285 - }; - return table[l]; -} - -void ZopfliInitOptions(ZopfliOptions* options) { - options->verbose = 0; - options->verbose_more = 0; - options->numiterations = 15; - options->blocksplitting = 1; - options->blocksplittinglast = 0; - options->blocksplittingmax = 15; -} diff --git a/util.h b/util.h deleted file mode 100644 index 4188f51..0000000 --- a/util.h +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -Several utilities, including: #defines to try different compression results, -basic deflate specification values and generic program options. -*/ - -#ifndef ZOPFLI_UTIL_H_ -#define ZOPFLI_UTIL_H_ - -#include -#include - -/* Minimum and maximum length that can be encoded in deflate. */ -#define ZOPFLI_MAX_MATCH 258 -#define ZOPFLI_MIN_MATCH 3 - -/* -The window size for deflate. Must be a power of two. This should be 32768, the -maximum possible by the deflate spec. Anything less hurts compression more than -speed. -*/ -#define ZOPFLI_WINDOW_SIZE 32768 - -/* -The window mask used to wrap indices into the window. This is why the -window size must be a power of two. -*/ -#define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1) - -/* -A block structure of huge, non-smart, blocks to divide the input into, to allow -operating on huge files without exceeding memory, such as the 1GB wiki9 corpus. -The whole compression algorithm, including the smarter block splitting, will -be executed independently on each huge block. -Dividing into huge blocks hurts compression, but not much relative to the size. -Set this to, for example, 20MB (20000000). Set it to 0 to disable master blocks. -*/ -#define ZOPFLI_MASTER_BLOCK_SIZE 20000000 - -/* -Used to initialize costs for example -*/ -#define ZOPFLI_LARGE_FLOAT 1e30 - -/* -For longest match cache. max 256. Uses huge amounts of memory but makes it -faster. Uses this many times three bytes per single byte of the input data. -This is so because longest match finding has to find the exact distance -that belongs to each length for the best lz77 strategy. -Good values: e.g. 5, 8. -*/ -#define ZOPFLI_CACHE_LENGTH 8 - -/* -limit the max hash chain hits for this hash value. This has an effect only -on files where the hash value is the same very often. On these files, this -gives worse compression (the value should ideally be 32768, which is the -ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it -faster on some specific files. -Good value: e.g. 8192. -*/ -#define ZOPFLI_MAX_CHAIN_HITS 8192 - -/* -Whether to use the longest match cache for ZopfliFindLongestMatch. This cache -consumes a lot of memory but speeds it up. No effect on compression size. -*/ -#define ZOPFLI_LONGEST_MATCH_CACHE - -/* -Enable to remember amount of successive identical bytes in the hash chain for -finding longest match -required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS -This has no effect on the compression result, and enabling it increases speed. -*/ -#define ZOPFLI_HASH_SAME - -/* -Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the -best length so far is long enough. This is way faster for files with lots of -identical bytes, on which the compressor is otherwise too slow. Regular files -are unaffected or maybe a tiny bit slower. -This has no effect on the compression result, only on speed. -*/ -#define ZOPFLI_HASH_SAME_HASH - -/* -Enable this, to avoid slowness for files which are a repetition of the same -character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect -the compression result. -*/ -#define ZOPFLI_SHORTCUT_LONG_REPETITIONS - -/* -Whether to use lazy matching in the greedy LZ77 implementation. This gives a -better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77 -varies from file to file. -*/ -#define ZOPFLI_LAZY_MATCHING - -/* -Gets the symbol for the given length, cfr. the DEFLATE spec. -Returns the symbol in the range [257-285] (inclusive) -*/ -int ZopfliGetLengthSymbol(int l); - -/* Gets the amount of extra bits for the given length, cfr. the DEFLATE spec. */ -int ZopfliGetLengthExtraBits(int l); - -/* Gets value of the extra bits for the given length, cfr. the DEFLATE spec. */ -int ZopfliGetLengthExtraBitsValue(int l); - -/* Gets the symbol for the given dist, cfr. the DEFLATE spec. */ -int ZopfliGetDistSymbol(int dist); - -/* Gets the amount of extra bits for the given dist, cfr. the DEFLATE spec. */ -int ZopfliGetDistExtraBits(int dist); - -/* Gets value of the extra bits for the given dist, cfr. the DEFLATE spec. */ -int ZopfliGetDistExtraBitsValue(int dist); - -/* -Appends value to dynamically allocated memory, doubling its allocation size -whenever needed. - -value: the value to append, type T -data: pointer to the dynamic array to append to, type T** -size: pointer to the size of the array to append to, type size_t*. This is the -size that you consider the array to be, not the internal allocation size. -Precondition: allocated size of data is at least a power of two greater than or -equal than *size. -*/ -#ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */ -#define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\ - if (!((*size) & ((*size) - 1))) {\ - /*double alloc size if it's a power of two*/\ - void** data_void = reinterpret_cast(data);\ - *data_void = (*size) == 0 ? malloc(sizeof(**data))\ - : realloc((*data), (*size) * 2 * sizeof(**data));\ - }\ - (*data)[(*size)] = (value);\ - (*size)++;\ -} -#else /* C gives problems with strict-aliasing rules for (void**) cast */ -#define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\ - if (!((*size) & ((*size) - 1))) {\ - /*double alloc size if it's a power of two*/\ - (*data) = (*size) == 0 ? malloc(sizeof(**data))\ - : realloc((*data), (*size) * 2 * sizeof(**data));\ - }\ - (*data)[(*size)] = (value);\ - (*size)++;\ -} -#endif - - -#endif /* ZOPFLI_UTIL_H_ */ diff --git a/zlib_container.c b/zlib_container.c deleted file mode 100644 index 5b7d0aa..0000000 --- a/zlib_container.c +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2013 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "zlib_container.h" -#include "util.h" - -#include - -#include "deflate.h" - - -/* Calculates the adler32 checksum of the data */ -static unsigned adler32(const unsigned char* data, size_t size) -{ - static const unsigned sums_overflow = 5550; - unsigned s1 = 1; - unsigned s2 = 1 >> 16; - - while (size > 0) { - size_t amount = size > sums_overflow ? sums_overflow : size; - size -= amount; - while (amount > 0) { - s1 += (*data++); - s2 += s1; - amount--; - } - s1 %= 65521; - s2 %= 65521; - } - - return (s2 << 16) | s1; -} - -void ZopfliZlibCompress(const ZopfliOptions* options, - const unsigned char* in, size_t insize, - unsigned char** out, size_t* outsize) { - unsigned char bitpointer = 0; - unsigned checksum = adler32(in, (unsigned)insize); - unsigned cmf = 120; /* CM 8, CINFO 7. See zlib spec.*/ - unsigned flevel = 0; - unsigned fdict = 0; - unsigned cmfflg = 256 * cmf + fdict * 32 + flevel * 64; - unsigned fcheck = 31 - cmfflg % 31; - cmfflg += fcheck; - - ZOPFLI_APPEND_DATA(cmfflg / 256, out, outsize); - ZOPFLI_APPEND_DATA(cmfflg % 256, out, outsize); - - ZopfliDeflate(options, 2 /* dynamic block */, 1 /* final */, - in, insize, &bitpointer, out, outsize); - - ZOPFLI_APPEND_DATA((checksum >> 24) % 256, out, outsize); - ZOPFLI_APPEND_DATA((checksum >> 16) % 256, out, outsize); - ZOPFLI_APPEND_DATA((checksum >> 8) % 256, out, outsize); - ZOPFLI_APPEND_DATA(checksum % 256, out, outsize); - - if (options->verbose) { - fprintf(stderr, - "Original Size: %d, Zlib: %d, Compression: %f%% Removed\n", - (int)insize, (int)*outsize, - 100.0 * (double)(insize - *outsize) / (double)insize); - } -} diff --git a/zlib_container.h b/zlib_container.h deleted file mode 100644 index 9ddfb9c..0000000 --- a/zlib_container.h +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2013 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#ifndef ZOPFLI_ZLIB_H_ -#define ZOPFLI_ZLIB_H_ - -/* -Functions to compress according to the Zlib specification. -*/ - -#include "zopfli.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* -Compresses according to the zlib specification and append the compressed -result to the output. - -options: global program options -out: pointer to the dynamic output array to which the result is appended. Must - be freed after use. -outsize: pointer to the dynamic output array size. -*/ -void ZopfliZlibCompress(const ZopfliOptions* options, - const unsigned char* in, size_t insize, - unsigned char** out, size_t* outsize); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* ZOPFLI_ZLIB_H_ */ diff --git a/zopfli.h b/zopfli.h deleted file mode 100644 index 56512a2..0000000 --- a/zopfli.h +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#ifndef ZOPFLI_ZOPFLI_H_ -#define ZOPFLI_ZOPFLI_H_ - -#include -#include /* for size_t */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* -Options used throughout the program. -*/ -typedef struct ZopfliOptions { - /* Whether to print output */ - int verbose; - - /* Whether to print more detailed output */ - int verbose_more; - - /* - Maximum amount of times to rerun forward and backward pass to optimize LZ77 - compression cost. Good values: 10, 15 for small files, 5 for files over - several MB in size or it will be too slow. - */ - int numiterations; - - /* - If true, splits the data in multiple deflate blocks with optimal choice - for the block boundaries. Block splitting gives better compression. Default: - true (1). - */ - int blocksplitting; - - /* - If true, chooses the optimal block split points only after doing the iterative - LZ77 compression. If false, chooses the block split points first, then does - iterative LZ77 on each individual block. Depending on the file, either first - or last gives the best compression. Default: false (0). - */ - int blocksplittinglast; - - /* - Maximum amount of blocks to split into (0 for unlimited, but this can give - extreme results that hurt compression on some files). Default value: 15. - */ - int blocksplittingmax; -} ZopfliOptions; - -/* Initializes options with default values. */ -void ZopfliInitOptions(ZopfliOptions* options); - -/* Output format */ -typedef enum { - ZOPFLI_FORMAT_GZIP, - ZOPFLI_FORMAT_ZLIB, - ZOPFLI_FORMAT_DEFLATE -} ZopfliFormat; - -/* -Compresses according to the given output format and appends the result to the -output. - -options: global program options -output_type: the output format to use -out: pointer to the dynamic output array to which the result is appended. Must - be freed after use -outsize: pointer to the dynamic output array size -*/ -void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type, - const unsigned char* in, size_t insize, - unsigned char** out, size_t* outsize); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* ZOPFLI_ZOPFLI_H_ */ diff --git a/zopfli/blocksplitter.c b/zopfli/blocksplitter.c new file mode 100644 index 0000000..68f5ff3 --- /dev/null +++ b/zopfli/blocksplitter.c @@ -0,0 +1,342 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "blocksplitter.h" + +#include +#include +#include + +#include "deflate.h" +#include "lz77.h" +#include "squeeze.h" +#include "tree.h" +#include "util.h" + +/* +The "f" for the FindMinimum function below. +i: the current parameter of f(i) +context: for your implementation +*/ +typedef double FindMinimumFun(size_t i, void* context); + +/* +Finds minimum of function f(i) where is is of type size_t, f(i) is of type +double, i is in range start-end (excluding end). +*/ +static size_t FindMinimum(FindMinimumFun f, void* context, + size_t start, size_t end) { + if (end - start < 1024) { + double best = ZOPFLI_LARGE_FLOAT; + size_t result = start; + size_t i; + for (i = start; i < end; i++) { + double v = f(i, context); + if (v < best) { + best = v; + result = i; + } + } + return result; + } else { + /* Try to find minimum faster by recursively checking multiple points. */ +#define NUM 9 /* Good value: 9. */ + size_t i; + size_t p[NUM]; + double vp[NUM]; + size_t besti; + double best; + double lastbest = ZOPFLI_LARGE_FLOAT; + size_t pos = start; + + for (;;) { + if (end - start <= NUM) break; + + for (i = 0; i < NUM; i++) { + p[i] = start + (i + 1) * ((end - start) / (NUM + 1)); + vp[i] = f(p[i], context); + } + besti = 0; + best = vp[0]; + for (i = 1; i < NUM; i++) { + if (vp[i] < best) { + best = vp[i]; + besti = i; + } + } + if (best > lastbest) break; + + start = besti == 0 ? start : p[besti - 1]; + end = besti == NUM - 1 ? end : p[besti + 1]; + + pos = p[besti]; + lastbest = best; + } + return pos; +#undef NUM + } +} + +/* +Returns estimated cost of a block in bits. It includes the size to encode the +tree and the size to encode all literal, length and distance symbols and their +extra bits. + +litlens: lz77 lit/lengths +dists: ll77 distances +lstart: start of block +lend: end of block (not inclusive) +*/ +static double EstimateCost(const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend) { + return ZopfliCalculateBlockSize(litlens, dists, lstart, lend, 2); +} + +typedef struct SplitCostContext { + const unsigned short* litlens; + const unsigned short* dists; + size_t llsize; + size_t start; + size_t end; +} SplitCostContext; + + +/* +Gets the cost which is the sum of the cost of the left and the right section +of the data. +type: FindMinimumFun +*/ +static double SplitCost(size_t i, void* context) { + SplitCostContext* c = (SplitCostContext*)context; + return EstimateCost(c->litlens, c->dists, c->start, i) + + EstimateCost(c->litlens, c->dists, i, c->end); +} + +static void AddSorted(size_t value, size_t** out, size_t* outsize) { + size_t i; + ZOPFLI_APPEND_DATA(value, out, outsize); + for (i = 0; i + 1 < *outsize; i++) { + if ((*out)[i] > value) { + size_t j; + for (j = *outsize - 1; j > i; j--) { + (*out)[j] = (*out)[j - 1]; + } + (*out)[i] = value; + break; + } + } +} + +/* +Prints the block split points as decimal and hex values in the terminal. +*/ +static void PrintBlockSplitPoints(const unsigned short* litlens, + const unsigned short* dists, + size_t llsize, const size_t* lz77splitpoints, + size_t nlz77points) { + size_t* splitpoints = 0; + size_t npoints = 0; + size_t i; + /* The input is given as lz77 indices, but we want to see the uncompressed + index values. */ + size_t pos = 0; + if (nlz77points > 0) { + for (i = 0; i < llsize; i++) { + size_t length = dists[i] == 0 ? 1 : litlens[i]; + if (lz77splitpoints[npoints] == i) { + ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints); + if (npoints == nlz77points) break; + } + pos += length; + } + } + assert(npoints == nlz77points); + + fprintf(stderr, "block split points: "); + for (i = 0; i < npoints; i++) { + fprintf(stderr, "%d ", (int)splitpoints[i]); + } + fprintf(stderr, "(hex:"); + for (i = 0; i < npoints; i++) { + fprintf(stderr, " %x", (int)splitpoints[i]); + } + fprintf(stderr, ")\n"); + + free(splitpoints); +} + +/* +Finds next block to try to split, the largest of the available ones. +The largest is chosen to make sure that if only a limited amount of blocks is +requested, their sizes are spread evenly. +llsize: the size of the LL77 data, which is the size of the done array here. +done: array indicating which blocks starting at that position are no longer + splittable (splitting them increases rather than decreases cost). +splitpoints: the splitpoints found so far. +npoints: the amount of splitpoints found so far. +lstart: output variable, giving start of block. +lend: output variable, giving end of block. +returns 1 if a block was found, 0 if no block found (all are done). +*/ +static int FindLargestSplittableBlock( + size_t llsize, const unsigned char* done, + const size_t* splitpoints, size_t npoints, + size_t* lstart, size_t* lend) { + size_t longest = 0; + int found = 0; + size_t i; + for (i = 0; i <= npoints; i++) { + size_t start = i == 0 ? 0 : splitpoints[i - 1]; + size_t end = i == npoints ? llsize - 1 : splitpoints[i]; + if (!done[start] && end - start > longest) { + *lstart = start; + *lend = end; + found = 1; + longest = end - start; + } + } + return found; +} + +void ZopfliBlockSplitLZ77(const ZopfliOptions* options, + const unsigned short* litlens, + const unsigned short* dists, + size_t llsize, size_t maxblocks, + size_t** splitpoints, size_t* npoints) { + size_t lstart, lend; + size_t i; + size_t llpos = 0; + size_t numblocks = 1; + unsigned char* done; + double splitcost, origcost; + + if (llsize < 10) return; /* This code fails on tiny files. */ + + done = (unsigned char*)malloc(llsize); + if (!done) exit(-1); /* Allocation failed. */ + for (i = 0; i < llsize; i++) done[i] = 0; + + lstart = 0; + lend = llsize; + for (;;) { + SplitCostContext c; + + if (maxblocks > 0 && numblocks >= maxblocks) { + break; + } + + c.litlens = litlens; + c.dists = dists; + c.llsize = llsize; + c.start = lstart; + c.end = lend; + assert(lstart < lend); + llpos = FindMinimum(SplitCost, &c, lstart + 1, lend); + + assert(llpos > lstart); + assert(llpos < lend); + + splitcost = EstimateCost(litlens, dists, lstart, llpos) + + EstimateCost(litlens, dists, llpos, lend); + origcost = EstimateCost(litlens, dists, lstart, lend); + + if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) { + done[lstart] = 1; + } else { + AddSorted(llpos, splitpoints, npoints); + numblocks++; + } + + if (!FindLargestSplittableBlock( + llsize, done, *splitpoints, *npoints, &lstart, &lend)) { + break; /* No further split will probably reduce compression. */ + } + + if (lend - lstart < 10) { + break; + } + } + + if (options->verbose) { + PrintBlockSplitPoints(litlens, dists, llsize, *splitpoints, *npoints); + } + + free(done); +} + +void ZopfliBlockSplit(const ZopfliOptions* options, + const unsigned char* in, size_t instart, size_t inend, + size_t maxblocks, size_t** splitpoints, size_t* npoints) { + size_t pos = 0; + size_t i; + ZopfliBlockState s; + size_t* lz77splitpoints = 0; + size_t nlz77points = 0; + ZopfliLZ77Store store; + + ZopfliInitLZ77Store(&store); + + s.options = options; + s.blockstart = instart; + s.blockend = inend; +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + s.lmc = 0; +#endif + + *npoints = 0; + *splitpoints = 0; + + /* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal + results in better blocks. */ + ZopfliLZ77Greedy(&s, in, instart, inend, &store); + + ZopfliBlockSplitLZ77(options, + store.litlens, store.dists, store.size, maxblocks, + &lz77splitpoints, &nlz77points); + + /* Convert LZ77 positions to positions in the uncompressed input. */ + pos = instart; + if (nlz77points > 0) { + for (i = 0; i < store.size; i++) { + size_t length = store.dists[i] == 0 ? 1 : store.litlens[i]; + if (lz77splitpoints[*npoints] == i) { + ZOPFLI_APPEND_DATA(pos, splitpoints, npoints); + if (*npoints == nlz77points) break; + } + pos += length; + } + } + assert(*npoints == nlz77points); + + free(lz77splitpoints); + ZopfliCleanLZ77Store(&store); +} + +void ZopfliBlockSplitSimple(const unsigned char* in, + size_t instart, size_t inend, + size_t blocksize, + size_t** splitpoints, size_t* npoints) { + size_t i = instart; + while (i < inend) { + ZOPFLI_APPEND_DATA(i, splitpoints, npoints); + i += blocksize; + } + (void)in; +} diff --git a/zopfli/blocksplitter.h b/zopfli/blocksplitter.h new file mode 100644 index 0000000..6791702 --- /dev/null +++ b/zopfli/blocksplitter.h @@ -0,0 +1,77 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +Functions to choose good boundaries for block splitting. Deflate allows encoding +the data in multiple blocks, with a separate Huffman tree for each block. The +Huffman tree itself requires some bytes to encode, so by choosing certain +blocks, you can either hurt, or enhance compression. These functions choose good +ones that enhance it. +*/ + +#ifndef ZOPFLI_BLOCKSPLITTER_H_ +#define ZOPFLI_BLOCKSPLITTER_H_ + +#include + +#include "zopfli.h" + + +/* +Does blocksplitting on LZ77 data. +The output splitpoints are indices in the LZ77 data. +litlens: lz77 lit/lengths +dists: lz77 distances +llsize: size of litlens and dists +maxblocks: set a limit to the amount of blocks. Set to 0 to mean no limit. +*/ +void ZopfliBlockSplitLZ77(const ZopfliOptions* options, + const unsigned short* litlens, + const unsigned short* dists, + size_t llsize, size_t maxblocks, + size_t** splitpoints, size_t* npoints); + +/* +Does blocksplitting on uncompressed data. +The output splitpoints are indices in the uncompressed bytes. + +options: general program options. +in: uncompressed input data +instart: where to start splitting +inend: where to end splitting (not inclusive) +maxblocks: maximum amount of blocks to split into, or 0 for no limit +splitpoints: dynamic array to put the resulting split point coordinates into. + The coordinates are indices in the input array. +npoints: pointer to amount of splitpoints, for the dynamic array. The amount of + blocks is the amount of splitpoitns + 1. +*/ +void ZopfliBlockSplit(const ZopfliOptions* options, + const unsigned char* in, size_t instart, size_t inend, + size_t maxblocks, size_t** splitpoints, size_t* npoints); + +/* +Divides the input into equal blocks, does not even take LZ77 lengths into +account. +*/ +void ZopfliBlockSplitSimple(const unsigned char* in, + size_t instart, size_t inend, + size_t blocksize, + size_t** splitpoints, size_t* npoints); + +#endif /* ZOPFLI_BLOCKSPLITTER_H_ */ diff --git a/zopfli/cache.c b/zopfli/cache.c new file mode 100644 index 0000000..88a49ac --- /dev/null +++ b/zopfli/cache.c @@ -0,0 +1,119 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "cache.h" + +#include +#include +#include + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + +void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc) { + size_t i; + lmc->length = (unsigned short*)malloc(sizeof(unsigned short) * blocksize); + lmc->dist = (unsigned short*)malloc(sizeof(unsigned short) * blocksize); + /* Rather large amount of memory. */ + lmc->sublen = (unsigned char*)malloc(ZOPFLI_CACHE_LENGTH * 3 * blocksize); + + /* length > 0 and dist 0 is invalid combination, which indicates on purpose + that this cache value is not filled in yet. */ + for (i = 0; i < blocksize; i++) lmc->length[i] = 1; + for (i = 0; i < blocksize; i++) lmc->dist[i] = 0; + for (i = 0; i < ZOPFLI_CACHE_LENGTH * blocksize * 3; i++) lmc->sublen[i] = 0; +} + +void ZopfliCleanCache(ZopfliLongestMatchCache* lmc) { + free(lmc->length); + free(lmc->dist); + free(lmc->sublen); +} + +void ZopfliSublenToCache(const unsigned short* sublen, + size_t pos, size_t length, + ZopfliLongestMatchCache* lmc) { + size_t i; + size_t j = 0; + unsigned bestlength = 0; + unsigned char* cache; + +#if ZOPFLI_CACHE_LENGTH == 0 + return; +#endif + + cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3]; + if (length < 3) return; + for (i = 3; i <= length; i++) { + if (i == length || sublen[i] != sublen[i + 1]) { + cache[j * 3] = i - 3; + cache[j * 3 + 1] = sublen[i] % 256; + cache[j * 3 + 2] = (sublen[i] >> 8) % 256; + bestlength = i; + j++; + if (j >= ZOPFLI_CACHE_LENGTH) break; + } + } + if (j < ZOPFLI_CACHE_LENGTH) { + assert(bestlength == length); + cache[(ZOPFLI_CACHE_LENGTH - 1) * 3] = bestlength - 3; + } else { + assert(bestlength <= length); + } + assert(bestlength == ZopfliMaxCachedSublen(lmc, pos, length)); +} + +void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc, + size_t pos, size_t length, + unsigned short* sublen) { + size_t i, j; + unsigned maxlength = ZopfliMaxCachedSublen(lmc, pos, length); + unsigned prevlength = 0; + unsigned char* cache; +#if ZOPFLI_CACHE_LENGTH == 0 + return; +#endif + if (length < 3) return; + cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3]; + for (j = 0; j < ZOPFLI_CACHE_LENGTH; j++) { + unsigned length = cache[j * 3] + 3; + unsigned dist = cache[j * 3 + 1] + 256 * cache[j * 3 + 2]; + for (i = prevlength; i <= length; i++) { + sublen[i] = dist; + } + if (length == maxlength) break; + prevlength = length + 1; + } +} + +/* +Returns the length up to which could be stored in the cache. +*/ +unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc, + size_t pos, size_t length) { + unsigned char* cache; +#if ZOPFLI_CACHE_LENGTH == 0 + return 0; +#endif + cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3]; + (void)length; + if (cache[1] == 0 && cache[2] == 0) return 0; /* No sublen cached. */ + return cache[(ZOPFLI_CACHE_LENGTH - 1) * 3] + 3; +} + +#endif /* ZOPFLI_LONGEST_MATCH_CACHE */ diff --git a/zopfli/cache.h b/zopfli/cache.h new file mode 100644 index 0000000..5ca0c50 --- /dev/null +++ b/zopfli/cache.h @@ -0,0 +1,66 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +The cache that speeds up ZopfliFindLongestMatch of lz77.c. +*/ + +#ifndef ZOPFLI_CACHE_H_ +#define ZOPFLI_CACHE_H_ + +#include "util.h" + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + +/* +Cache used by ZopfliFindLongestMatch to remember previously found length/dist +values. +This is needed because the squeeze runs will ask these values multiple times for +the same position. +Uses large amounts of memory, since it has to remember the distance belonging +to every possible shorter-than-the-best length (the so called "sublen" array). +*/ +typedef struct ZopfliLongestMatchCache { + unsigned short* length; + unsigned short* dist; + unsigned char* sublen; +} ZopfliLongestMatchCache; + +/* Initializes the ZopfliLongestMatchCache. */ +void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc); + +/* Frees up the memory of the ZopfliLongestMatchCache. */ +void ZopfliCleanCache(ZopfliLongestMatchCache* lmc); + +/* Stores sublen array in the cache. */ +void ZopfliSublenToCache(const unsigned short* sublen, + size_t pos, size_t length, + ZopfliLongestMatchCache* lmc); + +/* Extracts sublen array from the cache. */ +void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc, + size_t pos, size_t length, + unsigned short* sublen); +/* Returns the length up to which could be stored in the cache. */ +unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc, + size_t pos, size_t length); + +#endif /* ZOPFLI_LONGEST_MATCH_CACHE */ + +#endif /* ZOPFLI_CACHE_H_ */ diff --git a/zopfli/deflate.c b/zopfli/deflate.c new file mode 100644 index 0000000..4b0724b --- /dev/null +++ b/zopfli/deflate.c @@ -0,0 +1,866 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "deflate.h" + +#include +#include +#include + +#include "blocksplitter.h" +#include "lz77.h" +#include "squeeze.h" +#include "tree.h" + +/* +bp = bitpointer, always in range [0, 7]. +The outsize is number of necessary bytes to encode the bits. +Given the value of bp and the amount of bytes, the amount of bits represented +is not simply bytesize * 8 + bp because even representing one bit requires a +whole byte. It is: (bp == 0) ? (bytesize * 8) : ((bytesize - 1) * 8 + bp) +*/ +static void AddBit(int bit, + unsigned char* bp, unsigned char** out, size_t* outsize) { + if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); + (*out)[*outsize - 1] |= bit << *bp; + *bp = (*bp + 1) & 7; +} + +static void AddBits(unsigned symbol, unsigned length, + unsigned char* bp, unsigned char** out, size_t* outsize) { + /* TODO(lode): make more efficient (add more bits at once). */ + unsigned i; + for (i = 0; i < length; i++) { + unsigned bit = (symbol >> i) & 1; + if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); + (*out)[*outsize - 1] |= bit << *bp; + *bp = (*bp + 1) & 7; + } +} + +/* +Adds bits, like AddBits, but the order is inverted. The deflate specification +uses both orders in one standard. +*/ +static void AddHuffmanBits(unsigned symbol, unsigned length, + unsigned char* bp, unsigned char** out, + size_t* outsize) { + /* TODO(lode): make more efficient (add more bits at once). */ + unsigned i; + for (i = 0; i < length; i++) { + unsigned bit = (symbol >> (length - i - 1)) & 1; + if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); + (*out)[*outsize - 1] |= bit << *bp; + *bp = (*bp + 1) & 7; + } +} + +/* +Ensures there are at least 2 distance codes to support buggy decoders. +Zlib 1.2.1 and below have a bug where it fails if there isn't at least 1 +distance code (with length > 0), even though it's valid according to the +deflate spec to have 0 distance codes. On top of that, some mobile phones +require at least two distance codes. To support these decoders too (but +potentially at the cost of a few bytes), add dummy code lengths of 1. +References to this bug can be found in the changelog of +Zlib 1.2.2 and here: http://www.jonof.id.au/forum/index.php?topic=515.0. + +d_lengths: the 32 lengths of the distance codes. +*/ +static void PatchDistanceCodesForBuggyDecoders(unsigned* d_lengths) { + int num_dist_codes = 0; /* Amount of non-zero distance codes */ + int i; + for (i = 0; i < 30 /* Ignore the two unused codes from the spec */; i++) { + if (d_lengths[i]) num_dist_codes++; + if (num_dist_codes >= 2) return; /* Two or more codes is fine. */ + } + + if (num_dist_codes == 0) { + d_lengths[0] = d_lengths[1] = 1; + } else if (num_dist_codes == 1) { + d_lengths[d_lengths[0] ? 1 : 0] = 1; + } +} + +/* +Encodes the Huffman tree and returns how many bits its encoding takes. If out +is a null pointer, only returns the size and runs faster. +*/ +static size_t EncodeTree(const unsigned* ll_lengths, + const unsigned* d_lengths, + int use_16, int use_17, int use_18, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + unsigned lld_total; /* Total amount of literal, length, distance codes. */ + /* Runlength encoded version of lengths of litlen and dist trees. */ + unsigned* rle = 0; + unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */ + size_t rle_size = 0; /* Size of rle array. */ + size_t rle_bits_size = 0; /* Should have same value as rle_size. */ + unsigned hlit = 29; /* 286 - 257 */ + unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/ + unsigned hclen; + unsigned hlit2; + size_t i, j; + size_t clcounts[19]; + unsigned clcl[19]; /* Code length code lengths. */ + unsigned clsymbols[19]; + /* The order in which code length code lengths are encoded as per deflate. */ + static const unsigned order[19] = { + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 + }; + int size_only = !out; + size_t result_size = 0; + + for(i = 0; i < 19; i++) clcounts[i] = 0; + + /* Trim zeros. */ + while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--; + while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--; + hlit2 = hlit + 257; + + lld_total = hlit2 + hdist + 1; + + for (i = 0; i < lld_total; i++) { + /* This is an encoding of a huffman tree, so now the length is a symbol */ + unsigned char symbol = i < hlit2 ? ll_lengths[i] : d_lengths[i - hlit2]; + unsigned count = 1; + if(use_16 || (symbol == 0 && (use_17 || use_18))) { + for (j = i + 1; j < lld_total && symbol == + (j < hlit2 ? ll_lengths[j] : d_lengths[j - hlit2]); j++) { + count++; + } + } + i += count - 1; + + /* Repetitions of zeroes */ + if (symbol == 0 && count >= 3) { + if (use_18) { + while (count >= 11) { + unsigned count2 = count > 138 ? 138 : count; + if (!size_only) { + ZOPFLI_APPEND_DATA(18, &rle, &rle_size); + ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size); + } + clcounts[18]++; + count -= count2; + } + } + if (use_17) { + while (count >= 3) { + unsigned count2 = count > 10 ? 10 : count; + if (!size_only) { + ZOPFLI_APPEND_DATA(17, &rle, &rle_size); + ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); + } + clcounts[17]++; + count -= count2; + } + } + } + + /* Repetitions of any symbol */ + if (use_16 && count >= 4) { + count--; /* Since the first one is hardcoded. */ + clcounts[symbol]++; + if (!size_only) { + ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); + ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); + } + while (count >= 3) { + unsigned count2 = count > 6 ? 6 : count; + if (!size_only) { + ZOPFLI_APPEND_DATA(16, &rle, &rle_size); + ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); + } + clcounts[16]++; + count -= count2; + } + } + + /* No or insufficient repetition */ + clcounts[symbol] += count; + while (count > 0) { + if (!size_only) { + ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); + ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); + } + count--; + } + } + + ZopfliCalculateBitLengths(clcounts, 19, 7, clcl); + if (!size_only) ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols); + + hclen = 15; + /* Trim zeros. */ + while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--; + + if (!size_only) { + AddBits(hlit, 5, bp, out, outsize); + AddBits(hdist, 5, bp, out, outsize); + AddBits(hclen, 4, bp, out, outsize); + + for (i = 0; i < hclen + 4; i++) { + AddBits(clcl[order[i]], 3, bp, out, outsize); + } + + for (i = 0; i < rle_size; i++) { + unsigned symbol = clsymbols[rle[i]]; + AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize); + /* Extra bits. */ + if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize); + else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize); + else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize); + } + } + + result_size += 14; /* hlit, hdist, hclen bits */ + result_size += (hclen + 4) * 3; /* clcl bits */ + for(i = 0; i < 19; i++) { + result_size += clcl[i] * clcounts[i]; + } + /* Extra bits. */ + result_size += clcounts[16] * 2; + result_size += clcounts[17] * 3; + result_size += clcounts[18] * 7; + + /* Note: in case of "size_only" these are null pointers so no effect. */ + free(rle); + free(rle_bits); + + return result_size; +} + +static void AddDynamicTree(const unsigned* ll_lengths, + const unsigned* d_lengths, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + int i; + int best = 0; + size_t bestsize = 0; + + for(i = 0; i < 8; i++) { + size_t size = EncodeTree(ll_lengths, d_lengths, + i & 1, i & 2, i & 4, + 0, 0, 0); + if (bestsize == 0 || size < bestsize) { + bestsize = size; + best = i; + } + } + + EncodeTree(ll_lengths, d_lengths, + best & 1, best & 2, best & 4, + bp, out, outsize); +} + +/* +Gives the exact size of the tree, in bits, as it will be encoded in DEFLATE. +*/ +static size_t CalculateTreeSize(const unsigned* ll_lengths, + const unsigned* d_lengths) { + size_t result = 0; + int i; + + for(i = 0; i < 8; i++) { + size_t size = EncodeTree(ll_lengths, d_lengths, + i & 1, i & 2, i & 4, + 0, 0, 0); + if (result == 0 || size < result) result = size; + } + + return result; +} + +/* +Adds all lit/len and dist codes from the lists as huffman symbols. Does not add +end code 256. expected_data_size is the uncompressed block size, used for +assert, but you can set it to 0 to not do the assertion. +*/ +static void AddLZ77Data(const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend, + size_t expected_data_size, + const unsigned* ll_symbols, const unsigned* ll_lengths, + const unsigned* d_symbols, const unsigned* d_lengths, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + size_t testlength = 0; + size_t i; + + for (i = lstart; i < lend; i++) { + unsigned dist = dists[i]; + unsigned litlen = litlens[i]; + if (dist == 0) { + assert(litlen < 256); + assert(ll_lengths[litlen] > 0); + AddHuffmanBits(ll_symbols[litlen], ll_lengths[litlen], bp, out, outsize); + testlength++; + } else { + unsigned lls = ZopfliGetLengthSymbol(litlen); + unsigned ds = ZopfliGetDistSymbol(dist); + assert(litlen >= 3 && litlen <= 288); + assert(ll_lengths[lls] > 0); + assert(d_lengths[ds] > 0); + AddHuffmanBits(ll_symbols[lls], ll_lengths[lls], bp, out, outsize); + AddBits(ZopfliGetLengthExtraBitsValue(litlen), + ZopfliGetLengthExtraBits(litlen), + bp, out, outsize); + AddHuffmanBits(d_symbols[ds], d_lengths[ds], bp, out, outsize); + AddBits(ZopfliGetDistExtraBitsValue(dist), + ZopfliGetDistExtraBits(dist), + bp, out, outsize); + testlength += litlen; + } + } + assert(expected_data_size == 0 || testlength == expected_data_size); +} + +static void GetFixedTree(unsigned* ll_lengths, unsigned* d_lengths) { + size_t i; + for (i = 0; i < 144; i++) ll_lengths[i] = 8; + for (i = 144; i < 256; i++) ll_lengths[i] = 9; + for (i = 256; i < 280; i++) ll_lengths[i] = 7; + for (i = 280; i < 288; i++) ll_lengths[i] = 8; + for (i = 0; i < 32; i++) d_lengths[i] = 5; +} + +/* +Calculates size of the part after the header and tree of an LZ77 block, in bits. +*/ +static size_t CalculateBlockSymbolSize(const unsigned* ll_lengths, + const unsigned* d_lengths, + const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend) { + size_t result = 0; + size_t i; + for (i = lstart; i < lend; i++) { + if (dists[i] == 0) { + result += ll_lengths[litlens[i]]; + } else { + result += ll_lengths[ZopfliGetLengthSymbol(litlens[i])]; + result += d_lengths[ZopfliGetDistSymbol(dists[i])]; + result += ZopfliGetLengthExtraBits(litlens[i]); + result += ZopfliGetDistExtraBits(dists[i]); + } + } + result += ll_lengths[256]; /*end symbol*/ + return result; +} + +static size_t AbsDiff(size_t x, size_t y) { + if (x > y) + return x - y; + else + return y - x; +} + +/* +Change the population counts in a way that the consequent Hufmann tree +compression, especially its rle-part will be more likely to compress this data +more efficiently. length containts the size of the histogram. +*/ +void OptimizeHuffmanForRle(int length, size_t* counts) { + int i, k, stride; + size_t symbol, sum, limit; + int* good_for_rle; + + /* 1) We don't want to touch the trailing zeros. We may break the + rules of the format by adding more data in the distance codes. */ + for (; length >= 0; --length) { + if (length == 0) { + return; + } + if (counts[length - 1] != 0) { + /* Now counts[0..length - 1] does not have trailing zeros. */ + break; + } + } + /* 2) Let's mark all population counts that already can be encoded + with an rle code.*/ + good_for_rle = (int*)malloc(length * sizeof(int)); + for (i = 0; i < length; ++i) good_for_rle[i] = 0; + + /* Let's not spoil any of the existing good rle codes. + Mark any seq of 0's that is longer than 5 as a good_for_rle. + Mark any seq of non-0's that is longer than 7 as a good_for_rle.*/ + symbol = counts[0]; + stride = 0; + for (i = 0; i < length + 1; ++i) { + if (i == length || counts[i] != symbol) { + if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) { + for (k = 0; k < stride; ++k) { + good_for_rle[i - k - 1] = 1; + } + } + stride = 1; + if (i != length) { + symbol = counts[i]; + } + } else { + ++stride; + } + } + + /* 3) Let's replace those population counts that lead to more rle codes. */ + stride = 0; + limit = counts[0]; + sum = 0; + for (i = 0; i < length + 1; ++i) { + if (i == length || good_for_rle[i] + /* Heuristic for selecting the stride ranges to collapse. */ + || AbsDiff(counts[i], limit) >= 4) { + if (stride >= 4 || (stride >= 3 && sum == 0)) { + /* The stride must end, collapse what we have, if we have enough (4). */ + int count = (sum + stride / 2) / stride; + if (count < 1) count = 1; + if (sum == 0) { + /* Don't make an all zeros stride to be upgraded to ones. */ + count = 0; + } + for (k = 0; k < stride; ++k) { + /* We don't want to change value at counts[i], + that is already belonging to the next stride. Thus - 1. */ + counts[i - k - 1] = count; + } + } + stride = 0; + sum = 0; + if (i < length - 3) { + /* All interesting strides have a count of at least 4, + at least when non-zeros. */ + limit = (counts[i] + counts[i + 1] + + counts[i + 2] + counts[i + 3] + 2) / 4; + } else if (i < length) { + limit = counts[i]; + } else { + limit = 0; + } + } + ++stride; + if (i != length) { + sum += counts[i]; + } + } + + free(good_for_rle); +} + +/* +Calculates the bit lengths for the symbols for dynamic blocks. Chooses bit +lengths that give the smallest size of tree encoding + encoding of all the +symbols to have smallest output size. This are not necessarily the ideal Huffman +bit lengths. +*/ +static void GetDynamicLengths(const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend, + unsigned* ll_lengths, unsigned* d_lengths) { + size_t ll_counts[288]; + size_t d_counts[32]; + + ZopfliLZ77Counts(litlens, dists, lstart, lend, ll_counts, d_counts); + OptimizeHuffmanForRle(288, ll_counts); + OptimizeHuffmanForRle(32, d_counts); + ZopfliCalculateBitLengths(ll_counts, 288, 15, ll_lengths); + ZopfliCalculateBitLengths(d_counts, 32, 15, d_lengths); + PatchDistanceCodesForBuggyDecoders(d_lengths); +} + +double ZopfliCalculateBlockSize(const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend, int btype) { + unsigned ll_lengths[288]; + unsigned d_lengths[32]; + + double result = 3; /* bfinal and btype bits */ + + assert(btype == 1 || btype == 2); /* This is not for uncompressed blocks. */ + + if(btype == 1) { + GetFixedTree(ll_lengths, d_lengths); + } else { + GetDynamicLengths(litlens, dists, lstart, lend, ll_lengths, d_lengths); + result += CalculateTreeSize(ll_lengths, d_lengths); + } + + result += CalculateBlockSymbolSize( + ll_lengths, d_lengths, litlens, dists, lstart, lend); + + return result; +} + +/* +Adds a deflate block with the given LZ77 data to the output. +options: global program options +btype: the block type, must be 1 or 2 +final: whether to set the "final" bit on this block, must be the last block +litlens: literal/length array of the LZ77 data, in the same format as in + ZopfliLZ77Store. +dists: distance array of the LZ77 data, in the same format as in + ZopfliLZ77Store. +lstart: where to start in the LZ77 data +lend: where to end in the LZ77 data (not inclusive) +expected_data_size: the uncompressed block size, used for assert, but you can + set it to 0 to not do the assertion. +bp: output bit pointer +out: dynamic output array to append to +outsize: dynamic output array size +*/ +static void AddLZ77Block(const ZopfliOptions* options, int btype, int final, + const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend, + size_t expected_data_size, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + unsigned ll_lengths[288]; + unsigned d_lengths[32]; + unsigned ll_symbols[288]; + unsigned d_symbols[32]; + size_t detect_block_size = *outsize; + size_t compressed_size; + size_t uncompressed_size = 0; + size_t i; + + AddBit(final, bp, out, outsize); + AddBit(btype & 1, bp, out, outsize); + AddBit((btype & 2) >> 1, bp, out, outsize); + + if (btype == 1) { + /* Fixed block. */ + GetFixedTree(ll_lengths, d_lengths); + } else { + /* Dynamic block. */ + unsigned detect_tree_size; + assert(btype == 2); + + GetDynamicLengths(litlens, dists, lstart, lend, ll_lengths, d_lengths); + + detect_tree_size = *outsize; + AddDynamicTree(ll_lengths, d_lengths, bp, out, outsize); + if (options->verbose) { + fprintf(stderr, "treesize: %d\n", (int)(*outsize - detect_tree_size)); + } + } + + ZopfliLengthsToSymbols(ll_lengths, 288, 15, ll_symbols); + ZopfliLengthsToSymbols(d_lengths, 32, 15, d_symbols); + + detect_block_size = *outsize; + AddLZ77Data(litlens, dists, lstart, lend, expected_data_size, + ll_symbols, ll_lengths, d_symbols, d_lengths, + bp, out, outsize); + /* End symbol. */ + AddHuffmanBits(ll_symbols[256], ll_lengths[256], bp, out, outsize); + + for (i = lstart; i < lend; i++) { + uncompressed_size += dists[i] == 0 ? 1 : litlens[i]; + } + compressed_size = *outsize - detect_block_size; + if (options->verbose) { + fprintf(stderr, "compressed block size: %d (%dk) (unc: %d)\n", + (int)compressed_size, (int)(compressed_size / 1024), + (int)(uncompressed_size)); + } +} + +static void DeflateDynamicBlock(const ZopfliOptions* options, int final, + const unsigned char* in, + size_t instart, size_t inend, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + ZopfliBlockState s; + size_t blocksize = inend - instart; + ZopfliLZ77Store store; + int btype = 2; + + ZopfliInitLZ77Store(&store); + + s.options = options; + s.blockstart = instart; + s.blockend = inend; +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache)); + ZopfliInitCache(blocksize, s.lmc); +#endif + + ZopfliLZ77Optimal(&s, in, instart, inend, &store); + + /* For small block, encoding with fixed tree can be smaller. For large block, + don't bother doing this expensive test, dynamic tree will be better.*/ + if (store.size < 1000) { + double dyncost, fixedcost; + ZopfliLZ77Store fixedstore; + ZopfliInitLZ77Store(&fixedstore); + ZopfliLZ77OptimalFixed(&s, in, instart, inend, &fixedstore); + dyncost = ZopfliCalculateBlockSize(store.litlens, store.dists, + 0, store.size, 2); + fixedcost = ZopfliCalculateBlockSize(fixedstore.litlens, fixedstore.dists, + 0, fixedstore.size, 1); + if (fixedcost < dyncost) { + btype = 1; + ZopfliCleanLZ77Store(&store); + store = fixedstore; + } else { + ZopfliCleanLZ77Store(&fixedstore); + } + } + + AddLZ77Block(s.options, btype, final, + store.litlens, store.dists, 0, store.size, + blocksize, bp, out, outsize); + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + ZopfliCleanCache(s.lmc); + free(s.lmc); +#endif + ZopfliCleanLZ77Store(&store); +} + +static void DeflateFixedBlock(const ZopfliOptions* options, int final, + const unsigned char* in, + size_t instart, size_t inend, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + ZopfliBlockState s; + size_t blocksize = inend - instart; + ZopfliLZ77Store store; + + ZopfliInitLZ77Store(&store); + + s.options = options; + s.blockstart = instart; + s.blockend = inend; +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache)); + ZopfliInitCache(blocksize, s.lmc); +#endif + + ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store); + + AddLZ77Block(s.options, 1, final, store.litlens, store.dists, 0, store.size, + blocksize, bp, out, outsize); + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + ZopfliCleanCache(s.lmc); + free(s.lmc); +#endif + ZopfliCleanLZ77Store(&store); +} + +static void DeflateNonCompressedBlock(const ZopfliOptions* options, int final, + const unsigned char* in, size_t instart, + size_t inend, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + size_t i; + size_t blocksize = inend - instart; + unsigned short nlen = ~blocksize; + + (void)options; + assert(blocksize < 65536); /* Non compressed blocks are max this size. */ + + AddBit(final, bp, out, outsize); + /* BTYPE 00 */ + AddBit(0, bp, out, outsize); + AddBit(0, bp, out, outsize); + + /* Any bits of input up to the next byte boundary are ignored. */ + *bp = 0; + + ZOPFLI_APPEND_DATA(blocksize % 256, out, outsize); + ZOPFLI_APPEND_DATA((blocksize / 256) % 256, out, outsize); + ZOPFLI_APPEND_DATA(nlen % 256, out, outsize); + ZOPFLI_APPEND_DATA((nlen / 256) % 256, out, outsize); + + for (i = instart; i < inend; i++) { + ZOPFLI_APPEND_DATA(in[i], out, outsize); + } +} + +static void DeflateBlock(const ZopfliOptions* options, + int btype, int final, + const unsigned char* in, size_t instart, size_t inend, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + if (btype == 0) { + DeflateNonCompressedBlock( + options, final, in, instart, inend, bp, out, outsize); + } else if (btype == 1) { + DeflateFixedBlock(options, final, in, instart, inend, bp, out, outsize); + } else { + assert (btype == 2); + DeflateDynamicBlock(options, final, in, instart, inend, bp, out, outsize); + } +} + +/* +Does squeeze strategy where first block splitting is done, then each block is +squeezed. +Parameters: see description of the ZopfliDeflate function. +*/ +static void DeflateSplittingFirst(const ZopfliOptions* options, + int btype, int final, + const unsigned char* in, + size_t instart, size_t inend, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + size_t i; + size_t* splitpoints = 0; + size_t npoints = 0; + if (btype == 0) { + ZopfliBlockSplitSimple(in, instart, inend, 65535, &splitpoints, &npoints); + } else if (btype == 1) { + /* If all blocks are fixed tree, splitting into separate blocks only + increases the total size. Leave npoints at 0, this represents 1 block. */ + } else { + ZopfliBlockSplit(options, in, instart, inend, + options->blocksplittingmax, &splitpoints, &npoints); + } + + for (i = 0; i <= npoints; i++) { + size_t start = i == 0 ? instart : splitpoints[i - 1]; + size_t end = i == npoints ? inend : splitpoints[i]; + DeflateBlock(options, btype, i == npoints && final, in, start, end, + bp, out, outsize); + } + + free(splitpoints); +} + +/* +Does squeeze strategy where first the best possible lz77 is done, and then based +on that data, block splitting is done. +Parameters: see description of the ZopfliDeflate function. +*/ +static void DeflateSplittingLast(const ZopfliOptions* options, + int btype, int final, + const unsigned char* in, + size_t instart, size_t inend, + unsigned char* bp, + unsigned char** out, size_t* outsize) { + size_t i; + ZopfliBlockState s; + ZopfliLZ77Store store; + size_t* splitpoints = 0; + size_t npoints = 0; + + if (btype == 0) { + /* This function only supports LZ77 compression. DeflateSplittingFirst + supports the special case of noncompressed data. Punt it to that one. */ + DeflateSplittingFirst(options, btype, final, + in, instart, inend, + bp, out, outsize); + } + assert(btype == 1 || btype == 2); + + ZopfliInitLZ77Store(&store); + + s.options = options; + s.blockstart = instart; + s.blockend = inend; +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache)); + ZopfliInitCache(inend - instart, s.lmc); +#endif + + if (btype == 2) { + ZopfliLZ77Optimal(&s, in, instart, inend, &store); + } else { + assert (btype == 1); + ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store); + } + + if (btype == 1) { + /* If all blocks are fixed tree, splitting into separate blocks only + increases the total size. Leave npoints at 0, this represents 1 block. */ + } else { + ZopfliBlockSplitLZ77(options, store.litlens, store.dists, store.size, + options->blocksplittingmax, &splitpoints, &npoints); + } + + for (i = 0; i <= npoints; i++) { + size_t start = i == 0 ? 0 : splitpoints[i - 1]; + size_t end = i == npoints ? store.size : splitpoints[i]; + AddLZ77Block(options, btype, i == npoints && final, + store.litlens, store.dists, start, end, 0, + bp, out, outsize); + } + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + ZopfliCleanCache(s.lmc); + free(s.lmc); +#endif + + ZopfliCleanLZ77Store(&store); + free(splitpoints); +} + +/* +Deflate a part, to allow ZopfliDeflate() to use multiple master blocks if +needed. +It is possible to call this function multiple times in a row, shifting +instart and inend to next bytes of the data. If instart is larger than 0, then +previous bytes are used as the initial dictionary for LZ77. +This function will usually output multiple deflate blocks. If final is 1, then +the final bit will be set on the last block. +*/ +void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, + const unsigned char* in, size_t instart, size_t inend, + unsigned char* bp, unsigned char** out, + size_t* outsize) { + if (options->blocksplitting) { + if (options->blocksplittinglast) { + DeflateSplittingLast(options, btype, final, in, instart, inend, + bp, out, outsize); + } else { + DeflateSplittingFirst(options, btype, final, in, instart, inend, + bp, out, outsize); + } + } else { + DeflateBlock(options, btype, final, in, instart, inend, bp, out, outsize); + } +} + +void ZopfliDeflate(const ZopfliOptions* options, int btype, int final, + const unsigned char* in, size_t insize, + unsigned char* bp, unsigned char** out, size_t* outsize) { +#if ZOPFLI_MASTER_BLOCK_SIZE == 0 + ZopfliDeflatePart(options, btype, final, in, 0, insize, bp, out, outsize); +#else + size_t i = 0; + while (i < insize) { + int masterfinal = (i + ZOPFLI_MASTER_BLOCK_SIZE >= insize); + int final2 = final && masterfinal; + size_t size = masterfinal ? insize - i : ZOPFLI_MASTER_BLOCK_SIZE; + ZopfliDeflatePart(options, btype, final2, + in, i, i + size, bp, out, outsize); + i += size; + } +#endif + if (options->verbose) { + fprintf(stderr, + "Original Size: %d, Deflate: %d, Compression: %f%% Removed\n", + (int)insize, (int)*outsize, + 100.0 * (double)(insize - *outsize) / (double)insize); + } +} diff --git a/zopfli/deflate.h b/zopfli/deflate.h new file mode 100644 index 0000000..189c77a --- /dev/null +++ b/zopfli/deflate.h @@ -0,0 +1,86 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#ifndef ZOPFLI_DEFLATE_H_ +#define ZOPFLI_DEFLATE_H_ + +/* +Functions to compress according to the DEFLATE specification, using the +"squeeze" LZ77 compression backend. +*/ + +#include "zopfli.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +Compresses according to the deflate specification and append the compressed +result to the output. +This function will usually output multiple deflate blocks. If final is 1, then +the final bit will be set on the last block. + +options: global program options +btype: the deflate block type. Use 2 for best compression. + -0: non compressed blocks (00) + -1: blocks with fixed tree (01) + -2: blocks with dynamic tree (10) +final: whether this is the last section of the input, sets the final bit to the + last deflate block. +in: the input bytes +insize: number of input bytes +bp: bit pointer for the output array. This must initially be 0, and for + consecutive calls must be reused (it can have values from 0-7). This is + because deflate appends blocks as bit-based data, rather than on byte + boundaries. +out: pointer to the dynamic output array to which the result is appended. Must + be freed after use. +outsize: pointer to the dynamic output array size. +*/ +void ZopfliDeflate(const ZopfliOptions* options, int btype, int final, + const unsigned char* in, size_t insize, + unsigned char* bp, unsigned char** out, size_t* outsize); + +/* +Like ZopfliDeflate, but allows to specify start and end byte with instart and +inend. Only that part is compressed, but earlier bytes are still used for the +back window. +*/ +void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, + const unsigned char* in, size_t instart, size_t inend, + unsigned char* bp, unsigned char** out, + size_t* outsize); + +/* +Calculates block size in bits. +litlens: lz77 lit/lengths +dists: ll77 distances +lstart: start of block +lend: end of block (not inclusive) +*/ +double ZopfliCalculateBlockSize(const unsigned short* litlens, + const unsigned short* dists, + size_t lstart, size_t lend, int btype); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* ZOPFLI_DEFLATE_H_ */ diff --git a/zopfli/gzip_container.c b/zopfli/gzip_container.c new file mode 100644 index 0000000..8a062f2 --- /dev/null +++ b/zopfli/gzip_container.c @@ -0,0 +1,117 @@ +/* +Copyright 2013 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "gzip_container.h" +#include "util.h" + +#include + +#include "deflate.h" + +/* Table of CRCs of all 8-bit messages. */ +static unsigned long crc_table[256]; + +/* Flag: has the table been computed? Initially false. */ +static int crc_table_computed = 0; + +/* Makes the table for a fast CRC. */ +static void MakeCRCTable() { + unsigned long c; + int n, k; + for (n = 0; n < 256; n++) { + c = (unsigned long) n; + for (k = 0; k < 8; k++) { + if (c & 1) { + c = 0xedb88320L ^ (c >> 1); + } else { + c = c >> 1; + } + } + crc_table[n] = c; + } + crc_table_computed = 1; +} + + +/* +Updates a running crc with the bytes buf[0..len-1] and returns +the updated crc. The crc should be initialized to zero. +*/ +static unsigned long UpdateCRC(unsigned long crc, + const unsigned char *buf, size_t len) { + unsigned long c = crc ^ 0xffffffffL; + unsigned n; + + if (!crc_table_computed) + MakeCRCTable(); + for (n = 0; n < len; n++) { + c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8); + } + return c ^ 0xffffffffL; +} + +/* Returns the CRC of the bytes buf[0..len-1]. */ +static unsigned long CRC(const unsigned char* buf, int len) { + return UpdateCRC(0L, buf, len); +} + +/* +Compresses the data according to the gzip specification. +*/ +void ZopfliGzipCompress(const ZopfliOptions* options, + const unsigned char* in, size_t insize, + unsigned char** out, size_t* outsize) { + unsigned long crcvalue = CRC(in, insize); + unsigned char bp = 0; + + ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */ + ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */ + ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */ + ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */ + /* MTIME */ + ZOPFLI_APPEND_DATA(0, out, outsize); + ZOPFLI_APPEND_DATA(0, out, outsize); + ZOPFLI_APPEND_DATA(0, out, outsize); + ZOPFLI_APPEND_DATA(0, out, outsize); + + ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */ + ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */ + + ZopfliDeflate(options, 2 /* Dynamic block */, 1, + in, insize, &bp, out, outsize); + + /* CRC */ + ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize); + ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize); + ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize); + ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize); + + /* ISIZE */ + ZOPFLI_APPEND_DATA(insize % 256, out, outsize); + ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize); + ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize); + ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize); + + if (options->verbose) { + fprintf(stderr, + "Original Size: %d, Gzip: %d, Compression: %f%% Removed\n", + (int)insize, (int)*outsize, + 100.0 * (double)(insize - *outsize) / (double)insize); + } +} diff --git a/zopfli/gzip_container.h b/zopfli/gzip_container.h new file mode 100644 index 0000000..8f5ed90 --- /dev/null +++ b/zopfli/gzip_container.h @@ -0,0 +1,50 @@ +/* +Copyright 2013 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#ifndef ZOPFLI_GZIP_H_ +#define ZOPFLI_GZIP_H_ + +/* +Functions to compress according to the Gzip specification. +*/ + +#include "zopfli.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +Compresses according to the gzip specification and append the compressed +result to the output. + +options: global program options +out: pointer to the dynamic output array to which the result is appended. Must + be freed after use. +outsize: pointer to the dynamic output array size. +*/ +void ZopfliGzipCompress(const ZopfliOptions* options, + const unsigned char* in, size_t insize, + unsigned char** out, size_t* outsize); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* ZOPFLI_GZIP_H_ */ diff --git a/zopfli/hash.c b/zopfli/hash.c new file mode 100644 index 0000000..a3b294f --- /dev/null +++ b/zopfli/hash.c @@ -0,0 +1,135 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "hash.h" + +#include +#include +#include + +#define HASH_SHIFT 5 +#define HASH_MASK 32767 + +void ZopfliInitHash(size_t window_size, ZopfliHash* h) { + size_t i; + + h->val = 0; + h->head = (int*)malloc(sizeof(*h->head) * 65536); + h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size); + h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size); + for (i = 0; i < 65536; i++) { + h->head[i] = -1; /* -1 indicates no head so far. */ + } + for (i = 0; i < window_size; i++) { + h->prev[i] = i; /* If prev[j] == j, then prev[j] is uninitialized. */ + h->hashval[i] = -1; + } + +#ifdef ZOPFLI_HASH_SAME + h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size); + for (i = 0; i < window_size; i++) { + h->same[i] = 0; + } +#endif + +#ifdef ZOPFLI_HASH_SAME_HASH + h->val2 = 0; + h->head2 = (int*)malloc(sizeof(*h->head2) * 65536); + h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size); + h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size); + for (i = 0; i < 65536; i++) { + h->head2[i] = -1; + } + for (i = 0; i < window_size; i++) { + h->prev2[i] = i; + h->hashval2[i] = -1; + } +#endif +} + +void ZopfliCleanHash(ZopfliHash* h) { + free(h->head); + free(h->prev); + free(h->hashval); + +#ifdef ZOPFLI_HASH_SAME_HASH + free(h->head2); + free(h->prev2); + free(h->hashval2); +#endif + +#ifdef ZOPFLI_HASH_SAME + free(h->same); +#endif +} + +/* +Update the sliding hash value with the given byte. All calls to this function +must be made on consecutive input characters. Since the hash value exists out +of multiple input bytes, a few warmups with this function are needed initially. +*/ +static void UpdateHashValue(ZopfliHash* h, unsigned char c) { + h->val = (((h->val) << HASH_SHIFT) ^ (c)) & HASH_MASK; +} + +void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end, + ZopfliHash* h) { + unsigned short hpos = pos & ZOPFLI_WINDOW_MASK; +#ifdef ZOPFLI_HASH_SAME + size_t amount = 0; +#endif + + UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ? + array[pos + ZOPFLI_MIN_MATCH - 1] : 0); + h->hashval[hpos] = h->val; + if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) { + h->prev[hpos] = h->head[h->val]; + } + else h->prev[hpos] = hpos; + h->head[h->val] = hpos; + +#ifdef ZOPFLI_HASH_SAME + /* Update "same". */ + if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) { + amount = h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] - 1; + } + while (pos + amount + 1 < end && + array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) { + amount++; + } + h->same[hpos] = amount; +#endif + +#ifdef ZOPFLI_HASH_SAME_HASH + h->val2 = ((h->same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h->val; + h->hashval2[hpos] = h->val2; + if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) { + h->prev2[hpos] = h->head2[h->val2]; + } + else h->prev2[hpos] = hpos; + h->head2[h->val2] = hpos; +#endif +} + +void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end, + ZopfliHash* h) { + (void)end; + UpdateHashValue(h, array[pos + 0]); + UpdateHashValue(h, array[pos + 1]); +} diff --git a/zopfli/hash.h b/zopfli/hash.h new file mode 100644 index 0000000..79c2479 --- /dev/null +++ b/zopfli/hash.h @@ -0,0 +1,70 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +The hash for ZopfliFindLongestMatch of lz77.c. +*/ + +#ifndef ZOPFLI_HASH_H_ +#define ZOPFLI_HASH_H_ + +#include "util.h" + +typedef struct ZopfliHash { + int* head; /* Hash value to index of its most recent occurance. */ + unsigned short* prev; /* Index to index of prev. occurance of same hash. */ + int* hashval; /* Index to hash value at this index. */ + int val; /* Current hash value. */ + +#ifdef ZOPFLI_HASH_SAME_HASH + /* Fields with similar purpose as the above hash, but for the second hash with + a value that is calculated differently. */ + int* head2; /* Hash value to index of its most recent occurance. */ + unsigned short* prev2; /* Index to index of prev. occurance of same hash. */ + int* hashval2; /* Index to hash value at this index. */ + int val2; /* Current hash value. */ +#endif + +#ifdef ZOPFLI_HASH_SAME + unsigned short* same; /* Amount of repetitions of same byte after this .*/ +#endif +} ZopfliHash; + +/* Allocates and initializes all fields of ZopfliHash. */ +void ZopfliInitHash(size_t window_size, ZopfliHash* h); + +/* Frees all fields of ZopfliHash. */ +void ZopfliCleanHash(ZopfliHash* h); + +/* +Updates the hash values based on the current position in the array. All calls +to this must be made for consecutive bytes. +*/ +void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end, + ZopfliHash* h); + +/* +Prepopulates hash: +Fills in the initial values in the hash, before ZopfliUpdateHash can be used +correctly. +*/ +void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end, + ZopfliHash* h); + +#endif /* ZOPFLI_HASH_H_ */ diff --git a/zopfli/katajainen.c b/zopfli/katajainen.c new file mode 100644 index 0000000..783ea08 --- /dev/null +++ b/zopfli/katajainen.c @@ -0,0 +1,251 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +Bounded package merge algorithm, based on the paper +"A Fast and Space-Economical Algorithm for Length-Limited Coding +Jyrki Katajainen, Alistair Moffat, Andrew Turpin". +*/ + +#include "katajainen.h" +#include +#include + +typedef struct Node Node; + +/* +Nodes forming chains. Also used to represent leaves. +*/ +struct Node { + size_t weight; /* Total weight (symbol count) of this chain. */ + Node* tail; /* Previous node(s) of this chain, or 0 if none. */ + int count; /* Leaf symbol index, or number of leaves before this chain. */ + char inuse; /* Tracking for garbage collection. */ +}; + +/* +Memory pool for nodes. +*/ +typedef struct NodePool { + Node* nodes; /* The pool. */ + Node* next; /* Pointer to a possibly free node in the pool. */ + int size; /* Size of the memory pool. */ +} NodePool; + +/* +Initializes a chain node with the given values and marks it as in use. +*/ +static void InitNode(size_t weight, int count, Node* tail, Node* node) { + node->weight = weight; + node->count = count; + node->tail = tail; + node->inuse = 1; +} + +/* +Finds a free location in the memory pool. Performs garbage collection if needed. +lists: If given, used to mark in-use nodes during garbage collection. +maxbits: Size of lists. +pool: Memory pool to get free node from. +*/ +static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) { + for (;;) { + if (pool->next >= &pool->nodes[pool->size]) { + /* Garbage collection. */ + int i; + for (i = 0; i < pool->size; i++) { + pool->nodes[i].inuse = 0; + } + if (lists) { + for (i = 0; i < maxbits * 2; i++) { + Node* node; + for (node = lists[i / 2][i % 2]; node; node = node->tail) { + node->inuse = 1; + } + } + } + pool->next = &pool->nodes[0]; + } + if (!pool->next->inuse) break; /* Found one. */ + pool->next++; + } + return pool->next++; +} + + +/* +Performs a Boundary Package-Merge step. Puts a new chain in the given list. The +new chain is, depending on the weights, a leaf or a combination of two chains +from the previous list. +lists: The lists of chains. +maxbits: Number of lists. +leaves: The leaves, one per symbol. +numsymbols: Number of leaves. +pool: the node memory pool. +index: The index of the list in which a new chain or leaf is required. +final: Whether this is the last time this function is called. If it is then it + is no more needed to recursively call self. +*/ +static void BoundaryPM(Node* (*lists)[2], int maxbits, + Node* leaves, int numsymbols, NodePool* pool, int index, char final) { + Node* newchain; + Node* oldchain; + int lastcount = lists[index][1]->count; /* Count of last chain of list. */ + + if (index == 0 && lastcount >= numsymbols) return; + + newchain = GetFreeNode(lists, maxbits, pool); + oldchain = lists[index][1]; + + /* These are set up before the recursive calls below, so that there is a list + pointing to the new node, to let the garbage collection know it's in use. */ + lists[index][0] = oldchain; + lists[index][1] = newchain; + + if (index == 0) { + /* New leaf node in list 0. */ + InitNode(leaves[lastcount].weight, lastcount + 1, 0, newchain); + } else { + size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight; + if (lastcount < numsymbols && sum > leaves[lastcount].weight) { + /* New leaf inserted in list, so count is incremented. */ + InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail, + newchain); + } else { + InitNode(sum, lastcount, lists[index - 1][1], newchain); + if (!final) { + /* Two lookahead chains of previous list used up, create new ones. */ + BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0); + BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0); + } + } + } +} + +/* +Initializes each list with as lookahead chains the two leaves with lowest +weights. +*/ +static void InitLists( + NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) { + int i; + Node* node0 = GetFreeNode(0, maxbits, pool); + Node* node1 = GetFreeNode(0, maxbits, pool); + InitNode(leaves[0].weight, 1, 0, node0); + InitNode(leaves[1].weight, 2, 0, node1); + for (i = 0; i < maxbits; i++) { + lists[i][0] = node0; + lists[i][1] = node1; + } +} + +/* +Converts result of boundary package-merge to the bitlengths. The result in the +last chain of the last list contains the amount of active leaves in each list. +chain: Chain to extract the bit length from (last chain from last list). +*/ +static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) { + Node* node; + for (node = chain; node; node = node->tail) { + int i; + for (i = 0; i < node->count; i++) { + bitlengths[leaves[i].count]++; + } + } +} + +/* +Comparator for sorting the leaves. Has the function signature for qsort. +*/ +static int LeafComparator(const void* a, const void* b) { + return ((const Node*)a)->weight - ((const Node*)b)->weight; +} + +int ZopfliLengthLimitedCodeLengths( + const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) { + NodePool pool; + int i; + int numsymbols = 0; /* Amount of symbols with frequency > 0. */ + int numBoundaryPMRuns; + + /* Array of lists of chains. Each list requires only two lookahead chains at + a time, so each list is a array of two Node*'s. */ + Node* (*lists)[2]; + + /* One leaf per symbol. Only numsymbols leaves will be used. */ + Node* leaves = (Node*)malloc(n * sizeof(*leaves)); + + /* Initialize all bitlengths at 0. */ + for (i = 0; i < n; i++) { + bitlengths[i] = 0; + } + + /* Count used symbols and place them in the leaves. */ + for (i = 0; i < n; i++) { + if (frequencies[i]) { + leaves[numsymbols].weight = frequencies[i]; + leaves[numsymbols].count = i; /* Index of symbol this leaf represents. */ + numsymbols++; + } + } + + /* Check special cases and error conditions. */ + if ((1 << maxbits) < numsymbols) { + free(leaves); + return 1; /* Error, too few maxbits to represent symbols. */ + } + if (numsymbols == 0) { + free(leaves); + return 0; /* No symbols at all. OK. */ + } + if (numsymbols == 1) { + bitlengths[leaves[0].count] = 1; + free(leaves); + return 0; /* Only one symbol, give it bitlength 1, not 0. OK. */ + } + + /* Sort the leaves from lightest to heaviest. */ + qsort(leaves, numsymbols, sizeof(Node), LeafComparator); + + /* Initialize node memory pool. */ + pool.size = 2 * maxbits * (maxbits + 1); + pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes)); + pool.next = pool.nodes; + for (i = 0; i < pool.size; i++) { + pool.nodes[i].inuse = 0; + } + + lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists)); + InitLists(&pool, leaves, maxbits, lists); + + /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two + are already created in the initialization. Each BoundaryPM run creates one. */ + numBoundaryPMRuns = 2 * numsymbols - 4; + for (i = 0; i < numBoundaryPMRuns; i++) { + char final = i == numBoundaryPMRuns - 1; + BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final); + } + + ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths); + + free(lists); + free(leaves); + free(pool.nodes); + return 0; /* OK. */ +} diff --git a/zopfli/katajainen.h b/zopfli/katajainen.h new file mode 100644 index 0000000..ee8a91e --- /dev/null +++ b/zopfli/katajainen.h @@ -0,0 +1,42 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#ifndef ZOPFLI_KATAJAINEN_H_ +#define ZOPFLI_KATAJAINEN_H_ + +#include + +/* +Outputs minimum-redundancy length-limited code bitlengths for symbols with the +given counts. The bitlengths are limited by maxbits. + +The output is tailored for DEFLATE: symbols that never occur, get a bit length +of 0, and if only a single symbol occurs at least once, its bitlength will be 1, +and not 0 as would theoretically be needed for a single symbol. + +frequencies: The amount of occurances of each symbol. +n: The amount of symbols. +maxbits: Maximum bit length, inclusive. +bitlengths: Output, the bitlengths for the symbol prefix codes. +return: 0 for OK, non-0 for error. +*/ +int ZopfliLengthLimitedCodeLengths( + const size_t* frequencies, int n, int maxbits, unsigned* bitlengths); + +#endif /* ZOPFLI_KATAJAINEN_H_ */ diff --git a/zopfli/lz77.c b/zopfli/lz77.c new file mode 100644 index 0000000..26186b4 --- /dev/null +++ b/zopfli/lz77.c @@ -0,0 +1,482 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "lz77.h" +#include "util.h" + +#include +#include +#include + +void ZopfliInitLZ77Store(ZopfliLZ77Store* store) { + store->size = 0; + store->litlens = 0; + store->dists = 0; +} + +void ZopfliCleanLZ77Store(ZopfliLZ77Store* store) { + free(store->litlens); + free(store->dists); +} + +void ZopfliCopyLZ77Store( + const ZopfliLZ77Store* source, ZopfliLZ77Store* dest) { + size_t i; + ZopfliCleanLZ77Store(dest); + dest->litlens = + (unsigned short*)malloc(sizeof(*dest->litlens) * source->size); + dest->dists = (unsigned short*)malloc(sizeof(*dest->dists) * source->size); + + if (!dest->litlens || !dest->dists) exit(-1); /* Allocation failed. */ + + dest->size = source->size; + for (i = 0; i < source->size; i++) { + dest->litlens[i] = source->litlens[i]; + dest->dists[i] = source->dists[i]; + } +} + +/* +Appends the length and distance to the LZ77 arrays of the ZopfliLZ77Store. +context must be a ZopfliLZ77Store*. +*/ +void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist, + ZopfliLZ77Store* store) { + size_t size2 = store->size; /* Needed for using ZOPFLI_APPEND_DATA twice. */ + ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size); + ZOPFLI_APPEND_DATA(dist, &store->dists, &size2); +} + +/* +Gets a score of the length given the distance. Typically, the score of the +length is the length itself, but if the distance is very long, decrease the +score of the length a bit to make up for the fact that long distances use large +amounts of extra bits. + +This is not an accurate score, it is a heuristic only for the greedy LZ77 +implementation. More accurate cost models are employed later. Making this +heuristic more accurate may hurt rather than improve compression. + +The two direct uses of this heuristic are: +-avoid using a length of 3 in combination with a long distance. This only has + an effect if length == 3. +-make a slightly better choice between the two options of the lazy matching. + +Indirectly, this affects: +-the block split points if the default of block splitting first is used, in a + rather unpredictable way +-the first zopfli run, so it affects the chance of the first run being closer + to the optimal output +*/ +static int GetLengthScore(int length, int distance) { + /* + At 1024, the distance uses 9+ extra bits and this seems to be the sweet spot + on tested files. + */ + return distance > 1024 ? length - 1 : length; +} + +void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos, + unsigned short dist, unsigned short length) { + + /* TODO(lode): make this only run in a debug compile, it's for assert only. */ + size_t i; + + assert(pos + length <= datasize); + for (i = 0; i < length; i++) { + if (data[pos - dist + i] != data[pos + i]) { + assert(data[pos - dist + i] == data[pos + i]); + break; + } + } +} + +/* +Finds how long the match of scan and match is. Can be used to find how many +bytes starting from scan, and from match, are equal. Returns the last byte +after scan, which is still equal to the correspondinb byte after match. +scan is the position to compare +match is the earlier position to compare. +end is the last possible byte, beyond which to stop looking. +safe_end is a few (8) bytes before end, for comparing multiple bytes at once. +*/ +static const unsigned char* GetMatch(const unsigned char* scan, + const unsigned char* match, + const unsigned char* end, + const unsigned char* safe_end) { + + if (sizeof(size_t) == 8) { + /* 8 checks at once per array bounds check (size_t is 64-bit). */ + while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) { + scan += 8; + match += 8; + } + } else if (sizeof(unsigned int) == 4) { + /* 4 checks at once per array bounds check (unsigned int is 32-bit). */ + while (scan < safe_end + && *((unsigned int*)scan) == *((unsigned int*)match)) { + scan += 4; + match += 4; + } + } else { + /* do 8 checks at once per array bounds check. */ + while (scan < safe_end && *scan == *match && *++scan == *++match + && *++scan == *++match && *++scan == *++match + && *++scan == *++match && *++scan == *++match + && *++scan == *++match && *++scan == *++match) { + scan++; match++; + } + } + + /* The remaining few bytes. */ + while (scan != end && *scan == *match) { + scan++; match++; + } + + return scan; +} + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE +/* +Gets distance, length and sublen values from the cache if possible. +Returns 1 if it got the values from the cache, 0 if not. +Updates the limit value to a smaller one if possible with more limited +information from the cache. +*/ +static int TryGetFromLongestMatchCache(ZopfliBlockState* s, + size_t pos, size_t* limit, + unsigned short* sublen, unsigned short* distance, unsigned short* length) { + /* The LMC cache starts at the beginning of the block rather than the + beginning of the whole array. */ + size_t lmcpos = pos - s->blockstart; + + /* Length > 0 and dist 0 is invalid combination, which indicates on purpose + that this cache value is not filled in yet. */ + unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || + s->lmc->dist[lmcpos] != 0); + unsigned char limit_ok_for_cache = cache_available && + (*limit == ZOPFLI_MAX_MATCH || s->lmc->length[lmcpos] <= *limit || + (sublen && ZopfliMaxCachedSublen(s->lmc, + lmcpos, s->lmc->length[lmcpos]) >= *limit)); + + if (s->lmc && limit_ok_for_cache && cache_available) { + if (!sublen || s->lmc->length[lmcpos] + <= ZopfliMaxCachedSublen(s->lmc, lmcpos, s->lmc->length[lmcpos])) { + *length = s->lmc->length[lmcpos]; + if (*length > *limit) *length = *limit; + if (sublen) { + ZopfliCacheToSublen(s->lmc, lmcpos, *length, sublen); + *distance = sublen[*length]; + if (*limit == ZOPFLI_MAX_MATCH && *length >= ZOPFLI_MIN_MATCH) { + assert(sublen[*length] == s->lmc->dist[lmcpos]); + } + } else { + *distance = s->lmc->dist[lmcpos]; + } + return 1; + } + /* Can't use much of the cache, since the "sublens" need to be calculated, + but at least we already know when to stop. */ + *limit = s->lmc->length[lmcpos]; + } + + return 0; +} + +/* +Stores the found sublen, distance and length in the longest match cache, if +possible. +*/ +static void StoreInLongestMatchCache(ZopfliBlockState* s, + size_t pos, size_t limit, + const unsigned short* sublen, + unsigned short distance, unsigned short length) { + /* The LMC cache starts at the beginning of the block rather than the + beginning of the whole array. */ + size_t lmcpos = pos - s->blockstart; + + /* Length > 0 and dist 0 is invalid combination, which indicates on purpose + that this cache value is not filled in yet. */ + unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || + s->lmc->dist[lmcpos] != 0); + + if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) { + assert(s->lmc->length[lmcpos] == 1 && s->lmc->dist[lmcpos] == 0); + s->lmc->dist[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : distance; + s->lmc->length[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : length; + assert(!(s->lmc->length[lmcpos] == 1 && s->lmc->dist[lmcpos] == 0)); + ZopfliSublenToCache(sublen, lmcpos, length, s->lmc); + } +} +#endif + +void ZopfliFindLongestMatch(ZopfliBlockState* s, const ZopfliHash* h, + const unsigned char* array, + size_t pos, size_t size, size_t limit, + unsigned short* sublen, unsigned short* distance, unsigned short* length) { + unsigned short hpos = pos & ZOPFLI_WINDOW_MASK, p, pp; + unsigned short bestdist = 0; + unsigned short bestlength = 1; + const unsigned char* scan; + const unsigned char* match; + const unsigned char* arrayend; + const unsigned char* arrayend_safe; +#if ZOPFLI_MAX_CHAIN_HITS < ZOPFLI_WINDOW_SIZE + int chain_counter = ZOPFLI_MAX_CHAIN_HITS; /* For quitting early. */ +#endif + + unsigned dist = 0; /* Not unsigned short on purpose. */ + + int* hhead = h->head; + unsigned short* hprev = h->prev; + int* hhashval = h->hashval; + int hval = h->val; + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + if (TryGetFromLongestMatchCache(s, pos, &limit, sublen, distance, length)) { + assert(pos + *length <= size); + return; + } +#endif + + assert(limit <= ZOPFLI_MAX_MATCH); + assert(limit >= ZOPFLI_MIN_MATCH); + assert(pos < size); + + if (size - pos < ZOPFLI_MIN_MATCH) { + /* The rest of the code assumes there are at least ZOPFLI_MIN_MATCH bytes to + try. */ + *length = 0; + *distance = 0; + return; + } + + if (pos + limit > size) { + limit = size - pos; + } + arrayend = &array[pos] + limit; + arrayend_safe = arrayend - 8; + + assert(hval < 65536); + + pp = hhead[hval]; /* During the whole loop, p == hprev[pp]. */ + p = hprev[pp]; + + assert(pp == hpos); + + dist = p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp); + + /* Go through all distances. */ + while (dist < ZOPFLI_WINDOW_SIZE) { + unsigned short currentlength = 0; + + assert(p < ZOPFLI_WINDOW_SIZE); + assert(p == hprev[pp]); + assert(hhashval[p] == hval); + + if (dist > 0) { + assert(pos < size); + assert(dist <= pos); + scan = &array[pos]; + match = &array[pos - dist]; + + /* Testing the byte at position bestlength first, goes slightly faster. */ + if (pos + bestlength >= size + || *(scan + bestlength) == *(match + bestlength)) { + +#ifdef ZOPFLI_HASH_SAME + unsigned short same0 = h->same[pos & ZOPFLI_WINDOW_MASK]; + if (same0 > 2 && *scan == *match) { + unsigned short same1 = h->same[(pos - dist) & ZOPFLI_WINDOW_MASK]; + unsigned short same = same0 < same1 ? same0 : same1; + if (same > limit) same = limit; + scan += same; + match += same; + } +#endif + scan = GetMatch(scan, match, arrayend, arrayend_safe); + currentlength = scan - &array[pos]; /* The found length. */ + } + + if (currentlength > bestlength) { + if (sublen) { + unsigned short j; + for (j = bestlength + 1; j <= currentlength; j++) { + sublen[j] = dist; + } + } + bestdist = dist; + bestlength = currentlength; + if (currentlength >= limit) break; + } + } + + +#ifdef ZOPFLI_HASH_SAME_HASH + /* Switch to the other hash once this will be more efficient. */ + if (hhead != h->head2 && bestlength >= h->same[hpos] && + h->val2 == h->hashval2[p]) { + /* Now use the hash that encodes the length and first byte. */ + hhead = h->head2; + hprev = h->prev2; + hhashval = h->hashval2; + hval = h->val2; + } +#endif + + pp = p; + p = hprev[p]; + if (p == pp) break; /* Uninited prev value. */ + + dist += p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp); + +#if ZOPFLI_MAX_CHAIN_HITS < ZOPFLI_WINDOW_SIZE + chain_counter--; + if (chain_counter <= 0) break; +#endif + } + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + StoreInLongestMatchCache(s, pos, limit, sublen, bestdist, bestlength); +#endif + + assert(bestlength <= limit); + + *distance = bestdist; + *length = bestlength; + assert(pos + *length <= size); +} + +void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, + size_t instart, size_t inend, + ZopfliLZ77Store* store) { + size_t i = 0, j; + unsigned short leng; + unsigned short dist; + int lengthscore; + size_t windowstart = instart > ZOPFLI_WINDOW_SIZE + ? instart - ZOPFLI_WINDOW_SIZE : 0; + unsigned short dummysublen[259]; + + ZopfliHash hash; + ZopfliHash* h = &hash; + +#ifdef ZOPFLI_LAZY_MATCHING + /* Lazy matching. */ + unsigned prev_length = 0; + unsigned prev_match = 0; + int prevlengthscore; + int match_available = 0; +#endif + + if (instart == inend) return; + + ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h); + ZopfliWarmupHash(in, windowstart, inend, h); + for (i = windowstart; i < instart; i++) { + ZopfliUpdateHash(in, i, inend, h); + } + + for (i = instart; i < inend; i++) { + ZopfliUpdateHash(in, i, inend, h); + + ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, dummysublen, + &dist, &leng); + lengthscore = GetLengthScore(leng, dist); + +#ifdef ZOPFLI_LAZY_MATCHING + /* Lazy matching. */ + prevlengthscore = GetLengthScore(prev_length, prev_match); + if (match_available) { + match_available = 0; + if (lengthscore > prevlengthscore + 1) { + ZopfliStoreLitLenDist(in[i - 1], 0, store); + if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { + match_available = 1; + prev_length = leng; + prev_match = dist; + continue; + } + } else { + /* Add previous to output. */ + leng = prev_length; + dist = prev_match; + lengthscore = prevlengthscore; + /* Add to output. */ + ZopfliVerifyLenDist(in, inend, i - 1, dist, leng); + ZopfliStoreLitLenDist(leng, dist, store); + for (j = 2; j < leng; j++) { + assert(i < inend); + i++; + ZopfliUpdateHash(in, i, inend, h); + } + continue; + } + } + else if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { + match_available = 1; + prev_length = leng; + prev_match = dist; + continue; + } + /* End of lazy matching. */ +#endif + + /* Add to output. */ + if (lengthscore >= ZOPFLI_MIN_MATCH) { + ZopfliVerifyLenDist(in, inend, i, dist, leng); + ZopfliStoreLitLenDist(leng, dist, store); + } else { + leng = 1; + ZopfliStoreLitLenDist(in[i], 0, store); + } + for (j = 1; j < leng; j++) { + assert(i < inend); + i++; + ZopfliUpdateHash(in, i, inend, h); + } + } + + ZopfliCleanHash(h); +} + +void ZopfliLZ77Counts(const unsigned short* litlens, + const unsigned short* dists, + size_t start, size_t end, + size_t* ll_count, size_t* d_count) { + size_t i; + + for (i = 0; i < 288; i++) { + ll_count[i] = 0; + } + for (i = 0; i < 32; i++) { + d_count[i] = 0; + } + + for (i = start; i < end; i++) { + if (dists[i] == 0) { + ll_count[litlens[i]]++; + } else { + ll_count[ZopfliGetLengthSymbol(litlens[i])]++; + d_count[ZopfliGetDistSymbol(dists[i])]++; + } + } + + ll_count[256] = 1; /* End symbol. */ +} diff --git a/zopfli/lz77.h b/zopfli/lz77.h new file mode 100644 index 0000000..55186a7 --- /dev/null +++ b/zopfli/lz77.h @@ -0,0 +1,129 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +Functions for basic LZ77 compression and utilities for the "squeeze" LZ77 +compression. +*/ + +#ifndef ZOPFLI_LZ77_H_ +#define ZOPFLI_LZ77_H_ + +#include + +#include "cache.h" +#include "hash.h" +#include "zopfli.h" + +/* +Stores lit/length and dist pairs for LZ77. +Parameter litlens: Contains the literal symbols or length values. +Parameter dists: Contains the distances. A value is 0 to indicate that there is +no dist and the corresponding litlens value is a literal instead of a length. +Parameter size: The size of both the litlens and dists arrays. +The memory can best be managed by using ZopfliInitLZ77Store to initialize it, +ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values. + +*/ +typedef struct ZopfliLZ77Store { + unsigned short* litlens; /* Lit or len. */ + unsigned short* dists; /* If 0: indicates literal in corresponding litlens, + if > 0: length in corresponding litlens, this is the distance. */ + size_t size; +} ZopfliLZ77Store; + +void ZopfliInitLZ77Store(ZopfliLZ77Store* store); +void ZopfliCleanLZ77Store(ZopfliLZ77Store* store); +void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest); +void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist, + ZopfliLZ77Store* store); + +/* +Some state information for compressing a block. +This is currently a bit under-used (with mainly only the longest match cache), +but is kept for easy future expansion. +*/ +typedef struct ZopfliBlockState { + const ZopfliOptions* options; + +#ifdef ZOPFLI_LONGEST_MATCH_CACHE + /* Cache for length/distance pairs found so far. */ + ZopfliLongestMatchCache* lmc; +#endif + + /* The start (inclusive) and end (not inclusive) of the current block. */ + size_t blockstart; + size_t blockend; +} ZopfliBlockState; + +/* +Finds the longest match (length and corresponding distance) for LZ77 +compression. +Even when not using "sublen", it can be more efficient to provide an array, +because only then the caching is used. +array: the data +pos: position in the data to find the match for +size: size of the data +limit: limit length to maximum this value (default should be 258). This allows + finding a shorter dist for that length (= less extra bits). Must be + in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH]. +sublen: output array of 259 elements, or null. Has, for each length, the + smallest distance required to reach this length. Only 256 of its 259 values + are used, the first 3 are ignored (the shortest length is 3. It is purely + for convenience that the array is made 3 longer). +*/ +void ZopfliFindLongestMatch( + ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array, + size_t pos, size_t size, size_t limit, + unsigned short* sublen, unsigned short* distance, unsigned short* length); + +/* +Verifies if length and dist are indeed valid, only used for assertion. +*/ +void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos, + unsigned short dist, unsigned short length); + +/* +Counts the number of literal, length and distance symbols in the given lz77 +arrays. +litlens: lz77 lit/lengths +dists: ll77 distances +start: where to begin counting in litlens and dists +end: where to stop counting in litlens and dists (not inclusive) +ll_count: count of each lit/len symbol, must have size 288 (see deflate + standard) +d_count: count of each dist symbol, must have size 32 (see deflate standard) +*/ +void ZopfliLZ77Counts(const unsigned short* litlens, + const unsigned short* dists, + size_t start, size_t end, + size_t* ll_count, size_t* d_count); + +/* +Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than +with the slow but better "squeeze" implementation. +The result is placed in the ZopfliLZ77Store. +If instart is larger than 0, it uses values before instart as starting +dictionary. +*/ +void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, + size_t instart, size_t inend, + ZopfliLZ77Store* store); + +#endif /* ZOPFLI_LZ77_H_ */ diff --git a/zopfli/squeeze.c b/zopfli/squeeze.c new file mode 100644 index 0000000..09e7e2e --- /dev/null +++ b/zopfli/squeeze.c @@ -0,0 +1,546 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "squeeze.h" + +#include +#include +#include + +#include "blocksplitter.h" +#include "deflate.h" +#include "tree.h" +#include "util.h" + +typedef struct SymbolStats { + /* The literal and length symbols. */ + size_t litlens[288]; + /* The 32 unique dist symbols, not the 32768 possible dists. */ + size_t dists[32]; + + double ll_symbols[288]; /* Length of each lit/len symbol in bits. */ + double d_symbols[32]; /* Length of each dist symbol in bits. */ +} SymbolStats; + +/* Sets everything to 0. */ +static void InitStats(SymbolStats* stats) { + memset(stats->litlens, 0, 288 * sizeof(stats->litlens[0])); + memset(stats->dists, 0, 32 * sizeof(stats->dists[0])); + + memset(stats->ll_symbols, 0, 288 * sizeof(stats->ll_symbols[0])); + memset(stats->d_symbols, 0, 32 * sizeof(stats->d_symbols[0])); +} + +static void CopyStats(SymbolStats* source, SymbolStats* dest) { + memcpy(dest->litlens, source->litlens, 288 * sizeof(dest->litlens[0])); + memcpy(dest->dists, source->dists, 32 * sizeof(dest->dists[0])); + + memcpy(dest->ll_symbols, source->ll_symbols, + 288 * sizeof(dest->ll_symbols[0])); + memcpy(dest->d_symbols, source->d_symbols, 32 * sizeof(dest->d_symbols[0])); +} + +/* Adds the bit lengths. */ +static void AddWeighedStatFreqs(const SymbolStats* stats1, double w1, + const SymbolStats* stats2, double w2, + SymbolStats* result) { + size_t i; + for (i = 0; i < 288; i++) { + result->litlens[i] = + (size_t) (stats1->litlens[i] * w1 + stats2->litlens[i] * w2); + } + for (i = 0; i < 32; i++) { + result->dists[i] = + (size_t) (stats1->dists[i] * w1 + stats2->dists[i] * w2); + } + result->litlens[256] = 1; /* End symbol. */ +} + +typedef struct RanState { + unsigned int m_w, m_z; +} RanState; + +static void InitRanState(RanState* state) { + state->m_w = 1; + state->m_z = 2; +} + +/* Get random number: "Multiply-With-Carry" generator of G. Marsaglia */ +static unsigned int Ran(RanState* state) { + state->m_z = 36969 * (state->m_z & 65535) + (state->m_z >> 16); + state->m_w = 18000 * (state->m_w & 65535) + (state->m_w >> 16); + return (state->m_z << 16) + state->m_w; /* 32-bit result. */ +} + +static void RandomizeFreqs(RanState* state, size_t* freqs, int n) { + int i; + for (i = 0; i < n; i++) { + if ((Ran(state) >> 4) % 3 == 0) freqs[i] = freqs[Ran(state) % n]; + } +} + +static void RandomizeStatFreqs(RanState* state, SymbolStats* stats) { + RandomizeFreqs(state, stats->litlens, 288); + RandomizeFreqs(state, stats->dists, 32); + stats->litlens[256] = 1; /* End symbol. */ +} + +static void ClearStatFreqs(SymbolStats* stats) { + size_t i; + for (i = 0; i < 288; i++) stats->litlens[i] = 0; + for (i = 0; i < 32; i++) stats->dists[i] = 0; +} + +/* +Function that calculates a cost based on a model for the given LZ77 symbol. +litlen: means literal symbol if dist is 0, length otherwise. +*/ +typedef double CostModelFun(unsigned litlen, unsigned dist, void* context); + +/* +Cost model which should exactly match fixed tree. +type: CostModelFun +*/ +static double GetCostFixed(unsigned litlen, unsigned dist, void* unused) { + (void)unused; + if (dist == 0) { + if (litlen <= 143) return 8; + else return 9; + } else { + int dbits = ZopfliGetDistExtraBits(dist); + int lbits = ZopfliGetLengthExtraBits(litlen); + int lsym = ZopfliGetLengthSymbol(litlen); + double cost = 0; + if (lsym <= 279) cost += 7; + else cost += 8; + cost += 5; /* Every dist symbol has length 5. */ + return cost + dbits + lbits; + } +} + +/* +Cost model based on symbol statistics. +type: CostModelFun +*/ +static double GetCostStat(unsigned litlen, unsigned dist, void* context) { + SymbolStats* stats = (SymbolStats*)context; + if (dist == 0) { + return stats->ll_symbols[litlen]; + } else { + int lsym = ZopfliGetLengthSymbol(litlen); + int lbits = ZopfliGetLengthExtraBits(litlen); + int dsym = ZopfliGetDistSymbol(dist); + int dbits = ZopfliGetDistExtraBits(dist); + return stats->ll_symbols[lsym] + lbits + stats->d_symbols[dsym] + dbits; + } +} + +/* +Finds the minimum possible cost this cost model can return for valid length and +distance symbols. +*/ +static double GetCostModelMinCost(CostModelFun* costmodel, void* costcontext) { + double mincost; + int bestlength = 0; /* length that has lowest cost in the cost model */ + int bestdist = 0; /* distance that has lowest cost in the cost model */ + int i; + /* + Table of distances that have a different distance symbol in the deflate + specification. Each value is the first distance that has a new symbol. Only + different symbols affect the cost model so only these need to be checked. + See RFC 1951 section 3.2.5. Compressed blocks (length and distance codes). + */ + static const int dsymbols[30] = { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, + 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 + }; + + mincost = ZOPFLI_LARGE_FLOAT; + for (i = 3; i < 259; i++) { + double c = costmodel(i, 1, costcontext); + if (c < mincost) { + bestlength = i; + mincost = c; + } + } + + mincost = ZOPFLI_LARGE_FLOAT; + for (i = 0; i < 30; i++) { + double c = costmodel(3, dsymbols[i], costcontext); + if (c < mincost) { + bestdist = dsymbols[i]; + mincost = c; + } + } + + return costmodel(bestlength, bestdist, costcontext); +} + +/* +Performs the forward pass for "squeeze". Gets the most optimal length to reach +every byte from a previous byte, using cost calculations. +s: the ZopfliBlockState +in: the input data array +instart: where to start +inend: where to stop (not inclusive) +costmodel: function to calculate the cost of some lit/len/dist pair. +costcontext: abstract context for the costmodel function +length_array: output array of size (inend - instart) which will receive the best + length to reach this byte from a previous byte. +returns the cost that was, according to the costmodel, needed to get to the end. +*/ +static double GetBestLengths(ZopfliBlockState *s, + const unsigned char* in, + size_t instart, size_t inend, + CostModelFun* costmodel, void* costcontext, + unsigned short* length_array) { + /* Best cost to get here so far. */ + size_t blocksize = inend - instart; + float* costs; + size_t i = 0, k; + unsigned short leng; + unsigned short dist; + unsigned short sublen[259]; + size_t windowstart = instart > ZOPFLI_WINDOW_SIZE + ? instart - ZOPFLI_WINDOW_SIZE : 0; + ZopfliHash hash; + ZopfliHash* h = &hash; + double result; + double mincost = GetCostModelMinCost(costmodel, costcontext); + + if (instart == inend) return 0; + + costs = (float*)malloc(sizeof(float) * (blocksize + 1)); + if (!costs) exit(-1); /* Allocation failed. */ + + ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h); + ZopfliWarmupHash(in, windowstart, inend, h); + for (i = windowstart; i < instart; i++) { + ZopfliUpdateHash(in, i, inend, h); + } + + for (i = 1; i < blocksize + 1; i++) costs[i] = ZOPFLI_LARGE_FLOAT; + costs[0] = 0; /* Because it's the start. */ + length_array[0] = 0; + + for (i = instart; i < inend; i++) { + size_t j = i - instart; /* Index in the costs array and length_array. */ + ZopfliUpdateHash(in, i, inend, h); + +#ifdef ZOPFLI_SHORTCUT_LONG_REPETITIONS + /* If we're in a long repetition of the same character and have more than + ZOPFLI_MAX_MATCH characters before and after our position. */ + if (h->same[i & ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH * 2 + && i > instart + ZOPFLI_MAX_MATCH + 1 + && i + ZOPFLI_MAX_MATCH * 2 + 1 < inend + && h->same[(i - ZOPFLI_MAX_MATCH) & ZOPFLI_WINDOW_MASK] + > ZOPFLI_MAX_MATCH) { + double symbolcost = costmodel(ZOPFLI_MAX_MATCH, 1, costcontext); + /* Set the length to reach each one to ZOPFLI_MAX_MATCH, and the cost to + the cost corresponding to that length. Doing this, we skip + ZOPFLI_MAX_MATCH values to avoid calling ZopfliFindLongestMatch. */ + for (k = 0; k < ZOPFLI_MAX_MATCH; k++) { + costs[j + ZOPFLI_MAX_MATCH] = costs[j] + symbolcost; + length_array[j + ZOPFLI_MAX_MATCH] = ZOPFLI_MAX_MATCH; + i++; + j++; + ZopfliUpdateHash(in, i, inend, h); + } + } +#endif + + ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, sublen, + &dist, &leng); + + /* Literal. */ + if (i + 1 <= inend) { + double newCost = costs[j] + costmodel(in[i], 0, costcontext); + assert(newCost >= 0); + if (newCost < costs[j + 1]) { + costs[j + 1] = newCost; + length_array[j + 1] = 1; + } + } + /* Lengths. */ + for (k = 3; k <= leng && i + k <= inend; k++) { + double newCost; + + /* Calling the cost model is expensive, avoid this if we are already at + the minimum possible cost that it can return. */ + if (costs[j + k] - costs[j] <= mincost) continue; + + newCost = costs[j] + costmodel(k, sublen[k], costcontext); + assert(newCost >= 0); + if (newCost < costs[j + k]) { + assert(k <= ZOPFLI_MAX_MATCH); + costs[j + k] = newCost; + length_array[j + k] = k; + } + } + } + + assert(costs[blocksize] >= 0); + result = costs[blocksize]; + + ZopfliCleanHash(h); + free(costs); + + return result; +} + +/* +Calculates the optimal path of lz77 lengths to use, from the calculated +length_array. The length_array must contain the optimal length to reach that +byte. The path will be filled with the lengths to use, so its data size will be +the amount of lz77 symbols. +*/ +static void TraceBackwards(size_t size, const unsigned short* length_array, + unsigned short** path, size_t* pathsize) { + size_t index = size; + if (size == 0) return; + for (;;) { + ZOPFLI_APPEND_DATA(length_array[index], path, pathsize); + assert(length_array[index] <= index); + assert(length_array[index] <= ZOPFLI_MAX_MATCH); + assert(length_array[index] != 0); + index -= length_array[index]; + if (index == 0) break; + } + + /* Mirror result. */ + for (index = 0; index < *pathsize / 2; index++) { + unsigned short temp = (*path)[index]; + (*path)[index] = (*path)[*pathsize - index - 1]; + (*path)[*pathsize - index - 1] = temp; + } +} + +static void FollowPath(ZopfliBlockState* s, + const unsigned char* in, size_t instart, size_t inend, + unsigned short* path, size_t pathsize, + ZopfliLZ77Store* store) { + size_t i, j, pos = 0; + size_t windowstart = instart > ZOPFLI_WINDOW_SIZE + ? instart - ZOPFLI_WINDOW_SIZE : 0; + + size_t total_length_test = 0; + + ZopfliHash hash; + ZopfliHash* h = &hash; + + if (instart == inend) return; + + ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h); + ZopfliWarmupHash(in, windowstart, inend, h); + for (i = windowstart; i < instart; i++) { + ZopfliUpdateHash(in, i, inend, h); + } + + pos = instart; + for (i = 0; i < pathsize; i++) { + unsigned short length = path[i]; + unsigned short dummy_length; + unsigned short dist; + assert(pos < inend); + + ZopfliUpdateHash(in, pos, inend, h); + + /* Add to output. */ + if (length >= ZOPFLI_MIN_MATCH) { + /* Get the distance by recalculating longest match. The found length + should match the length from the path. */ + ZopfliFindLongestMatch(s, h, in, pos, inend, length, 0, + &dist, &dummy_length); + assert(!(dummy_length != length && length > 2 && dummy_length > 2)); + ZopfliVerifyLenDist(in, inend, pos, dist, length); + ZopfliStoreLitLenDist(length, dist, store); + total_length_test += length; + } else { + length = 1; + ZopfliStoreLitLenDist(in[pos], 0, store); + total_length_test++; + } + + + assert(pos + length <= inend); + for (j = 1; j < length; j++) { + ZopfliUpdateHash(in, pos + j, inend, h); + } + + pos += length; + } + + ZopfliCleanHash(h); +} + +/* Calculates the entropy of the statistics */ +static void CalculateStatistics(SymbolStats* stats) { + ZopfliCalculateEntropy(stats->litlens, 288, stats->ll_symbols); + ZopfliCalculateEntropy(stats->dists, 32, stats->d_symbols); +} + +/* Appends the symbol statistics from the store. */ +static void GetStatistics(const ZopfliLZ77Store* store, SymbolStats* stats) { + size_t i; + for (i = 0; i < store->size; i++) { + if (store->dists[i] == 0) { + stats->litlens[store->litlens[i]]++; + } else { + stats->litlens[ZopfliGetLengthSymbol(store->litlens[i])]++; + stats->dists[ZopfliGetDistSymbol(store->dists[i])]++; + } + } + stats->litlens[256] = 1; /* End symbol. */ + + CalculateStatistics(stats); +} + +/* +Does a single run for ZopfliLZ77Optimal. For good compression, repeated runs +with updated statistics should be performed. + +s: the block state +in: the input data array +instart: where to start +inend: where to stop (not inclusive) +path: pointer to dynamically allocated memory to store the path +pathsize: pointer to the size of the dynamic path array +length_array: array if size (inend - instart) used to store lengths +costmodel: function to use as the cost model for this squeeze run +costcontext: abstract context for the costmodel function +store: place to output the LZ77 data +returns the cost that was, according to the costmodel, needed to get to the end. + This is not the actual cost. +*/ +static double LZ77OptimalRun(ZopfliBlockState* s, + const unsigned char* in, size_t instart, size_t inend, + unsigned short** path, size_t* pathsize, + unsigned short* length_array, CostModelFun* costmodel, + void* costcontext, ZopfliLZ77Store* store) { + double cost = GetBestLengths( + s, in, instart, inend, costmodel, costcontext, length_array); + free(*path); + *path = 0; + *pathsize = 0; + TraceBackwards(inend - instart, length_array, path, pathsize); + FollowPath(s, in, instart, inend, *path, *pathsize, store); + assert(cost < ZOPFLI_LARGE_FLOAT); + return cost; +} + +void ZopfliLZ77Optimal(ZopfliBlockState *s, + const unsigned char* in, size_t instart, size_t inend, + ZopfliLZ77Store* store) { + /* Dist to get to here with smallest cost. */ + size_t blocksize = inend - instart; + unsigned short* length_array = + (unsigned short*)malloc(sizeof(unsigned short) * (blocksize + 1)); + unsigned short* path = 0; + size_t pathsize = 0; + ZopfliLZ77Store currentstore; + SymbolStats stats, beststats, laststats; + int i; + double cost; + double bestcost = ZOPFLI_LARGE_FLOAT; + double lastcost = 0; + /* Try randomizing the costs a bit once the size stabilizes. */ + RanState ran_state; + int lastrandomstep = -1; + + if (!length_array) exit(-1); /* Allocation failed. */ + + InitRanState(&ran_state); + InitStats(&stats); + ZopfliInitLZ77Store(¤tstore); + + /* Do regular deflate, then loop multiple shortest path runs, each time using + the statistics of the previous run. */ + + /* Initial run. */ + ZopfliLZ77Greedy(s, in, instart, inend, ¤tstore); + GetStatistics(¤tstore, &stats); + + /* Repeat statistics with each time the cost model from the previous stat + run. */ + for (i = 0; i < s->options->numiterations; i++) { + ZopfliCleanLZ77Store(¤tstore); + ZopfliInitLZ77Store(¤tstore); + LZ77OptimalRun(s, in, instart, inend, &path, &pathsize, + length_array, GetCostStat, (void*)&stats, + ¤tstore); + cost = ZopfliCalculateBlockSize(currentstore.litlens, currentstore.dists, + 0, currentstore.size, 2); + if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { + fprintf(stderr, "Iteration %d: %d bit\n", i, (int) cost); + } + if (cost < bestcost) { + /* Copy to the output store. */ + ZopfliCopyLZ77Store(¤tstore, store); + CopyStats(&stats, &beststats); + bestcost = cost; + } + CopyStats(&stats, &laststats); + ClearStatFreqs(&stats); + GetStatistics(¤tstore, &stats); + if (lastrandomstep != -1) { + /* This makes it converge slower but better. Do it only once the + randomness kicks in so that if the user does few iterations, it gives a + better result sooner. */ + AddWeighedStatFreqs(&stats, 1.0, &laststats, 0.5, &stats); + CalculateStatistics(&stats); + } + if (i > 5 && cost == lastcost) { + CopyStats(&beststats, &stats); + RandomizeStatFreqs(&ran_state, &stats); + CalculateStatistics(&stats); + lastrandomstep = i; + } + lastcost = cost; + } + + free(length_array); + free(path); + ZopfliCleanLZ77Store(¤tstore); +} + +void ZopfliLZ77OptimalFixed(ZopfliBlockState *s, + const unsigned char* in, + size_t instart, size_t inend, + ZopfliLZ77Store* store) +{ + /* Dist to get to here with smallest cost. */ + size_t blocksize = inend - instart; + unsigned short* length_array = + (unsigned short*)malloc(sizeof(unsigned short) * (blocksize + 1)); + unsigned short* path = 0; + size_t pathsize = 0; + + if (!length_array) exit(-1); /* Allocation failed. */ + + s->blockstart = instart; + s->blockend = inend; + + /* Shortest path for fixed tree This one should give the shortest possible + result for fixed tree, no repeated runs are needed since the tree is known. */ + LZ77OptimalRun(s, in, instart, inend, &path, &pathsize, + length_array, GetCostFixed, 0, store); + + free(length_array); + free(path); +} diff --git a/zopfli/squeeze.h b/zopfli/squeeze.h new file mode 100644 index 0000000..e850aaa --- /dev/null +++ b/zopfli/squeeze.h @@ -0,0 +1,60 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +The squeeze functions do enhanced LZ77 compression by optimal parsing with a +cost model, rather than greedily choosing the longest length or using a single +step of lazy matching like regular implementations. + +Since the cost model is based on the Huffman tree that can only be calculated +after the LZ77 data is generated, there is a chicken and egg problem, and +multiple runs are done with updated cost models to converge to a better +solution. +*/ + +#ifndef ZOPFLI_SQUEEZE_H_ +#define ZOPFLI_SQUEEZE_H_ + +#include "lz77.h" + +/* +Calculates lit/len and dist pairs for given data. +If instart is larger than 0, it uses values before instart as starting +dictionary. +*/ +void ZopfliLZ77Optimal(ZopfliBlockState *s, + const unsigned char* in, size_t instart, size_t inend, + ZopfliLZ77Store* store); + +/* +Does the same as ZopfliLZ77Optimal, but optimized for the fixed tree of the +deflate standard. +The fixed tree never gives the best compression. But this gives the best +possible LZ77 encoding possible with the fixed tree. +This does not create or output any fixed tree, only LZ77 data optimized for +using with a fixed tree. +If instart is larger than 0, it uses values before instart as starting +dictionary. +*/ +void ZopfliLZ77OptimalFixed(ZopfliBlockState *s, + const unsigned char* in, + size_t instart, size_t inend, + ZopfliLZ77Store* store); + +#endif /* ZOPFLI_SQUEEZE_H_ */ diff --git a/zopfli/tree.c b/zopfli/tree.c new file mode 100644 index 0000000..c457511 --- /dev/null +++ b/zopfli/tree.c @@ -0,0 +1,101 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "tree.h" + +#include +#include +#include +#include + +#include "katajainen.h" +#include "util.h" + +void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits, + unsigned* symbols) { + size_t* bl_count = (size_t*)malloc(sizeof(size_t) * (maxbits + 1)); + size_t* next_code = (size_t*)malloc(sizeof(size_t) * (maxbits + 1)); + unsigned bits, i; + unsigned code; + + for (i = 0; i < n; i++) { + symbols[i] = 0; + } + + /* 1) Count the number of codes for each code length. Let bl_count[N] be the + number of codes of length N, N >= 1. */ + for (bits = 0; bits <= maxbits; bits++) { + bl_count[bits] = 0; + } + for (i = 0; i < n; i++) { + assert(lengths[i] <= maxbits); + bl_count[lengths[i]]++; + } + /* 2) Find the numerical value of the smallest code for each code length. */ + code = 0; + bl_count[0] = 0; + for (bits = 1; bits <= maxbits; bits++) { + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = code; + } + /* 3) Assign numerical values to all codes, using consecutive values for all + codes of the same length with the base values determined at step 2. */ + for (i = 0; i < n; i++) { + unsigned len = lengths[i]; + if (len != 0) { + symbols[i] = next_code[len]; + next_code[len]++; + } + } + + free(bl_count); + free(next_code); +} + +void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths) { + static const double kInvLog2 = 1.4426950408889; /* 1.0 / log(2.0) */ + unsigned sum = 0; + unsigned i; + double log2sum; + for (i = 0; i < n; ++i) { + sum += count[i]; + } + log2sum = (sum == 0 ? log(n) : log(sum)) * kInvLog2; + for (i = 0; i < n; ++i) { + /* When the count of the symbol is 0, but its cost is requested anyway, it + means the symbol will appear at least once anyway, so give it the cost as if + its count is 1.*/ + if (count[i] == 0) bitlengths[i] = log2sum; + else bitlengths[i] = log2sum - log(count[i]) * kInvLog2; + /* Depending on compiler and architecture, the above subtraction of two + floating point numbers may give a negative result very close to zero + instead of zero (e.g. -5.973954e-17 with gcc 4.1.2 on Ubuntu 11.4). Clamp + it to zero. These floating point imprecisions do not affect the cost model + significantly so this is ok. */ + if (bitlengths[i] < 0 && bitlengths[i] > -1e-5) bitlengths[i] = 0; + assert(bitlengths[i] >= 0); + } +} + +void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits, + unsigned* bitlengths) { + int error = ZopfliLengthLimitedCodeLengths(count, n, maxbits, bitlengths); + (void) error; + assert(!error); +} diff --git a/zopfli/tree.h b/zopfli/tree.h new file mode 100644 index 0000000..4d6f469 --- /dev/null +++ b/zopfli/tree.h @@ -0,0 +1,51 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +Utilities for creating and using Huffman trees. +*/ + +#ifndef ZOPFLI_TREE_H_ +#define ZOPFLI_TREE_H_ + +#include + +/* +Calculates the bitlengths for the Huffman tree, based on the counts of each +symbol. +*/ +void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits, + unsigned *bitlengths); + +/* +Converts a series of Huffman tree bitlengths, to the bit values of the symbols. +*/ +void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits, + unsigned* symbols); + +/* +Calculates the entropy of each symbol, based on the counts of each symbol. The +result is similar to the result of ZopfliCalculateBitLengths, but with the +actual theoritical bit lengths according to the entropy. Since the resulting +values are fractional, they cannot be used to encode the tree specified by +DEFLATE. +*/ +void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths); + +#endif /* ZOPFLI_TREE_H_ */ diff --git a/zopfli/util.c b/zopfli/util.c new file mode 100644 index 0000000..d207145 --- /dev/null +++ b/zopfli/util.c @@ -0,0 +1,213 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "util.h" + +#include "zopfli.h" + +#include +#include +#include + +int ZopfliGetDistExtraBits(int dist) { +#ifdef __GNUC__ + if (dist < 5) return 0; + return (31 ^ __builtin_clz(dist - 1)) - 1; /* log2(dist - 1) - 1 */ +#else + if (dist < 5) return 0; + else if (dist < 9) return 1; + else if (dist < 17) return 2; + else if (dist < 33) return 3; + else if (dist < 65) return 4; + else if (dist < 129) return 5; + else if (dist < 257) return 6; + else if (dist < 513) return 7; + else if (dist < 1025) return 8; + else if (dist < 2049) return 9; + else if (dist < 4097) return 10; + else if (dist < 8193) return 11; + else if (dist < 16385) return 12; + else return 13; +#endif +} + +int ZopfliGetDistExtraBitsValue(int dist) { +#ifdef __GNUC__ + if (dist < 5) { + return 0; + } else { + int l = 31 ^ __builtin_clz(dist - 1); /* log2(dist - 1) */ + return (dist - (1 + (1 << l))) & ((1 << (l - 1)) - 1); + } +#else + if (dist < 5) return 0; + else if (dist < 9) return (dist - 5) & 1; + else if (dist < 17) return (dist - 9) & 3; + else if (dist < 33) return (dist - 17) & 7; + else if (dist < 65) return (dist - 33) & 15; + else if (dist < 129) return (dist - 65) & 31; + else if (dist < 257) return (dist - 129) & 63; + else if (dist < 513) return (dist - 257) & 127; + else if (dist < 1025) return (dist - 513) & 255; + else if (dist < 2049) return (dist - 1025) & 511; + else if (dist < 4097) return (dist - 2049) & 1023; + else if (dist < 8193) return (dist - 4097) & 2047; + else if (dist < 16385) return (dist - 8193) & 4095; + else return (dist - 16385) & 8191; +#endif +} + +int ZopfliGetDistSymbol(int dist) { +#ifdef __GNUC__ + if (dist < 5) { + return dist - 1; + } else { + int l = (31 ^ __builtin_clz(dist - 1)); /* log2(dist - 1) */ + int r = ((dist - 1) >> (l - 1)) & 1; + return l * 2 + r; + } +#else + if (dist < 193) { + if (dist < 13) { /* dist 0..13. */ + if (dist < 5) return dist - 1; + else if (dist < 7) return 4; + else if (dist < 9) return 5; + else return 6; + } else { /* dist 13..193. */ + if (dist < 17) return 7; + else if (dist < 25) return 8; + else if (dist < 33) return 9; + else if (dist < 49) return 10; + else if (dist < 65) return 11; + else if (dist < 97) return 12; + else if (dist < 129) return 13; + else return 14; + } + } else { + if (dist < 2049) { /* dist 193..2049. */ + if (dist < 257) return 15; + else if (dist < 385) return 16; + else if (dist < 513) return 17; + else if (dist < 769) return 18; + else if (dist < 1025) return 19; + else if (dist < 1537) return 20; + else return 21; + } else { /* dist 2049..32768. */ + if (dist < 3073) return 22; + else if (dist < 4097) return 23; + else if (dist < 6145) return 24; + else if (dist < 8193) return 25; + else if (dist < 12289) return 26; + else if (dist < 16385) return 27; + else if (dist < 24577) return 28; + else return 29; + } + } +#endif +} + +int ZopfliGetLengthExtraBits(int l) { + static const int table[259] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0 + }; + return table[l]; +} + +int ZopfliGetLengthExtraBitsValue(int l) { + static const int table[259] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0, + 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, + 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0 + }; + return table[l]; +} + +/* +Returns symbol in range [257-285] (inclusive). +*/ +int ZopfliGetLengthSymbol(int l) { + static const int table[259] = { + 0, 0, 0, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 265, 266, 266, 267, 267, 268, 268, + 269, 269, 269, 269, 270, 270, 270, 270, + 271, 271, 271, 271, 272, 272, 272, 272, + 273, 273, 273, 273, 273, 273, 273, 273, + 274, 274, 274, 274, 274, 274, 274, 274, + 275, 275, 275, 275, 275, 275, 275, 275, + 276, 276, 276, 276, 276, 276, 276, 276, + 277, 277, 277, 277, 277, 277, 277, 277, + 277, 277, 277, 277, 277, 277, 277, 277, + 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 278, 278, 278, 278, 278, 278, + 279, 279, 279, 279, 279, 279, 279, 279, + 279, 279, 279, 279, 279, 279, 279, 279, + 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, + 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, + 282, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, 282, + 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, + 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 285 + }; + return table[l]; +} + +void ZopfliInitOptions(ZopfliOptions* options) { + options->verbose = 0; + options->verbose_more = 0; + options->numiterations = 15; + options->blocksplitting = 1; + options->blocksplittinglast = 0; + options->blocksplittingmax = 15; +} diff --git a/zopfli/util.h b/zopfli/util.h new file mode 100644 index 0000000..4188f51 --- /dev/null +++ b/zopfli/util.h @@ -0,0 +1,175 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +Several utilities, including: #defines to try different compression results, +basic deflate specification values and generic program options. +*/ + +#ifndef ZOPFLI_UTIL_H_ +#define ZOPFLI_UTIL_H_ + +#include +#include + +/* Minimum and maximum length that can be encoded in deflate. */ +#define ZOPFLI_MAX_MATCH 258 +#define ZOPFLI_MIN_MATCH 3 + +/* +The window size for deflate. Must be a power of two. This should be 32768, the +maximum possible by the deflate spec. Anything less hurts compression more than +speed. +*/ +#define ZOPFLI_WINDOW_SIZE 32768 + +/* +The window mask used to wrap indices into the window. This is why the +window size must be a power of two. +*/ +#define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1) + +/* +A block structure of huge, non-smart, blocks to divide the input into, to allow +operating on huge files without exceeding memory, such as the 1GB wiki9 corpus. +The whole compression algorithm, including the smarter block splitting, will +be executed independently on each huge block. +Dividing into huge blocks hurts compression, but not much relative to the size. +Set this to, for example, 20MB (20000000). Set it to 0 to disable master blocks. +*/ +#define ZOPFLI_MASTER_BLOCK_SIZE 20000000 + +/* +Used to initialize costs for example +*/ +#define ZOPFLI_LARGE_FLOAT 1e30 + +/* +For longest match cache. max 256. Uses huge amounts of memory but makes it +faster. Uses this many times three bytes per single byte of the input data. +This is so because longest match finding has to find the exact distance +that belongs to each length for the best lz77 strategy. +Good values: e.g. 5, 8. +*/ +#define ZOPFLI_CACHE_LENGTH 8 + +/* +limit the max hash chain hits for this hash value. This has an effect only +on files where the hash value is the same very often. On these files, this +gives worse compression (the value should ideally be 32768, which is the +ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it +faster on some specific files. +Good value: e.g. 8192. +*/ +#define ZOPFLI_MAX_CHAIN_HITS 8192 + +/* +Whether to use the longest match cache for ZopfliFindLongestMatch. This cache +consumes a lot of memory but speeds it up. No effect on compression size. +*/ +#define ZOPFLI_LONGEST_MATCH_CACHE + +/* +Enable to remember amount of successive identical bytes in the hash chain for +finding longest match +required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS +This has no effect on the compression result, and enabling it increases speed. +*/ +#define ZOPFLI_HASH_SAME + +/* +Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the +best length so far is long enough. This is way faster for files with lots of +identical bytes, on which the compressor is otherwise too slow. Regular files +are unaffected or maybe a tiny bit slower. +This has no effect on the compression result, only on speed. +*/ +#define ZOPFLI_HASH_SAME_HASH + +/* +Enable this, to avoid slowness for files which are a repetition of the same +character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect +the compression result. +*/ +#define ZOPFLI_SHORTCUT_LONG_REPETITIONS + +/* +Whether to use lazy matching in the greedy LZ77 implementation. This gives a +better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77 +varies from file to file. +*/ +#define ZOPFLI_LAZY_MATCHING + +/* +Gets the symbol for the given length, cfr. the DEFLATE spec. +Returns the symbol in the range [257-285] (inclusive) +*/ +int ZopfliGetLengthSymbol(int l); + +/* Gets the amount of extra bits for the given length, cfr. the DEFLATE spec. */ +int ZopfliGetLengthExtraBits(int l); + +/* Gets value of the extra bits for the given length, cfr. the DEFLATE spec. */ +int ZopfliGetLengthExtraBitsValue(int l); + +/* Gets the symbol for the given dist, cfr. the DEFLATE spec. */ +int ZopfliGetDistSymbol(int dist); + +/* Gets the amount of extra bits for the given dist, cfr. the DEFLATE spec. */ +int ZopfliGetDistExtraBits(int dist); + +/* Gets value of the extra bits for the given dist, cfr. the DEFLATE spec. */ +int ZopfliGetDistExtraBitsValue(int dist); + +/* +Appends value to dynamically allocated memory, doubling its allocation size +whenever needed. + +value: the value to append, type T +data: pointer to the dynamic array to append to, type T** +size: pointer to the size of the array to append to, type size_t*. This is the +size that you consider the array to be, not the internal allocation size. +Precondition: allocated size of data is at least a power of two greater than or +equal than *size. +*/ +#ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */ +#define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\ + if (!((*size) & ((*size) - 1))) {\ + /*double alloc size if it's a power of two*/\ + void** data_void = reinterpret_cast(data);\ + *data_void = (*size) == 0 ? malloc(sizeof(**data))\ + : realloc((*data), (*size) * 2 * sizeof(**data));\ + }\ + (*data)[(*size)] = (value);\ + (*size)++;\ +} +#else /* C gives problems with strict-aliasing rules for (void**) cast */ +#define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\ + if (!((*size) & ((*size) - 1))) {\ + /*double alloc size if it's a power of two*/\ + (*data) = (*size) == 0 ? malloc(sizeof(**data))\ + : realloc((*data), (*size) * 2 * sizeof(**data));\ + }\ + (*data)[(*size)] = (value);\ + (*size)++;\ +} +#endif + + +#endif /* ZOPFLI_UTIL_H_ */ diff --git a/zopfli/zlib_container.c b/zopfli/zlib_container.c new file mode 100644 index 0000000..5b7d0aa --- /dev/null +++ b/zopfli/zlib_container.c @@ -0,0 +1,79 @@ +/* +Copyright 2013 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "zlib_container.h" +#include "util.h" + +#include + +#include "deflate.h" + + +/* Calculates the adler32 checksum of the data */ +static unsigned adler32(const unsigned char* data, size_t size) +{ + static const unsigned sums_overflow = 5550; + unsigned s1 = 1; + unsigned s2 = 1 >> 16; + + while (size > 0) { + size_t amount = size > sums_overflow ? sums_overflow : size; + size -= amount; + while (amount > 0) { + s1 += (*data++); + s2 += s1; + amount--; + } + s1 %= 65521; + s2 %= 65521; + } + + return (s2 << 16) | s1; +} + +void ZopfliZlibCompress(const ZopfliOptions* options, + const unsigned char* in, size_t insize, + unsigned char** out, size_t* outsize) { + unsigned char bitpointer = 0; + unsigned checksum = adler32(in, (unsigned)insize); + unsigned cmf = 120; /* CM 8, CINFO 7. See zlib spec.*/ + unsigned flevel = 0; + unsigned fdict = 0; + unsigned cmfflg = 256 * cmf + fdict * 32 + flevel * 64; + unsigned fcheck = 31 - cmfflg % 31; + cmfflg += fcheck; + + ZOPFLI_APPEND_DATA(cmfflg / 256, out, outsize); + ZOPFLI_APPEND_DATA(cmfflg % 256, out, outsize); + + ZopfliDeflate(options, 2 /* dynamic block */, 1 /* final */, + in, insize, &bitpointer, out, outsize); + + ZOPFLI_APPEND_DATA((checksum >> 24) % 256, out, outsize); + ZOPFLI_APPEND_DATA((checksum >> 16) % 256, out, outsize); + ZOPFLI_APPEND_DATA((checksum >> 8) % 256, out, outsize); + ZOPFLI_APPEND_DATA(checksum % 256, out, outsize); + + if (options->verbose) { + fprintf(stderr, + "Original Size: %d, Zlib: %d, Compression: %f%% Removed\n", + (int)insize, (int)*outsize, + 100.0 * (double)(insize - *outsize) / (double)insize); + } +} diff --git a/zopfli/zlib_container.h b/zopfli/zlib_container.h new file mode 100644 index 0000000..9ddfb9c --- /dev/null +++ b/zopfli/zlib_container.h @@ -0,0 +1,50 @@ +/* +Copyright 2013 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#ifndef ZOPFLI_ZLIB_H_ +#define ZOPFLI_ZLIB_H_ + +/* +Functions to compress according to the Zlib specification. +*/ + +#include "zopfli.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +Compresses according to the zlib specification and append the compressed +result to the output. + +options: global program options +out: pointer to the dynamic output array to which the result is appended. Must + be freed after use. +outsize: pointer to the dynamic output array size. +*/ +void ZopfliZlibCompress(const ZopfliOptions* options, + const unsigned char* in, size_t insize, + unsigned char** out, size_t* outsize); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* ZOPFLI_ZLIB_H_ */ diff --git a/zopfli/zopfli.h b/zopfli/zopfli.h new file mode 100644 index 0000000..56512a2 --- /dev/null +++ b/zopfli/zopfli.h @@ -0,0 +1,97 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#ifndef ZOPFLI_ZOPFLI_H_ +#define ZOPFLI_ZOPFLI_H_ + +#include +#include /* for size_t */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +Options used throughout the program. +*/ +typedef struct ZopfliOptions { + /* Whether to print output */ + int verbose; + + /* Whether to print more detailed output */ + int verbose_more; + + /* + Maximum amount of times to rerun forward and backward pass to optimize LZ77 + compression cost. Good values: 10, 15 for small files, 5 for files over + several MB in size or it will be too slow. + */ + int numiterations; + + /* + If true, splits the data in multiple deflate blocks with optimal choice + for the block boundaries. Block splitting gives better compression. Default: + true (1). + */ + int blocksplitting; + + /* + If true, chooses the optimal block split points only after doing the iterative + LZ77 compression. If false, chooses the block split points first, then does + iterative LZ77 on each individual block. Depending on the file, either first + or last gives the best compression. Default: false (0). + */ + int blocksplittinglast; + + /* + Maximum amount of blocks to split into (0 for unlimited, but this can give + extreme results that hurt compression on some files). Default value: 15. + */ + int blocksplittingmax; +} ZopfliOptions; + +/* Initializes options with default values. */ +void ZopfliInitOptions(ZopfliOptions* options); + +/* Output format */ +typedef enum { + ZOPFLI_FORMAT_GZIP, + ZOPFLI_FORMAT_ZLIB, + ZOPFLI_FORMAT_DEFLATE +} ZopfliFormat; + +/* +Compresses according to the given output format and appends the result to the +output. + +options: global program options +output_type: the output format to use +out: pointer to the dynamic output array to which the result is appended. Must + be freed after use +outsize: pointer to the dynamic output array size +*/ +void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type, + const unsigned char* in, size_t insize, + unsigned char** out, size_t* outsize); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* ZOPFLI_ZOPFLI_H_ */ diff --git a/zopfli/zopfli_bin.c b/zopfli/zopfli_bin.c new file mode 100644 index 0000000..8a147ef --- /dev/null +++ b/zopfli/zopfli_bin.c @@ -0,0 +1,203 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +/* +Zopfli compressor program. It can output gzip-, zlib- or deflate-compatible +data. By default it creates a .gz file. This tool can only compress, not +decompress. Decompression can be done by any standard gzip, zlib or deflate +decompressor. +*/ + +#include +#include +#include +#include + +#include "deflate.h" +#include "gzip_container.h" +#include "zlib_container.h" + +/* +Loads a file into a memory array. +*/ +static void LoadFile(const char* filename, + unsigned char** out, size_t* outsize) { + FILE* file; + + *out = 0; + *outsize = 0; + file = fopen(filename, "rb"); + if (!file) return; + + fseek(file , 0 , SEEK_END); + *outsize = ftell(file); + rewind(file); + + *out = (unsigned char*)malloc(*outsize); + + if (*outsize && (*out)) { + size_t testsize = fread(*out, 1, *outsize, file); + if (testsize != *outsize) { + /* It could be a directory */ + free(*out); + *out = 0; + *outsize = 0; + } + } + + assert(!(*outsize) || out); /* If size is not zero, out must be allocated. */ + fclose(file); +} + +/* +Saves a file from a memory array, overwriting the file if it existed. +*/ +static void SaveFile(const char* filename, + const unsigned char* in, size_t insize) { + FILE* file = fopen(filename, "wb" ); + assert(file); + fwrite((char*)in, 1, insize, file); + fclose(file); +} + +/* +outfilename: filename to write output to, or 0 to write to stdout instead +*/ +static void CompressFile(const ZopfliOptions* options, + ZopfliFormat output_type, + const char* infilename, + const char* outfilename) { + unsigned char* in; + size_t insize; + unsigned char* out = 0; + size_t outsize = 0; + LoadFile(infilename, &in, &insize); + if (insize == 0) { + fprintf(stderr, "Invalid filename: %s\n", infilename); + return; + } + + ZopfliCompress(options, output_type, in, insize, &out, &outsize); + + if (outfilename) { + SaveFile(outfilename, out, outsize); + } else { + size_t i; + for (i = 0; i < outsize; i++) { + /* Works only if terminal does not convert newlines. */ + printf("%c", out[i]); + } + } + + free(out); + free(in); +} + +/* +Add two strings together. Size does not matter. Result must be freed. +*/ +static char* AddStrings(const char* str1, const char* str2) { + size_t len = strlen(str1) + strlen(str2); + char* result = (char*)malloc(len + 1); + if (!result) exit(-1); /* Allocation failed. */ + strcpy(result, str1); + strcat(result, str2); + return result; +} + +static char StringsEqual(const char* str1, const char* str2) { + return strcmp(str1, str2) == 0; +} + +int main(int argc, char* argv[]) { + ZopfliOptions options; + ZopfliFormat output_type = ZOPFLI_FORMAT_GZIP; + const char* filename = 0; + int output_to_stdout = 0; + int i; + + ZopfliInitOptions(&options); + + for (i = 1; i < argc; i++) { + const char* arg = argv[i]; + if (StringsEqual(arg, "-v")) options.verbose = 1; + else if (StringsEqual(arg, "-c")) output_to_stdout = 1; + else if (StringsEqual(arg, "--deflate")) { + output_type = ZOPFLI_FORMAT_DEFLATE; + } + else if (StringsEqual(arg, "--zlib")) output_type = ZOPFLI_FORMAT_ZLIB; + else if (StringsEqual(arg, "--gzip")) output_type = ZOPFLI_FORMAT_GZIP; + else if (StringsEqual(arg, "--splitlast")) options.blocksplittinglast = 1; + else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i' + && arg[3] >= '0' && arg[3] <= '9') { + options.numiterations = atoi(arg + 3); + } + else if (StringsEqual(arg, "-h")) { + fprintf(stderr, + "Usage: zopfli [OPTION]... FILE\n" + " -h gives this help\n" + " -c write the result on standard output, instead of disk" + " filename + '.gz'\n" + " -v verbose mode\n" + " --i# perform # iterations (default 15). More gives" + " more compression but is slower." + " Examples: --i10, --i50, --i1000\n"); + fprintf(stderr, + " --gzip output to gzip format (default)\n" + " --zlib output to zlib format instead of gzip\n" + " --deflate output to deflate format instead of gzip\n" + " --splitlast do block splitting last instead of first\n"); + return 0; + } + } + + if (options.numiterations < 1) { + fprintf(stderr, "Error: must have 1 or more iterations"); + return 0; + } + + for (i = 1; i < argc; i++) { + if (argv[i][0] != '-') { + char* outfilename; + filename = argv[i]; + if (output_to_stdout) { + outfilename = 0; + } else if (output_type == ZOPFLI_FORMAT_GZIP) { + outfilename = AddStrings(filename, ".gz"); + } else if (output_type == ZOPFLI_FORMAT_ZLIB) { + outfilename = AddStrings(filename, ".zlib"); + } else { + assert(output_type == ZOPFLI_FORMAT_DEFLATE); + outfilename = AddStrings(filename, ".deflate"); + } + if (options.verbose && outfilename) { + fprintf(stderr, "Saving to: %s\n", outfilename); + } + CompressFile(&options, output_type, filename, outfilename); + free(outfilename); + } + } + + if (!filename) { + fprintf(stderr, + "Please provide filename\nFor help, type: %s -h\n", argv[0]); + } + + return 0; +} diff --git a/zopfli/zopfli_lib.c b/zopfli/zopfli_lib.c new file mode 100644 index 0000000..5f5b214 --- /dev/null +++ b/zopfli/zopfli_lib.c @@ -0,0 +1,42 @@ +/* +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) +*/ + +#include "zopfli.h" + +#include "deflate.h" +#include "gzip_container.h" +#include "zlib_container.h" + +#include + +void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type, + const unsigned char* in, size_t insize, + unsigned char** out, size_t* outsize) { + if (output_type == ZOPFLI_FORMAT_GZIP) { + ZopfliGzipCompress(options, in, insize, out, outsize); + } else if (output_type == ZOPFLI_FORMAT_ZLIB) { + ZopfliZlibCompress(options, in, insize, out, outsize); + } else if (output_type == ZOPFLI_FORMAT_DEFLATE) { + unsigned char bp = 0; + ZopfliDeflate(options, 2 /* Dynamic block */, 1, + in, insize, &bp, out, outsize); + } else { + assert(0); + } +} diff --git a/zopfli_bin.c b/zopfli_bin.c deleted file mode 100644 index 8a147ef..0000000 --- a/zopfli_bin.c +++ /dev/null @@ -1,203 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -/* -Zopfli compressor program. It can output gzip-, zlib- or deflate-compatible -data. By default it creates a .gz file. This tool can only compress, not -decompress. Decompression can be done by any standard gzip, zlib or deflate -decompressor. -*/ - -#include -#include -#include -#include - -#include "deflate.h" -#include "gzip_container.h" -#include "zlib_container.h" - -/* -Loads a file into a memory array. -*/ -static void LoadFile(const char* filename, - unsigned char** out, size_t* outsize) { - FILE* file; - - *out = 0; - *outsize = 0; - file = fopen(filename, "rb"); - if (!file) return; - - fseek(file , 0 , SEEK_END); - *outsize = ftell(file); - rewind(file); - - *out = (unsigned char*)malloc(*outsize); - - if (*outsize && (*out)) { - size_t testsize = fread(*out, 1, *outsize, file); - if (testsize != *outsize) { - /* It could be a directory */ - free(*out); - *out = 0; - *outsize = 0; - } - } - - assert(!(*outsize) || out); /* If size is not zero, out must be allocated. */ - fclose(file); -} - -/* -Saves a file from a memory array, overwriting the file if it existed. -*/ -static void SaveFile(const char* filename, - const unsigned char* in, size_t insize) { - FILE* file = fopen(filename, "wb" ); - assert(file); - fwrite((char*)in, 1, insize, file); - fclose(file); -} - -/* -outfilename: filename to write output to, or 0 to write to stdout instead -*/ -static void CompressFile(const ZopfliOptions* options, - ZopfliFormat output_type, - const char* infilename, - const char* outfilename) { - unsigned char* in; - size_t insize; - unsigned char* out = 0; - size_t outsize = 0; - LoadFile(infilename, &in, &insize); - if (insize == 0) { - fprintf(stderr, "Invalid filename: %s\n", infilename); - return; - } - - ZopfliCompress(options, output_type, in, insize, &out, &outsize); - - if (outfilename) { - SaveFile(outfilename, out, outsize); - } else { - size_t i; - for (i = 0; i < outsize; i++) { - /* Works only if terminal does not convert newlines. */ - printf("%c", out[i]); - } - } - - free(out); - free(in); -} - -/* -Add two strings together. Size does not matter. Result must be freed. -*/ -static char* AddStrings(const char* str1, const char* str2) { - size_t len = strlen(str1) + strlen(str2); - char* result = (char*)malloc(len + 1); - if (!result) exit(-1); /* Allocation failed. */ - strcpy(result, str1); - strcat(result, str2); - return result; -} - -static char StringsEqual(const char* str1, const char* str2) { - return strcmp(str1, str2) == 0; -} - -int main(int argc, char* argv[]) { - ZopfliOptions options; - ZopfliFormat output_type = ZOPFLI_FORMAT_GZIP; - const char* filename = 0; - int output_to_stdout = 0; - int i; - - ZopfliInitOptions(&options); - - for (i = 1; i < argc; i++) { - const char* arg = argv[i]; - if (StringsEqual(arg, "-v")) options.verbose = 1; - else if (StringsEqual(arg, "-c")) output_to_stdout = 1; - else if (StringsEqual(arg, "--deflate")) { - output_type = ZOPFLI_FORMAT_DEFLATE; - } - else if (StringsEqual(arg, "--zlib")) output_type = ZOPFLI_FORMAT_ZLIB; - else if (StringsEqual(arg, "--gzip")) output_type = ZOPFLI_FORMAT_GZIP; - else if (StringsEqual(arg, "--splitlast")) options.blocksplittinglast = 1; - else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i' - && arg[3] >= '0' && arg[3] <= '9') { - options.numiterations = atoi(arg + 3); - } - else if (StringsEqual(arg, "-h")) { - fprintf(stderr, - "Usage: zopfli [OPTION]... FILE\n" - " -h gives this help\n" - " -c write the result on standard output, instead of disk" - " filename + '.gz'\n" - " -v verbose mode\n" - " --i# perform # iterations (default 15). More gives" - " more compression but is slower." - " Examples: --i10, --i50, --i1000\n"); - fprintf(stderr, - " --gzip output to gzip format (default)\n" - " --zlib output to zlib format instead of gzip\n" - " --deflate output to deflate format instead of gzip\n" - " --splitlast do block splitting last instead of first\n"); - return 0; - } - } - - if (options.numiterations < 1) { - fprintf(stderr, "Error: must have 1 or more iterations"); - return 0; - } - - for (i = 1; i < argc; i++) { - if (argv[i][0] != '-') { - char* outfilename; - filename = argv[i]; - if (output_to_stdout) { - outfilename = 0; - } else if (output_type == ZOPFLI_FORMAT_GZIP) { - outfilename = AddStrings(filename, ".gz"); - } else if (output_type == ZOPFLI_FORMAT_ZLIB) { - outfilename = AddStrings(filename, ".zlib"); - } else { - assert(output_type == ZOPFLI_FORMAT_DEFLATE); - outfilename = AddStrings(filename, ".deflate"); - } - if (options.verbose && outfilename) { - fprintf(stderr, "Saving to: %s\n", outfilename); - } - CompressFile(&options, output_type, filename, outfilename); - free(outfilename); - } - } - - if (!filename) { - fprintf(stderr, - "Please provide filename\nFor help, type: %s -h\n", argv[0]); - } - - return 0; -} diff --git a/zopfli_lib.c b/zopfli_lib.c deleted file mode 100644 index 5f5b214..0000000 --- a/zopfli_lib.c +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: lode.vandevenne@gmail.com (Lode Vandevenne) -Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) -*/ - -#include "zopfli.h" - -#include "deflate.h" -#include "gzip_container.h" -#include "zlib_container.h" - -#include - -void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type, - const unsigned char* in, size_t insize, - unsigned char** out, size_t* outsize) { - if (output_type == ZOPFLI_FORMAT_GZIP) { - ZopfliGzipCompress(options, in, insize, out, outsize); - } else if (output_type == ZOPFLI_FORMAT_ZLIB) { - ZopfliZlibCompress(options, in, insize, out, outsize); - } else if (output_type == ZOPFLI_FORMAT_DEFLATE) { - unsigned char bp = 0; - ZopfliDeflate(options, 2 /* Dynamic block */, 1, - in, insize, &bp, out, outsize); - } else { - assert(0); - } -} -- cgit v1.2.3 From e898b4faa7158e62b6da20bea7be1fc3ab55ca92 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 14:19:51 -0400 Subject: Split license terms for extended/modified WOFF sources and bundled Zopfli sources into separate files. Add a COPYING file to summarize which terms apply to which sources. --- COPYING | 9 +++++++++ LICENSE | 33 --------------------------------- LICENSE-WOFF | 17 +++++++++++++++++ LICENSE-ZOPFLI | 13 +++++++++++++ 4 files changed, 39 insertions(+), 33 deletions(-) create mode 100644 COPYING delete mode 100644 LICENSE create mode 100644 LICENSE-WOFF create mode 100644 LICENSE-ZOPFLI diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..41f85b6 --- /dev/null +++ b/COPYING @@ -0,0 +1,9 @@ +Sources in the top-level directory belong to, or are based on, the WOFF +reference implementation originally published at +http://people.mozilla.org/~jkew/woff/, and are distributed under the terms +specified in the file LICENSE-WOFF (MPL 1.1/GPL 2.0/LGPL 2.1). + +Additionally, the zopfli/ directory contains a bundled copy of Zopfli +(https://github.com/google/zopfli), distributed under the terms in the file +LICENSE-ZOPFLI (Apache License 2.0). When this bundled library is used, these +terms also apply to the compiled software. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index aeb601f..0000000 --- a/LICENSE +++ /dev/null @@ -1,33 +0,0 @@ -Copyright 2011 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ---------------------------------------------------------------------------- - -Version: MPL 1.1/GPL 2.0/LGPL 2.1 - -The contents of this file are subject to the Mozilla Public License Version -1.1 (the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at -http://www.mozilla.org/MPL/ - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the -License. - -The Original Code is WOFF font packaging code. - -The Initial Developer of the Original Code is Mozilla Corporation. -Portions created by the Initial Developer are Copyright (C) 2009 -the Initial Developer. All Rights Reserved. diff --git a/LICENSE-WOFF b/LICENSE-WOFF new file mode 100644 index 0000000..722240a --- /dev/null +++ b/LICENSE-WOFF @@ -0,0 +1,17 @@ +Version: MPL 1.1/GPL 2.0/LGPL 2.1 + +The contents of this file are subject to the Mozilla Public License Version +1.1 (the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at +http://www.mozilla.org/MPL/ + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the +License. + +The Original Code is WOFF font packaging code. + +The Initial Developer of the Original Code is Mozilla Corporation. +Portions created by the Initial Developer are Copyright (C) 2009 +the Initial Developer. All Rights Reserved. diff --git a/LICENSE-ZOPFLI b/LICENSE-ZOPFLI new file mode 100644 index 0000000..6bff59a --- /dev/null +++ b/LICENSE-ZOPFLI @@ -0,0 +1,13 @@ +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. -- cgit v1.2.3 From b4611511e882f6064131f65732b5ba85b04f5ea3 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 14:23:14 -0400 Subject: Update the Zopfli license file (LICENSE-ZOPFLI) to contain the complete Apache 2.0 license text from https://raw.githubusercontent.com/google/zopfli/zopfli-1.0.3/COPYING rather than a brief summary. --- LICENSE-ZOPFLI | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 198 insertions(+), 10 deletions(-) diff --git a/LICENSE-ZOPFLI b/LICENSE-ZOPFLI index 6bff59a..2e64530 100644 --- a/LICENSE-ZOPFLI +++ b/LICENSE-ZOPFLI @@ -1,13 +1,201 @@ -Copyright 2011 Google Inc. All Rights Reserved. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2011 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -- cgit v1.2.3 From 073dd2a2d618bee39e387e8f23ecbecb382adfcd Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 14:28:13 -0400 Subject: Update the WOFF license file with the full text from upstream, matching the license comment blocks in the source files, from https://raw.githubusercontent.com/samboy/WOFF/2017-06-11/LICENSE. --- LICENSE-WOFF | 52 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/LICENSE-WOFF b/LICENSE-WOFF index 722240a..af86692 100644 --- a/LICENSE-WOFF +++ b/LICENSE-WOFF @@ -1,17 +1,35 @@ -Version: MPL 1.1/GPL 2.0/LGPL 2.1 - -The contents of this file are subject to the Mozilla Public License Version -1.1 (the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at -http://www.mozilla.org/MPL/ - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the -License. - -The Original Code is WOFF font packaging code. - -The Initial Developer of the Original Code is Mozilla Corporation. -Portions created by the Initial Developer are Copyright (C) 2009 -the Initial Developer. All Rights Reserved. + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is WOFF font packaging code. + * + * The Initial Developer of the Original Code is Mozilla Corporation. + * Portions created by the Initial Developer are Copyright (C) 2009 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Jonathan Kew + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** -- cgit v1.2.3 From 7e08f1c944142c8e37050d9e02d91ec326d60ba5 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 14:30:18 -0400 Subject: Add full text of the three WOFF licenses: - LICENSE-WOFF-MPL, from https://www.mozilla.org/media/MPL/1.1/index.0c5913925d40.txt - LICENSE-WOFF-GPL, from https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt - LICENSE-WOFF-LGPL, from https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt --- LICENSE-WOFF-GPL | 339 ++++++++++++++++++++++++++++++++++++ LICENSE-WOFF-LGPL | 502 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE-WOFF-MPL | 469 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1310 insertions(+) create mode 100644 LICENSE-WOFF-GPL create mode 100644 LICENSE-WOFF-LGPL create mode 100644 LICENSE-WOFF-MPL diff --git a/LICENSE-WOFF-GPL b/LICENSE-WOFF-GPL new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/LICENSE-WOFF-GPL @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/LICENSE-WOFF-LGPL b/LICENSE-WOFF-LGPL new file mode 100644 index 0000000..4362b49 --- /dev/null +++ b/LICENSE-WOFF-LGPL @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/LICENSE-WOFF-MPL b/LICENSE-WOFF-MPL new file mode 100644 index 0000000..5669081 --- /dev/null +++ b/LICENSE-WOFF-MPL @@ -0,0 +1,469 @@ + MOZILLA PUBLIC LICENSE + Version 1.1 + + --------------- + +1. Definitions. + + 1.0.1. "Commercial Use" means distribution or otherwise making the + Covered Code available to a third party. + + 1.1. "Contributor" means each entity that creates or contributes to + the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Code, prior Modifications used by a Contributor, and the Modifications + made by that particular Contributor. + + 1.3. "Covered Code" means the Original Code or Modifications or the + combination of the Original Code and Modifications, in each case + including portions thereof. + + 1.4. "Electronic Distribution Mechanism" means a mechanism generally + accepted in the software development community for the electronic + transfer of data. + + 1.5. "Executable" means Covered Code in any form other than Source + Code. + + 1.6. "Initial Developer" means the individual or entity identified + as the Initial Developer in the Source Code notice required by Exhibit + A. + + 1.7. "Larger Work" means a work which combines Covered Code or + portions thereof with code not governed by the terms of this License. + + 1.8. "License" means this document. + + 1.8.1. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means any addition to or deletion from the + substance or structure of either the Original Code or any previous + Modifications. When Covered Code is released as a series of files, a + Modification is: + A. Any addition to or deletion from the contents of a file + containing Original Code or previous Modifications. + + B. Any new file that contains any part of the Original Code or + previous Modifications. + + 1.10. "Original Code" means Source Code of computer software code + which is described in the Source Code notice required by Exhibit A as + Original Code, and which, at the time of its release under this + License is not already Covered Code governed by this License. + + 1.10.1. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.11. "Source Code" means the preferred form of the Covered Code for + making modifications to it, including all modules it contains, plus + any associated interface definition files, scripts used to control + compilation and installation of an Executable, or source code + differential comparisons against either the Original Code or another + well known, available Covered Code of the Contributor's choice. The + Source Code can be in a compressed or archival form, provided the + appropriate decompression or de-archiving software is widely available + for no charge. + + 1.12. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License or a future version of this License issued under Section 6.1. + For legal entities, "You" includes any entity which controls, is + controlled by, or is under common control with You. For purposes of + this definition, "control" means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by + contract or otherwise, or (b) ownership of more than fifty percent + (50%) of the outstanding shares or beneficial ownership of such + entity. + +2. Source Code License. + + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, + non-exclusive license, subject to third party intellectual property + claims: + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Code (or portions thereof) with or without Modifications, and/or + as part of a Larger Work; and + + (b) under Patents Claims infringed by the making, using or + selling of Original Code, to make, have made, use, practice, + sell, and offer for sale, and/or otherwise dispose of the + Original Code (or portions thereof). + + (c) the licenses granted in this Section 2.1(a) and (b) are + effective on the date Initial Developer first distributes + Original Code under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: 1) for code that You delete from the Original Code; 2) + separate from the Original Code; or 3) for infringements caused + by: i) the modification of the Original Code or ii) the + combination of the Original Code with other software or devices. + + 2.2. Contributor Grant. + Subject to third party intellectual property claims, each Contributor + hereby grants You a world-wide, royalty-free, non-exclusive license + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor, to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof) either on an + unmodified basis, with other Modifications, as Covered Code + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or + selling of Modifications made by that Contributor either alone + and/or in combination with its Contributor Version (or portions + of such combination), to make, use, sell, offer for sale, have + made, and/or otherwise dispose of: 1) Modifications made by that + Contributor (or portions thereof); and 2) the combination of + Modifications made by that Contributor with its Contributor + Version (or portions of such combination). + + (c) the licenses granted in Sections 2.2(a) and 2.2(b) are + effective on the date Contributor first makes Commercial Use of + the Covered Code. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: 1) for any code that Contributor has deleted from the + Contributor Version; 2) separate from the Contributor Version; + 3) for infringements caused by: i) third party modifications of + Contributor Version or ii) the combination of Modifications made + by that Contributor with other software (except as part of the + Contributor Version) or other devices; or 4) under Patent Claims + infringed by Covered Code in the absence of Modifications made by + that Contributor. + +3. Distribution Obligations. + + 3.1. Application of License. + The Modifications which You create or to which You contribute are + governed by the terms of this License, including without limitation + Section 2.2. The Source Code version of Covered Code may be + distributed only under the terms of this License or a future version + of this License released under Section 6.1, and You must include a + copy of this License with every copy of the Source Code You + distribute. You may not offer or impose any terms on any Source Code + version that alters or restricts the applicable version of this + License or the recipients' rights hereunder. However, You may include + an additional document offering the additional rights described in + Section 3.5. + + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be + made available in Source Code form under the terms of this License + either on the same media as an Executable version or via an accepted + Electronic Distribution Mechanism to anyone to whom you made an + Executable version available; and if made available via Electronic + Distribution Mechanism, must remain available for at least twelve (12) + months after the date it initially became available, or at least six + (6) months after a subsequent version of that particular Modification + has been made available to such recipients. You are responsible for + ensuring that the Source Code version remains available even if the + Electronic Distribution Mechanism is maintained by a third party. + + 3.3. Description of Modifications. + You must cause all Covered Code to which You contribute to contain a + file documenting the changes You made to create that Covered Code and + the date of any change. You must include a prominent statement that + the Modification is derived, directly or indirectly, from Original + Code provided by the Initial Developer and including the name of the + Initial Developer in (a) the Source Code, and (b) in any notice in an + Executable version or related documentation in which You describe the + origin or ownership of the Covered Code. + + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If Contributor has knowledge that a license under a third party's + intellectual property rights is required to exercise the rights + granted by such Contributor under Sections 2.1 or 2.2, + Contributor must include a text file with the Source Code + distribution titled "LEGAL" which describes the claim and the + party making the claim in sufficient detail that a recipient will + know whom to contact. If Contributor obtains such knowledge after + the Modification is made available as described in Section 3.2, + Contributor shall promptly modify the LEGAL file in all copies + Contributor makes available thereafter and shall take other steps + (such as notifying appropriate mailing lists or newsgroups) + reasonably calculated to inform those who received the Covered + Code that new knowledge has been obtained. + + (b) Contributor APIs. + If Contributor's Modifications include an application programming + interface and Contributor has knowledge of patent licenses which + are reasonably necessary to implement that API, Contributor must + also include this information in the LEGAL file. + + (c) Representations. + Contributor represents that, except as disclosed pursuant to + Section 3.4(a) above, Contributor believes that Contributor's + Modifications are Contributor's original creation(s) and/or + Contributor has sufficient rights to grant the rights conveyed by + this License. + + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source + Code. If it is not possible to put such notice in a particular Source + Code file due to its structure, then You must include such notice in a + location (such as a relevant directory) where a user would be likely + to look for such a notice. If You created one or more Modification(s) + You may add your name as a Contributor to the notice described in + Exhibit A. You must also duplicate this License in any documentation + for the Source Code where You describe recipients' rights or ownership + rights relating to Covered Code. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Code. However, You + may do so only on Your own behalf, and not on behalf of the Initial + Developer or any Contributor. You must make it absolutely clear than + any such warranty, support, indemnity or liability obligation is + offered by You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred by the + Initial Developer or such Contributor as a result of warranty, + support, indemnity or liability terms You offer. + + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the + requirements of Section 3.1-3.5 have been met for that Covered Code, + and if You include a notice stating that the Source Code version of + the Covered Code is available under the terms of this License, + including a description of how and where You have fulfilled the + obligations of Section 3.2. The notice must be conspicuously included + in any notice in an Executable version, related documentation or + collateral in which You describe recipients' rights relating to the + Covered Code. You may distribute the Executable version of Covered + Code or ownership rights under a license of Your choice, which may + contain terms different from this License, provided that You are in + compliance with the terms of this License and that the license for the + Executable version does not attempt to limit or alter the recipient's + rights in the Source Code version from the rights set forth in this + License. If You distribute the Executable version under a different + license You must make it absolutely clear that any terms which differ + from this License are offered by You alone, not by the Initial + Developer or any Contributor. You hereby agree to indemnify the + Initial Developer and every Contributor for any liability incurred by + the Initial Developer or such Contributor as a result of any such + terms You offer. + + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code + not governed by the terms of this License and distribute the Larger + Work as a single product. In such a case, You must make sure the + requirements of this License are fulfilled for the Covered Code. + +4. Inability to Comply Due to Statute or Regulation. + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Code due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description + must be included in the LEGAL file described in Section 3.4 and must + be included with all distributions of the Source Code. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Application of this License. + + This License applies to code to which the Initial Developer has + attached the notice in Exhibit A and to related Covered Code. + +6. Versions of the License. + + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised + and/or new versions of the License from time to time. Each version + will be given a distinguishing version number. + + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the + License, You may always continue to use it under the terms of that + version. You may also choose to use such Covered Code under the terms + of any subsequent version of the License published by Netscape. No one + other than Netscape has the right to modify the terms applicable to + Covered Code created under this License. + + 6.3. Derivative Works. + If You create or use a modified version of this License (which you may + only do in order to apply it to code which is not already Covered Code + governed by this License), You must (a) rename Your license so that + the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", + "MPL", "NPL" or any confusingly similar phrase do not appear in your + license (except to note that your license differs from this License) + and (b) otherwise make it clear that Your version of the license + contains terms which differ from the Mozilla Public License and + Netscape Public License. (Filling in the name of the Initial + Developer, Original Code or Contributor in the notice described in + Exhibit A shall not of themselves be deemed to be modifications of + this License.) + +7. DISCLAIMER OF WARRANTY. + + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, + WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF + DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. + THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE + IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, + YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE + COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER + OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +8. TERMINATION. + + 8.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to cure + such breach within 30 days of becoming aware of the breach. All + sublicenses to the Covered Code which are properly granted shall + survive any termination of this License. Provisions which, by their + nature, must remain in effect beyond the termination of this License + shall survive. + + 8.2. If You initiate litigation by asserting a patent infringement + claim (excluding declatory judgment actions) against Initial Developer + or a Contributor (the Initial Developer or Contributor against whom + You file such action is referred to as "Participant") alleging that: + + (a) such Participant's Contributor Version directly or indirectly + infringes any patent, then any and all rights granted by such + Participant to You under Sections 2.1 and/or 2.2 of this License + shall, upon 60 days notice from Participant terminate prospectively, + unless if within 60 days after receipt of notice You either: (i) + agree in writing to pay Participant a mutually agreeable reasonable + royalty for Your past and future use of Modifications made by such + Participant, or (ii) withdraw Your litigation claim with respect to + the Contributor Version against such Participant. If within 60 days + of notice, a reasonable royalty and payment arrangement are not + mutually agreed upon in writing by the parties or the litigation claim + is not withdrawn, the rights granted by Participant to You under + Sections 2.1 and/or 2.2 automatically terminate at the expiration of + the 60 day notice period specified above. + + (b) any software, hardware, or device, other than such Participant's + Contributor Version, directly or indirectly infringes any patent, then + any rights granted to You by such Participant under Sections 2.1(b) + and 2.2(b) are revoked effective as of the date You first made, used, + sold, distributed, or had made, Modifications made by that + Participant. + + 8.3. If You assert a patent infringement claim against Participant + alleging that such Participant's Contributor Version directly or + indirectly infringes any patent where such claim is resolved (such as + by license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 8.4. In the event of termination under Sections 8.1 or 8.2 above, + all end user license agreements (excluding distributors and resellers) + which have been validly granted by You or any distributor hereunder + prior to termination shall survive termination. + +9. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL + DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, + OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR + ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY + CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, + WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY + RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW + PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE + EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO + THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +10. U.S. GOVERNMENT END USERS. + + The Covered Code is a "commercial item," as that term is defined in + 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" and "commercial computer software documentation," as such + terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 + C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), + all U.S. Government End Users acquire Covered Code with only those + rights set forth herein. + +11. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + California law provisions (except to the extent applicable law, if + any, provides otherwise), excluding its conflict-of-law provisions. + With respect to disputes in which at least one party is a citizen of, + or an entity chartered or registered to do business in the United + States of America, any litigation relating to this License shall be + subject to the jurisdiction of the Federal Courts of the Northern + District of California, with venue lying in Santa Clara County, + California, with the losing party responsible for costs, including + without limitation, court costs and reasonable attorneys' fees and + expenses. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. + Any law or regulation which provides that the language of a contract + shall be construed against the drafter shall not apply to this + License. + +12. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +13. MULTIPLE-LICENSED CODE. + + Initial Developer may designate portions of the Covered Code as + "Multiple-Licensed". "Multiple-Licensed" means that the Initial + Developer permits you to utilize portions of the Covered Code under + Your choice of the MPL or the alternative licenses, if any, specified + by the Initial Developer in the file described in Exhibit A. + +EXHIBIT A -Mozilla Public License. + + ``The contents of this file are subject to the Mozilla Public License + Version 1.1 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + https://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + License for the specific language governing rights and limitations + under the License. + + The Original Code is ______________________________________. + + The Initial Developer of the Original Code is ________________________. + Portions created by ______________________ are Copyright (C) ______ + _______________________. All Rights Reserved. + + Contributor(s): ______________________________________. + + Alternatively, the contents of this file may be used under the terms + of the _____ license (the "[___] License"), in which case the + provisions of [______] License are applicable instead of those + above. If you wish to allow use of your version of this file only + under the terms of the [____] License and not to allow others to use + your version of this file under the MPL, indicate your decision by + deleting the provisions above and replace them with the notice and + other provisions required by the [___] License. If you do not delete + the provisions above, a recipient may use your version of this file + under either the MPL or the [___] License." + + [NOTE: The text of this Exhibit A may differ slightly from the text of + the notices in the Source Code files of the Original Code. You should + use the text of this Exhibit A rather than the text found in the + Original Code Source Code for Your Modifications.] -- cgit v1.2.3 From 3e6cf7a7b29698b69f96e848f84196aeda57b4a4 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 14:35:42 -0400 Subject: Add support for external CPPFLAGS/CFLAGS to the Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2f644f2..173394b 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ FILES=zopfli/blocksplitter.c zopfli/cache.c zopfli/deflate.c zopfli/gzip_contain all: sfnt2woff-zopfli woff2sfnt-zopfli sfnt2woff-zopfli: sfnt2woff.c $(FILES) Makefile - $(CC) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm + $(CC) $(CPPFLAGS) $(CFLAGS) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm woff2sfnt-zopfli: woff2sfnt.c $(FILES) Makefile - $(CC) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm + $(CC) $(CPPFLAGS) $(CFLAGS) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm clean: $(RM) -r *.o *.dSYM sfnt2woff-zopfli woff2sfnt-zopfli *.gch *.out -- cgit v1.2.3 From 9ed2430baaf3558d0844c8bc74fa904007133db7 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 14:43:05 -0400 Subject: Use only Zopfli public API functions from zopfli.h. The Zopfli library is normally installed only with zopfli.h (and zopflipng_lib.h). Using only APIs in this header makes it easier to use an external Zopfli library. This means including zopfli.h instead of zlib_container.h, and calling ZopfliCompress with ZOPFLI_FORMAT_ZLIB instead of calling ZopfliZlibCompress directly. --- woff.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/woff.c b/woff.c index c0f1192..dc77c42 100644 --- a/woff.c +++ b/woff.c @@ -42,7 +42,7 @@ #include #include #include -#include "zlib_container.h" +#include "zopfli.h" #ifdef WOFF_MOZILLA_CLIENT /* define this when building as part of Gecko */ # include "prmem.h" @@ -288,7 +288,7 @@ woffEncode(const uint8_t * sfntData, uint32_t sfntLen, FAIL(eWOFF_invalid); } - ZopfliZlibCompress(&options, (const uint8_t *) (sfntData + sourceOffset), sourceLen, &dest, &destLen); + ZopfliCompress(&options, ZOPFLI_FORMAT_ZLIB, (const uint8_t *) (sfntData + sourceOffset), sourceLen, &dest, &destLen); if (destLen < sourceLen) { /* compressed table was smaller */ @@ -548,7 +548,7 @@ woffSetMetadata(const uint8_t * woffData, uint32_t * woffLen, ZopfliOptions options; ZopfliInitOptions(&options); - ZopfliZlibCompress(&options, (const uint8_t *) metaData, metaLen, &compData, &compLen); + ZopfliCompress(&options, ZOPFLI_FORMAT_ZLIB, (const uint8_t *) metaData, metaLen, &compData, &compLen); } woffData = rebuildWoff(woffData, woffLen, -- cgit v1.2.3 From 6323a64c051817854a7c992e0259cc0e2bc5fb38 Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 18:02:55 -0400 Subject: Add ZOPFLI_LIBS and ZOPFLI_CFLAGS variables to the Makefile to make it easy to build with an external Zopfli library. Set their default values such that the bundled Zopfli library is still used by default. Re-order the linker flags so that both cases work as expected. Fix linker flag order --- Makefile | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 173394b..c377015 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,35 @@ -FILES=zopfli/blocksplitter.c zopfli/cache.c zopfli/deflate.c zopfli/gzip_container.c zopfli/hash.c zopfli/katajainen.c zopfli/lz77.c zopfli/squeeze.c zopfli/tree.c zopfli/util.c zopfli/zlib_container.c zopfli/zopfli_lib.c woff.c +# When using the bundled Zopfli library, this is a list of sources to compile. +# To use an external Zopfli library, override it with the path to a static +# library, or with something like -lzopfli. +ZOPFLI_LIBS=zopfli/blocksplitter.c \ + zopfli/cache.c \ + zopfli/deflate.c \ + zopfli/gzip_container.c \ + zopfli/hash.c \ + zopfli/katajainen.c \ + zopfli/lz77.c \ + zopfli/squeeze.c \ + zopfli/tree.c \ + zopfli/util.c \ + zopfli/zlib_container.c \ + zopfli/zopfli_lib.c +# When not using the bundled Zopfli library, this should normally be overridden +# with the empty string to avoid finding bundled Zopfli headers. +ZOPFLI_CFLAGS=-Izopfli + +FILES=woff.c all: sfnt2woff-zopfli woff2sfnt-zopfli sfnt2woff-zopfli: sfnt2woff.c $(FILES) Makefile - $(CC) $(CPPFLAGS) $(CFLAGS) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm + $(CC) $(CPPFLAGS) $(CFLAGS) $(ZOPFLI_CFLAGS) \ + $(FILES) $< -o $@ \ + $(LDFLAGS) $(ZOPFLI_LIBS) -lz -lm woff2sfnt-zopfli: woff2sfnt.c $(FILES) Makefile - $(CC) $(CPPFLAGS) $(CFLAGS) -Izopfli $(LDFLAGS) $(FILES) $< -o $@ -lz -lm + $(CC) $(CPPFLAGS) $(CFLAGS) $(ZOPFLI_CFLAGS) \ + $(FILES) $< -o $@ \ + $(LDFLAGS) $(ZOPFLI_LIBS) -lz -lm clean: $(RM) -r *.o *.dSYM sfnt2woff-zopfli woff2sfnt-zopfli *.gch *.out -- cgit v1.2.3 From 6b63776f3d03f8b88d23cedb6fe2a9bd2947d93d Mon Sep 17 00:00:00 2001 From: Benjamin Beasley Date: Tue, 20 Oct 2020 18:05:40 -0400 Subject: Add ZLIB_LIBS and ZLIB_CFLAGS variables to the Makefile to make it easy to control linking against zlib, just as we added ZOPFLI_LIBS and ZOPFLI_CFLAGS for the Zopfli library. --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c377015..4fd80b1 100644 --- a/Makefile +++ b/Makefile @@ -17,19 +17,22 @@ ZOPFLI_LIBS=zopfli/blocksplitter.c \ # with the empty string to avoid finding bundled Zopfli headers. ZOPFLI_CFLAGS=-Izopfli +ZLIB_LIBS=-lz +ZLIB_CFLAGS= + FILES=woff.c all: sfnt2woff-zopfli woff2sfnt-zopfli sfnt2woff-zopfli: sfnt2woff.c $(FILES) Makefile - $(CC) $(CPPFLAGS) $(CFLAGS) $(ZOPFLI_CFLAGS) \ + $(CC) $(CPPFLAGS) $(CFLAGS) $(ZOPFLI_CFLAGS) $(ZLIB_CFLAGS) \ $(FILES) $< -o $@ \ - $(LDFLAGS) $(ZOPFLI_LIBS) -lz -lm + $(LDFLAGS) $(ZOPFLI_LIBS) $(ZLIB_LIBS) -lm woff2sfnt-zopfli: woff2sfnt.c $(FILES) Makefile - $(CC) $(CPPFLAGS) $(CFLAGS) $(ZOPFLI_CFLAGS) \ + $(CC) $(CPPFLAGS) $(CFLAGS) $(ZOPFLI_CFLAGS) $(ZLIB_CFLAGS) \ $(FILES) $< -o $@ \ - $(LDFLAGS) $(ZOPFLI_LIBS) -lz -lm + $(LDFLAGS) $(ZOPFLI_LIBS) $(ZLIB_LIBS) -lm clean: $(RM) -r *.o *.dSYM sfnt2woff-zopfli woff2sfnt-zopfli *.gch *.out -- cgit v1.2.3