mirror of
https://github.com/reactos/reactos.git
synced 2024-08-20 18:35:32 +00:00
1252 lines
37 KiB
C
1252 lines
37 KiB
C
/*
|
|
File lzx_layer.c, part of lzxcomp library
|
|
Copyright (C) 2002 Matthew T. Russotto
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU Lesser General Public License as published by
|
|
the Free Software Foundation; version 2.1 only
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
*/
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <stdint.h>
|
|
#include <string.h> /* for memset on Linux */
|
|
#include <assert.h>
|
|
#include <math.h>
|
|
#include "lzx_config.h"
|
|
#ifdef NONSLIDE
|
|
#include "lz_nonslide.h"
|
|
#else
|
|
#include "hash_slide.h"
|
|
#include "lz_slide.h"
|
|
#endif
|
|
#include "lzx_compress.h"
|
|
#include "lzx_constants.h"
|
|
|
|
/* Debugging defines useful during development. All add diagnostic output
|
|
at various points in the system */
|
|
|
|
/*#define DEBUG_MATCHES *//* When matches come in from the LZ engine */
|
|
/*#define DEBUG_MATCHES_2 *//* When matches are being output */
|
|
/*#define DEBUG_HUFFMAN *//* When huffman trees are built */
|
|
/*#define DEBUG_ENTROPY *//* In entropy calculation */
|
|
/*#define DEBUG_LZ *//* Uncompressed input reconstructed from
|
|
LZ engine */
|
|
/*#define DEBUG_BITBUF *//* Raw output to upper layer */
|
|
/*#define DEBUG_EXTRA_BITS *//* Savings due to extra bits huffman tree */
|
|
/*#define DEBUG_POSITION_SLOT_LOOKUP */
|
|
/*#define DEBUG_TREE_COMPRESSION *//* During RLE compression of trees */
|
|
|
|
/* number of position slots given window_size-5 */
|
|
/* as corrected by Caie */
|
|
short num_position_slots[] = {30, 32, 34, 36, 38, 42, 50};
|
|
unsigned long position_base[51];
|
|
u_char extra_bits[52];
|
|
double rloge2;
|
|
|
|
typedef struct ih_elem {
|
|
int freq;
|
|
short sym;
|
|
short pathlength;
|
|
struct ih_elem *parent;
|
|
struct ih_elem *left;
|
|
struct ih_elem *right;
|
|
} ih_elem;
|
|
|
|
typedef struct h_elem {
|
|
int freq;
|
|
short sym;
|
|
short pathlength;
|
|
struct ih_elem *parent;
|
|
unsigned short code;
|
|
} h_elem;
|
|
|
|
typedef struct huff_entry {
|
|
short codelength;
|
|
unsigned short code;
|
|
} huff_entry;
|
|
|
|
static int cmp_leaves(const void *in_a, const void *in_b)
|
|
{
|
|
const struct h_elem *a = in_a;
|
|
const struct h_elem *b = in_b;
|
|
|
|
if (!a->freq && b->freq)
|
|
return 1;
|
|
if (a->freq && !b->freq)
|
|
return -1;
|
|
|
|
if (a->freq == b->freq)
|
|
return a->sym - b->sym;
|
|
|
|
return a->freq - b->freq;
|
|
}
|
|
|
|
static int
|
|
cmp_pathlengths(const void *in_a, const void *in_b)
|
|
{
|
|
const struct h_elem *a = in_a;
|
|
const struct h_elem *b = in_b;
|
|
|
|
if (a->pathlength == b->pathlength)
|
|
#if 0
|
|
return a->sym - b->sym;
|
|
#else
|
|
/* see note on canonical pathlengths */
|
|
return b->sym - a->sym;
|
|
#endif
|
|
return b->pathlength - a->pathlength;
|
|
}
|
|
|
|
/* standard huffman building algorithm */
|
|
static void
|
|
build_huffman_tree(int nelem, int max_code_length, int *freq, huff_entry *tree)
|
|
{
|
|
h_elem *leaves = malloc(nelem * sizeof(h_elem));
|
|
ih_elem *inodes;
|
|
ih_elem *next_inode;
|
|
ih_elem *cur_inode;
|
|
h_elem *cur_leaf;
|
|
int leaves_left;
|
|
int nleaves;
|
|
int pathlength;
|
|
unsigned short cur_code;
|
|
short codes_too_long = 0;
|
|
ih_elem *f1, *f2;
|
|
int i;
|
|
|
|
for (i = 0; i < nelem; i++) {
|
|
leaves[i].freq = freq[i];
|
|
leaves[i].sym = i;
|
|
leaves[i].pathlength = 0;
|
|
}
|
|
qsort(leaves, nelem, sizeof(h_elem), cmp_leaves);
|
|
for (leaves_left = 0; leaves_left < nelem; leaves_left++) {
|
|
#ifdef DEBUG_HUFFMAN
|
|
fprintf(stderr, "%3d: %3d '%c'\n", leaves_left, leaves[leaves_left].freq,
|
|
leaves[leaves_left].sym);
|
|
#endif
|
|
if (!leaves[leaves_left].freq) break;
|
|
}
|
|
nleaves = leaves_left;
|
|
|
|
if (nleaves >= 2) {
|
|
inodes = malloc((nelem-1) * sizeof(ih_elem));
|
|
do {
|
|
if (codes_too_long) {
|
|
for (leaves_left = 0; leaves_left < nelem; leaves_left++) {
|
|
if (!leaves[leaves_left].freq) break;
|
|
if (leaves[leaves_left].freq != 1) {
|
|
leaves[leaves_left].freq >>= 1;
|
|
codes_too_long = 0;
|
|
}
|
|
}
|
|
assert (!codes_too_long);
|
|
}
|
|
|
|
cur_leaf = leaves;
|
|
next_inode = cur_inode = inodes;
|
|
|
|
do {
|
|
f1 = f2 = NULL;
|
|
if (leaves_left &&
|
|
((cur_inode == next_inode) ||
|
|
(cur_leaf->freq <= cur_inode->freq))) {
|
|
f1 = (ih_elem *)cur_leaf++;
|
|
leaves_left--;
|
|
}
|
|
else if (cur_inode != next_inode) {
|
|
f1 = cur_inode++;
|
|
}
|
|
|
|
if (leaves_left &&
|
|
((cur_inode == next_inode) ||
|
|
(cur_leaf->freq <= cur_inode->freq))) {
|
|
f2 = (ih_elem *)cur_leaf++;
|
|
leaves_left--;
|
|
}
|
|
else if (cur_inode != next_inode) {
|
|
f2 = cur_inode++;
|
|
}
|
|
|
|
#ifdef DEBUG_HUFFMAN
|
|
fprintf(stderr, "%d %d\n", f1, f2);
|
|
#endif
|
|
if (f1 && f2) {
|
|
next_inode->freq = f1->freq + f2->freq;
|
|
next_inode->sym = -1;
|
|
next_inode->left = f1;
|
|
next_inode->right = f2;
|
|
next_inode->parent = NULL;
|
|
f1->parent = next_inode;
|
|
f2->parent = next_inode;
|
|
if (f1->pathlength > f2->pathlength)
|
|
next_inode->pathlength = f1->pathlength + 1;
|
|
else
|
|
next_inode->pathlength = f2->pathlength + 1;
|
|
if (next_inode->pathlength > max_code_length) {
|
|
codes_too_long = 1;
|
|
break;
|
|
}
|
|
next_inode++;
|
|
}
|
|
}
|
|
while (f1 && f2);
|
|
}
|
|
while (codes_too_long);
|
|
|
|
#ifdef DEBUG_HUFFMAN
|
|
cur_inode = inodes;
|
|
while (cur_inode < next_inode) {
|
|
fprintf(stderr, "%d l: %3d%c r: %3d%c freq: %8d\n",
|
|
cur_inode - inodes,
|
|
(cur_inode->left->sym!=-1)?(((struct h_elem *)cur_inode->left)-leaves):(cur_inode->left-inodes),
|
|
(cur_inode->left->sym!=-1)?'l':'i',
|
|
(cur_inode->right->sym!=-1)?(((struct h_elem *)cur_inode->right)-leaves):(cur_inode->right-inodes),
|
|
(cur_inode->right->sym!=-1)?'l':'i',
|
|
(cur_inode->freq)
|
|
);
|
|
cur_inode++;
|
|
}
|
|
#endif
|
|
|
|
/* now traverse tree depth-first */
|
|
cur_inode = next_inode - 1;
|
|
pathlength = 0;
|
|
cur_inode->pathlength = -1;
|
|
do {
|
|
/* precondition: at unmarked node*/
|
|
if (cur_inode->sym == -1) /*&& (cur_inode->left)*/ {
|
|
/* left node of unmarked node is unmarked */
|
|
cur_inode = cur_inode->left;
|
|
cur_inode->pathlength = -1;
|
|
pathlength++;
|
|
}
|
|
else {
|
|
/* mark node */
|
|
cur_inode->pathlength = pathlength;
|
|
#if 0
|
|
if (cur_inode->right) {
|
|
/* right node of previously unmarked node is unmarked */
|
|
cur_inode = cur_inode->right;
|
|
cur_inode->pathlength = -1;
|
|
pathlength++;
|
|
}
|
|
else
|
|
#endif
|
|
{
|
|
|
|
/* time to come up. Keep coming up until an unmarked node is reached */
|
|
/* or the tree is exhausted */
|
|
do {
|
|
cur_inode = cur_inode->parent;
|
|
pathlength--;
|
|
}
|
|
while (cur_inode && (cur_inode->pathlength != -1));
|
|
if (cur_inode) {
|
|
/* found unmarked node; mark it and go right */
|
|
cur_inode->pathlength = pathlength;
|
|
cur_inode = cur_inode->right;
|
|
cur_inode->pathlength = -1;
|
|
pathlength++;
|
|
/* would be complex if cur_inode could be null here. It can't */
|
|
}
|
|
}
|
|
}
|
|
}
|
|
while (cur_inode);
|
|
|
|
#ifdef DEBUG_HUFFMAN
|
|
cur_inode = inodes;
|
|
while (cur_inode < next_inode) {
|
|
fprintf(stderr, "%d l: %3d%c r: %3d%c freq: %8d pathlength %4d\n",
|
|
cur_inode - inodes,
|
|
(cur_inode->left->sym!=-1)?(((struct h_elem *)cur_inode->left)-leaves):(cur_inode->left-inodes),
|
|
(cur_inode->left->sym!=-1)?'l':'i',
|
|
(cur_inode->right->sym!=-1)?(((struct h_elem *)cur_inode->right)-leaves):(cur_inode->right-inodes),
|
|
(cur_inode->right->sym!=-1)?'l':'i',
|
|
(cur_inode->freq),
|
|
(cur_inode->pathlength)
|
|
);
|
|
cur_inode++;
|
|
}
|
|
#endif
|
|
free(inodes);
|
|
|
|
/* the pathlengths are already in order, so this sorts by symbol */
|
|
qsort(leaves, nelem, sizeof(h_elem), cmp_pathlengths);
|
|
|
|
/**
|
|
Microsoft's second condition on its canonical huffman codes is:
|
|
|
|
For each level, starting at the deepest level of the tree and then
|
|
moving upwards, leaf nodes must start as far left as possible. An
|
|
alternative way of stating this constraint is that if any tree node
|
|
has children then all tree nodes to the left of it with the same path
|
|
length must also have children.
|
|
|
|
These 'alternatives' are not equivalent. The latter alternative gives
|
|
the common canonical code where the longest code is all zeros. The former
|
|
gives an opposite code where the longest code is all ones. Microsoft uses the
|
|
former alternative.
|
|
**/
|
|
|
|
#if 0
|
|
pathlength = leaves[0].pathlength;
|
|
cur_code = 0;
|
|
for (i = 0; i < nleaves; i++) {
|
|
while (leaves[i].pathlength < pathlength) {
|
|
assert(!(cur_code & 1));
|
|
cur_code >>= 1;
|
|
pathlength--;
|
|
}
|
|
leaves[i].code = cur_code;
|
|
cur_code++;
|
|
}
|
|
#else
|
|
pathlength = leaves[nleaves-1].pathlength;
|
|
assert(leaves[0].pathlength <= 16); /* this method cannot deal with bigger codes, though
|
|
the other canonical method can in some cases
|
|
(because it starts with zeros ) */
|
|
cur_code = 0;
|
|
for (i = nleaves - 1; i >= 0; i--) {
|
|
while (leaves[i].pathlength > pathlength) {
|
|
cur_code <<= 1;
|
|
pathlength++;
|
|
}
|
|
leaves[i].code = cur_code;
|
|
cur_code++;
|
|
}
|
|
#endif
|
|
|
|
#ifdef DEBUG_HUFFMAN
|
|
for (i = 0; i < nleaves; i++) {
|
|
char code[18];
|
|
int j;
|
|
|
|
cur_code = leaves[i].code;
|
|
code[leaves[i].pathlength] = 0;
|
|
for (j = leaves[i].pathlength-1; j >= 0; j--) {
|
|
if (cur_code & 1) code[j] = '1';
|
|
else code[j] = '0';
|
|
cur_code >>= 1;
|
|
}
|
|
fprintf(stderr, "%3d: %3d %3d %-16.16s '%c'\n", i, leaves[i].freq, leaves[i].pathlength, code,
|
|
leaves[i].sym);
|
|
}
|
|
#endif
|
|
}
|
|
else if (nleaves == 1) {
|
|
/* 0 symbols is OK (not according to doc, but according to Caie) */
|
|
/* but if only one symbol is present, two symbols are required */
|
|
nleaves = 2;
|
|
leaves[0].pathlength = leaves[1].pathlength = 1;
|
|
if (leaves[1].sym > leaves[0].sym) {
|
|
leaves[1].code = 1;
|
|
leaves[0].code = 0;
|
|
}
|
|
else {
|
|
leaves[0].code = 1;
|
|
leaves[1].code = 0;
|
|
}
|
|
}
|
|
|
|
memset(tree, 0, nelem * sizeof(huff_entry));
|
|
for (i = 0; i < nleaves; i++) {
|
|
tree[leaves[i].sym].codelength = leaves[i].pathlength;
|
|
tree[leaves[i].sym].code = leaves[i].code;
|
|
}
|
|
|
|
free(leaves);
|
|
}
|
|
|
|
/* from Stuart Caie's code -- I'm hoping this code is too small to encumber
|
|
this file. If not, you could rip it out and hard-code the tables */
|
|
|
|
static void lzx_init_static(void)
|
|
{
|
|
int i, j;
|
|
|
|
if (extra_bits[49]) return;
|
|
|
|
rloge2 = 1.0/log(2);
|
|
for (i=0, j=0; i <= 50; i += 2) {
|
|
extra_bits[i] = extra_bits[i+1] = j; /* 0,0,0,0,1,1,2,2,3,3... */
|
|
if ((i != 0) && (j < 17)) j++; /* 0,0,1,2,3,4...15,16,17,17,17,17... */
|
|
}
|
|
|
|
for (i=0, j=0; i <= 50; i++) {
|
|
position_base[i] = j; /* 0,1,2,3,4,6,8,12,16,24,32,... */
|
|
j += 1 << extra_bits[i]; /* 1,1,1,1,2,2,4,4,8,8,16,16,32,32,... */
|
|
}
|
|
}
|
|
|
|
struct lzx_data
|
|
{
|
|
void *in_arg;
|
|
void *out_arg;
|
|
void *mark_frame_arg;
|
|
lzx_get_bytes_t get_bytes;
|
|
lzx_at_eof_t at_eof;
|
|
lzx_put_bytes_t put_bytes;
|
|
lzx_mark_frame_t mark_frame;
|
|
struct lz_info *lzi;
|
|
/* a 'frame' is an 0x8000 byte thing. Called that because otherwise
|
|
I'd confuse myself overloading 'block' */
|
|
int left_in_frame;
|
|
int left_in_block;
|
|
int R0, R1, R2;
|
|
int num_position_slots;
|
|
/* this is the LZX block size */
|
|
int block_size;
|
|
int *main_freq_table;
|
|
int length_freq_table[NUM_SECONDARY_LENGTHS];
|
|
int aligned_freq_table[LZX_ALIGNED_SIZE];
|
|
uint32_t *block_codes;
|
|
uint32_t *block_codesp;
|
|
huff_entry *main_tree;
|
|
huff_entry length_tree[NUM_SECONDARY_LENGTHS];
|
|
huff_entry aligned_tree[LZX_ALIGNED_SIZE];
|
|
int main_tree_size;
|
|
uint16_t bit_buf;
|
|
int bits_in_buf;
|
|
double main_entropy;
|
|
double last_ratio;
|
|
uint8_t *prev_main_treelengths;
|
|
uint8_t prev_length_treelengths[NUM_SECONDARY_LENGTHS];
|
|
uint32_t len_uncompressed_input;
|
|
uint32_t len_compressed_output;
|
|
short need_1bit_header;
|
|
short subdivide; /* 0 = don't subdivide, 1 = allowed, -1 = requested */
|
|
};
|
|
|
|
static int
|
|
lzx_get_chars(lz_info *lzi, int n, u_char *buf)
|
|
{
|
|
/* force lz compression to stop after every block */
|
|
int chars_read;
|
|
int chars_pad;
|
|
|
|
lzx_data *lzud = (lzx_data *)lzi->user_data;
|
|
#ifdef OLDFRAMING
|
|
if (lzud->subdivide < 0) return 0;
|
|
if (n > lzud->left_in_frame)
|
|
n = lzud->left_in_frame;
|
|
if (n > lzud->left_in_block)
|
|
n = lzud->left_in_block;
|
|
#endif
|
|
chars_read = lzud->get_bytes(lzud->in_arg, n, buf);
|
|
#ifdef OLDFRAMING
|
|
lzud->left_in_frame -= chars_read;
|
|
lzud->left_in_block -= chars_read;
|
|
#else
|
|
lzud->left_in_frame -= chars_read % LZX_FRAME_SIZE;
|
|
if (lzud->left_in_frame < 0)
|
|
lzud->left_in_frame += LZX_FRAME_SIZE;
|
|
#endif
|
|
if ((chars_read < n) && (lzud->left_in_frame)) {
|
|
chars_pad = n - chars_read;
|
|
if (chars_pad > lzud->left_in_frame) chars_pad = lzud->left_in_frame;
|
|
/* never emit a full frame of padding. This prevents silliness when
|
|
lzx_compress is called when at EOF but EOF not yet detected */
|
|
if (chars_pad == LZX_FRAME_SIZE) chars_pad = 0;
|
|
#ifdef OLDFRAMING
|
|
if (chars_pad > lzud->left_in_block) chars_pad = lzud->left_in_block;
|
|
#endif
|
|
memset(buf + chars_read, 0, chars_pad);
|
|
lzud->left_in_frame -= chars_pad;
|
|
#ifdef OLDFRAMING
|
|
lzud->left_in_block -= chars_pad;
|
|
#endif
|
|
chars_read += chars_pad;
|
|
}
|
|
return chars_read;
|
|
}
|
|
|
|
#ifdef NONSLIDE
|
|
static int find_match_at(lz_info *lzi, int loc, int match_len, int *match_locp)
|
|
{
|
|
u_char *matchb;
|
|
u_char *nmatchb;
|
|
u_char *c1, *c2;
|
|
int j;
|
|
|
|
if (-*match_locp == loc) return -1;
|
|
if (loc < match_len) return -1;
|
|
|
|
matchb = lzi->block_buf + lzi->block_loc + *match_locp;
|
|
nmatchb = lzi->block_buf + lzi->block_loc - loc;
|
|
c1 = matchb;
|
|
c2 = nmatchb;
|
|
for (j = 0; j < match_len; j++) {
|
|
if (*c1++ != *c2++) break;
|
|
}
|
|
if (j == match_len) {
|
|
#ifdef DEBUG_MATCHES
|
|
fprintf(stderr, "match found %d, old = %d new = %d len = %d\n", lzi->cur_loc, -*match_locp, loc, match_len);
|
|
#endif
|
|
*match_locp = -loc;
|
|
return 0;
|
|
}
|
|
return -1;
|
|
}
|
|
#else
|
|
static int find_match_at(lz_info *lzi, int loc, int match_len, int *match_locp)
|
|
{
|
|
u_char *matchb;
|
|
u_char *nmatchb;
|
|
u_char *c1, *c2;
|
|
int j;
|
|
|
|
if (-*match_locp == loc) return -1;
|
|
if (loc < match_len) return -1;
|
|
|
|
matchb = lzi->slide_bufp + *match_locp;
|
|
if (matchb < lzi->slide_buf) matchb += lzi->slide_buf_size;
|
|
nmatchb = lzi->slide_bufp - loc;
|
|
if (nmatchb < lzi->slide_buf) nmatchb += lzi->slide_buf_size;
|
|
c1 = matchb;
|
|
c2 = nmatchb;
|
|
for (j = 0; j < match_len; j++) {
|
|
if (*c1++ != *c2++) break;
|
|
if (c1 == lzi->slide_bufe) c1 = lzi->slide_buf;
|
|
if (c2 == lzi->slide_bufe) c2 = lzi->slide_buf;
|
|
}
|
|
if (j == match_len) {
|
|
#ifdef DEBUG_MATCHES
|
|
fprintf(stderr, "match found %d, old = %d new = %d len = %d\n", lzi->cur_loc, -*match_locp, loc, match_len);
|
|
#endif
|
|
*match_locp = -loc;
|
|
return 0;
|
|
}
|
|
return -1;
|
|
}
|
|
#endif
|
|
static void check_entropy(lzx_data *lzud, int main_index)
|
|
{
|
|
/* entropy = - sum_alphabet P(x) * log2 P(x) */
|
|
/* entropy = - sum_alphabet f(x)/N * log2 (f(x)/N) */
|
|
/* entropy = - 1/N sum_alphabet f(x) * (log2 f(x) - log2 N) */
|
|
/* entropy = - 1/N (sum_alphabet f(x) * log2 f(x)) - sum_alphabet f(x) log2 N */
|
|
/* entropy = - 1/N (sum_alphabet f(x) * log2 f(x)) - log2 N sum_alphabet f(x) */
|
|
/* entropy = - 1/N (sum_alphabet f(x) * log2 f(x)) - N * log2 N */
|
|
|
|
/* entropy = - 1/N ((sum_alphabet f(x) * log2 f(x) ) - N * log2 N) */
|
|
/* entropy = - 1/N ((sum_alphabet f(x) * ln f(x) * 1/ln 2) - N * ln N * 1/ln 2) */
|
|
/* entropy = 1/(N ln 2) (N * ln N - (sum_alphabet f(x) * ln f(x))) */
|
|
/* entropy = 1/(N ln 2) (N * ln N + (sum_alphabet -f(x) * ln f(x))) */
|
|
|
|
/* entropy = 1/(N ln 2) ( sum_alphabet ln N * f(x) + (sum_alphabet -f(x) * ln f(x))) */
|
|
/* entropy = 1/(N ln 2) ( sum_alphabet ln N * f(x) + (-f(x) * ln f(x))) */
|
|
/* entropy = -1/(N ln 2) ( sum_alphabet -ln N * f(x) + (f(x) * ln f(x))) */
|
|
/* entropy = -1/(N ln 2) ( sum_alphabet f(x)(- ln N + ln f(x))) */
|
|
/* entropy = -1/(N ln 2) ( sum_alphabet f(x)(ln f(x)/N)) */
|
|
/* entropy = -1/N ( sum_alphabet (1/(ln 2))f(x)(ln f(x)/N)) */
|
|
/* entropy = -1/N ( sum_alphabet f(x)(log2 f(x)/N)) */
|
|
/* entropy = - ( sum_alphabet f(x)/N(log2 f(x)/N)) */
|
|
/* entropy = - ( sum_alphabet P(x)(log2 P(x))) */
|
|
|
|
|
|
double freq;
|
|
double n_ln_n;
|
|
double rn_ln2;
|
|
double cur_ratio;
|
|
int n;
|
|
|
|
/* delete old entropy accumulation */
|
|
if (lzud->main_freq_table[main_index] != 1) {
|
|
freq = (double)lzud->main_freq_table[main_index]-1;
|
|
lzud->main_entropy += freq * log(freq);
|
|
}
|
|
/* add new entropy accumulation */
|
|
freq = (double)lzud->main_freq_table[main_index];
|
|
lzud->main_entropy -= freq * log(freq);
|
|
n = lzud->block_codesp - lzud->block_codes;
|
|
|
|
if (((n & 0xFFF) == 0) && (lzud->left_in_block >= 0x1000)) {
|
|
n_ln_n = (double)n * log((double)n);
|
|
rn_ln2 = rloge2 / (double)n;
|
|
cur_ratio = (n * rn_ln2 *(n_ln_n + lzud->main_entropy) + 24 + 3 * 80 + NUM_CHARS + (lzud->main_tree_size-NUM_CHARS)*3 + NUM_SECONDARY_LENGTHS ) / (double)n;
|
|
#ifdef DEBUG_ENTROPY
|
|
fprintf(stderr, "n = %d\n", n);
|
|
fprintf(stderr, "main entropy = %f\n", rn_ln2 *(n_ln_n + lzud->main_entropy) );
|
|
fprintf(stderr, "compression ratio (raw) = %f\n", 100.0 * rn_ln2 *(n_ln_n + lzud->main_entropy) /9.0 );
|
|
fprintf(stderr, "compression ratio (ovh) = %f\n", 100.0 * cur_ratio/9.0);
|
|
#endif
|
|
if (cur_ratio > lzud->last_ratio) {
|
|
#ifdef DEBUG_ENTROPY
|
|
fprintf(stderr, "resetting huffman tables at %d\n", n);
|
|
#endif
|
|
lzud->subdivide = -1;
|
|
lz_stop_compressing(lzud->lzi);
|
|
}
|
|
lzud->last_ratio = cur_ratio;
|
|
}
|
|
}
|
|
|
|
static int
|
|
lzx_output_match(lz_info *lzi, int match_pos, int match_len)
|
|
{
|
|
lzx_data *lzud = (lzx_data *)lzi->user_data;
|
|
uint32_t formatted_offset;
|
|
uint32_t position_footer;
|
|
uint8_t length_footer;
|
|
uint8_t length_header;
|
|
uint16_t len_pos_header;
|
|
int position_slot;
|
|
short btdt;
|
|
|
|
#ifdef DEBUG_LZ
|
|
{
|
|
int i;
|
|
int pos;
|
|
for (i = 0; i < match_len; i++) {
|
|
|
|
#ifdef NONSLIDE
|
|
pos = match_pos + lzi->block_loc + i;
|
|
fprintf(stderr, "%c", lzi->block_buf[pos]);
|
|
#else
|
|
pos = match_pos + lzi->front_offset + i;
|
|
if (pos > lzi->slide_buf_size)
|
|
pos -= lzi->slide_buf_size;
|
|
fprintf(stderr, "%c", lzi->slide_buf[pos]);
|
|
#endif
|
|
}
|
|
}
|
|
#endif
|
|
position_footer = 0;
|
|
btdt = 0;
|
|
testforr:
|
|
if (match_pos == -lzud->R0) {
|
|
match_pos = 0;
|
|
formatted_offset = 0;
|
|
position_slot = 0;
|
|
}
|
|
else if (match_pos == -lzud->R1) {
|
|
lzud->R1 = lzud->R0;
|
|
lzud->R0 = -match_pos;
|
|
match_pos = 1;
|
|
formatted_offset = 1;
|
|
position_slot = 1;
|
|
}
|
|
else if (match_pos == -lzud->R2) {
|
|
lzud->R2 = lzud->R0;
|
|
lzud->R0 = -match_pos;
|
|
match_pos = 2;
|
|
formatted_offset = 2;
|
|
position_slot = 2;
|
|
}
|
|
else {
|
|
if (!btdt) {
|
|
btdt = 1;
|
|
if (find_match_at(lzi, lzud->R0, match_len, &match_pos) == 0)
|
|
goto testforr;
|
|
if (find_match_at(lzi, lzud->R1, match_len, &match_pos) == 0)
|
|
goto testforr;
|
|
if (find_match_at(lzi, lzud->R2, match_len, &match_pos) == 0)
|
|
goto testforr;
|
|
}
|
|
|
|
formatted_offset = -match_pos + 2;
|
|
|
|
if ((match_len < 3) ||
|
|
((formatted_offset >= 64) && (match_len < 4)) ||
|
|
((formatted_offset >= 2048) && (match_len < 5)) ||
|
|
((formatted_offset >= 65536) && (match_len < 6))) {
|
|
/* reject matches where extra_bits will likely be bigger than just outputting
|
|
literals. The numbers are basically derived through guessing
|
|
and trial and error */
|
|
return -1; /* reject the match */
|
|
}
|
|
|
|
lzud->R2 = lzud->R1;
|
|
lzud->R1 = lzud->R0;
|
|
lzud->R0 = -match_pos;
|
|
|
|
/* calculate position base using binary search of table; if log2 can be
|
|
done in hardware, approximation might work;
|
|
trunc(log2(formatted_offset*formatted_offset)) gets either the proper
|
|
position slot or the next one, except for slots 0, 1, and 39-49
|
|
|
|
Slots 0-1 are handled by the R0-R1 procedures
|
|
|
|
Slots 36-49 (formatted_offset >= 262144) can be found by
|
|
(formatted_offset/131072) + 34 ==
|
|
(formatted_offset >> 17) + 34;
|
|
*/
|
|
if (formatted_offset >= 262144) {
|
|
position_slot = (formatted_offset >> 17) + 34;
|
|
}
|
|
else {
|
|
int left, right, mid;
|
|
|
|
left = 3;
|
|
right = lzud->num_position_slots - 1;
|
|
position_slot = -1;
|
|
while (left <= right) {
|
|
mid = (left + right)/2;
|
|
if ((position_base[mid] <= formatted_offset) &&
|
|
position_base[mid+1] > formatted_offset) {
|
|
position_slot = mid;
|
|
break;
|
|
}
|
|
#if 0
|
|
fprintf(stderr, "BEFORE: %06x %06x %06x %06x\n",
|
|
position_base[left], position_base[mid],
|
|
formatted_offset, position_base[right]);
|
|
#endif
|
|
if (formatted_offset > position_base[mid])
|
|
/* too low */
|
|
left = mid + 1;
|
|
else /* too high */
|
|
right = mid;
|
|
#if 0
|
|
fprintf(stderr, "AFTER : %06x %06x %06x %06x\n",
|
|
position_base[left], position_base[mid],
|
|
formatted_offset, position_base[right]);
|
|
#endif
|
|
}
|
|
#ifdef DEBUG_POSITION_SLOT_LOOKUP
|
|
if (position_slot < 0) {
|
|
fprintf(stderr, "lmr npr: %d %d %d %d\n", left, mid, right, lzud->num_position_slots);
|
|
fprintf(stderr, "AFTER : %07d %07d %07d %07d\n",
|
|
position_base[left], position_base[mid],
|
|
formatted_offset, position_base[right]);
|
|
fprintf(stderr, "(%d, %d, %d, %d, %d)\n", match_pos, match_len, formatted_offset, position_slot, position_footer);
|
|
}
|
|
#endif
|
|
assert(position_slot >= 0);
|
|
/* FIXME precalc extra_mask table */
|
|
}
|
|
position_footer = ((1UL << extra_bits[position_slot]) - 1) & formatted_offset;
|
|
}
|
|
#ifdef DEBUG_MATCHES
|
|
#ifdef NONSLIDE
|
|
fprintf(stderr, "(%08x, %d, %d, %d, %d, %d)\n", lzud->lzi->cur_loc , match_pos, match_len, formatted_offset, position_slot, position_footer);
|
|
#else
|
|
fprintf(stderr, "(%08x, %d, %d, %d, %d, %d)\n", lzud->lzi->cur_loc - lzud->lzi->chars_in_match , match_pos, match_len, formatted_offset, position_slot, position_footer);
|
|
#endif
|
|
#endif
|
|
/* match length = 8 bits */
|
|
/* position_slot = 6 bits */
|
|
/* position_footer = 17 bits */
|
|
/* total = 31 bits */
|
|
/* plus one to say whether it's a literal or not */
|
|
*lzud->block_codesp++ = 0x80000000 | /* bit 31 in intelligent bit ordering */
|
|
(position_slot << 25) | /* bits 30-25 */
|
|
(position_footer << 8) | /* bits 8-24 */
|
|
(match_len - MIN_MATCH); /* bits 0-7 */
|
|
|
|
if (match_len < (NUM_PRIMARY_LENGTHS + MIN_MATCH)) {
|
|
length_header = match_len - MIN_MATCH;
|
|
/* length_footer = 255; */ /* not necessary */
|
|
}
|
|
else {
|
|
length_header = NUM_PRIMARY_LENGTHS;
|
|
length_footer = match_len - (NUM_PRIMARY_LENGTHS + MIN_MATCH);
|
|
lzud->length_freq_table[length_footer]++;
|
|
}
|
|
len_pos_header = (position_slot << 3) | length_header;
|
|
lzud->main_freq_table[len_pos_header + NUM_CHARS]++;
|
|
if (extra_bits[position_slot] >= 3) {
|
|
lzud->aligned_freq_table[position_footer & 7]++;
|
|
}
|
|
#ifndef OLDFRAMING
|
|
lzud->left_in_block -= match_len;
|
|
#endif
|
|
if (lzud->subdivide)
|
|
check_entropy(lzud, len_pos_header + NUM_CHARS);
|
|
return 0; /* accept the match */
|
|
}
|
|
|
|
static void
|
|
lzx_output_literal(lz_info *lzi, u_char ch)
|
|
{
|
|
lzx_data *lzud = (lzx_data *)lzi->user_data;
|
|
|
|
#ifndef OLDFRAMING
|
|
lzud->left_in_block--;
|
|
#endif
|
|
*lzud->block_codesp++ = ch;
|
|
#ifdef DEBUG_LZ
|
|
fprintf(stderr, "%c", ch);
|
|
#endif
|
|
lzud->main_freq_table[ch]++;
|
|
if (lzud->subdivide)
|
|
check_entropy(lzud, ch);
|
|
}
|
|
|
|
static void lzx_write_bits(lzx_data *lzxd, int nbits, uint32_t bits)
|
|
{
|
|
int cur_bits;
|
|
int shift_bits;
|
|
int rshift_bits;
|
|
uint16_t mask_bits;
|
|
|
|
#ifdef DEBUG_BITBUF
|
|
fprintf(stderr, "WB: %2d %08x\n", nbits, bits);
|
|
#endif
|
|
cur_bits = lzxd->bits_in_buf;
|
|
while ((cur_bits + nbits) >= 16) {
|
|
shift_bits = 16 - cur_bits;
|
|
rshift_bits = nbits - shift_bits;
|
|
if (shift_bits == 16) {
|
|
lzxd->bit_buf = (bits>>rshift_bits) & 0xFFFF;
|
|
}
|
|
else {
|
|
mask_bits = (1U << shift_bits) - 1;
|
|
lzxd->bit_buf <<= shift_bits;
|
|
lzxd->bit_buf |= (bits>>rshift_bits) & mask_bits;
|
|
}
|
|
#ifdef DEBUG_BITBUF
|
|
fprintf(stderr, "WBB: %04x\n", lzxd->bit_buf);
|
|
#endif
|
|
#ifdef LZX_BIG_ENDIAN
|
|
lzxd->bit_buf = ((lzxd->bit_buf & 0xFF)<<8) | (lzxd->bit_buf >> 8);
|
|
#endif
|
|
lzxd->put_bytes(lzxd->out_arg, sizeof(lzxd->bit_buf), &lzxd->bit_buf);
|
|
lzxd->len_compressed_output += sizeof(lzxd->bit_buf);
|
|
lzxd->bit_buf = 0;
|
|
nbits -= shift_bits;
|
|
cur_bits = 0;
|
|
}
|
|
/* (cur_bits + nbits) < 16. If nbits = 0, we're done.
|
|
otherwise move bits in */
|
|
shift_bits = nbits;
|
|
mask_bits = (1U << shift_bits) - 1;
|
|
lzxd->bit_buf <<= shift_bits;
|
|
lzxd->bit_buf |= bits & mask_bits;
|
|
cur_bits += nbits;
|
|
|
|
#ifdef DEBUG_BITBUF
|
|
fprintf(stderr, "OBB: %2d %04x\n", cur_bits, lzxd->bit_buf);
|
|
#endif
|
|
lzxd->bits_in_buf = cur_bits;
|
|
}
|
|
|
|
static void lzx_align_output(lzx_data *lzxd)
|
|
{
|
|
if (lzxd->bits_in_buf) {
|
|
lzx_write_bits(lzxd, 16 - lzxd->bits_in_buf, 0);
|
|
}
|
|
if (lzxd->mark_frame)
|
|
lzxd->mark_frame(lzxd->mark_frame_arg, lzxd->len_uncompressed_input, lzxd->len_compressed_output);
|
|
}
|
|
|
|
static void
|
|
lzx_write_compressed_literals(lzx_data *lzxd, int block_type)
|
|
{
|
|
uint32_t *cursor = lzxd->block_codes;
|
|
uint32_t *endp = lzxd->block_codesp;
|
|
uint16_t position_slot;
|
|
uint32_t position_footer;
|
|
uint32_t match_len_m2; /* match length minus 2, which is MIN_MATCH */
|
|
uint32_t verbatim_bits;
|
|
uint32_t block_code;
|
|
uint16_t length_header;
|
|
uint16_t length_footer;
|
|
uint16_t len_pos_header;
|
|
huff_entry *huffe;
|
|
int frame_count = (lzxd->len_uncompressed_input % LZX_FRAME_SIZE);
|
|
|
|
lzxd->len_uncompressed_input -= frame_count; /* will be added back in later */
|
|
while (cursor < endp) {
|
|
block_code = *cursor++;
|
|
if (block_code & 0x80000000) {
|
|
/*
|
|
* 0x80000000 | bit 31 in intelligent bit ordering
|
|
* (position_slot << 25) | bits 30-25
|
|
* (position_footer << 8) | bits 8-24
|
|
* (match_len - MIN_MATCH); bits 0-7
|
|
*
|
|
*/
|
|
|
|
match_len_m2 = block_code & 0xFF; /* 8 bits */
|
|
position_footer = (block_code >> 8)& 0x1FFFF; /* 17 bits */
|
|
position_slot = (block_code >> 25) & 0x3F; /* 6 bits */
|
|
|
|
#ifdef DEBUG_MATCHES_2
|
|
fprintf(stderr, "%08x, %3d %2d %d\n", lzxd->len_uncompressed_input + frame_count, match_len_m2, position_slot, position_footer);
|
|
#endif
|
|
if (match_len_m2 < NUM_PRIMARY_LENGTHS) {
|
|
length_header = match_len_m2;
|
|
length_footer = 255; /* personal encoding for NULL */
|
|
}
|
|
else {
|
|
length_header = NUM_PRIMARY_LENGTHS;
|
|
length_footer = match_len_m2 - NUM_PRIMARY_LENGTHS;
|
|
}
|
|
len_pos_header = (position_slot << 3) | length_header;
|
|
huffe = &lzxd->main_tree[len_pos_header+NUM_CHARS];
|
|
lzx_write_bits(lzxd, huffe->codelength, huffe->code);
|
|
if (length_footer != 255) {
|
|
huffe = &lzxd->length_tree[length_footer];
|
|
lzx_write_bits(lzxd, huffe->codelength, huffe->code);
|
|
}
|
|
if ((block_type == LZX_ALIGNED_OFFSET_BLOCK) && (extra_bits[position_slot] >= 3)) {
|
|
/* aligned offset block and code */
|
|
verbatim_bits = position_footer >> 3;
|
|
lzx_write_bits(lzxd, extra_bits[position_slot] - 3, verbatim_bits);
|
|
huffe = &lzxd->aligned_tree[position_footer&7];
|
|
lzx_write_bits(lzxd, huffe->codelength, huffe->code);
|
|
}
|
|
else {
|
|
verbatim_bits = position_footer;
|
|
lzx_write_bits(lzxd, extra_bits[position_slot], verbatim_bits);
|
|
}
|
|
frame_count += match_len_m2 + 2;
|
|
}
|
|
else {
|
|
/* literal */
|
|
assert(block_code < NUM_CHARS);
|
|
huffe = &lzxd->main_tree[block_code];
|
|
lzx_write_bits(lzxd, huffe->codelength, huffe->code);
|
|
frame_count++;
|
|
}
|
|
if (frame_count == LZX_FRAME_SIZE) {
|
|
lzxd->len_uncompressed_input += frame_count;
|
|
lzx_align_output(lzxd);
|
|
frame_count = 0;
|
|
}
|
|
#ifdef DEBUG_MATCHES_2
|
|
if (frame_count > LZX_FRAME_SIZE) {
|
|
fprintf(stderr, "uncomp_len = %x, frame_count = %x, block_code = %08x, match_len_m2 = %d", lzxd->len_uncompressed_input, frame_count, block_code, match_len_m2);
|
|
}
|
|
#endif
|
|
assert (frame_count < LZX_FRAME_SIZE);
|
|
}
|
|
lzxd->len_uncompressed_input += frame_count;
|
|
}
|
|
|
|
static int
|
|
lzx_write_compressed_tree(struct lzx_data *lzxd,
|
|
struct huff_entry *tree, uint8_t *prevlengths,
|
|
int treesize)
|
|
{
|
|
u_char *codes;
|
|
u_char *runs;
|
|
int freqs[LZX_PRETREE_SIZE];
|
|
int cur_run;
|
|
int last_len;
|
|
huff_entry pretree[20];
|
|
u_char *codep;
|
|
u_char *codee;
|
|
u_char *runp;
|
|
int excess;
|
|
int i;
|
|
int cur_code;
|
|
|
|
codep = codes = malloc(treesize*sizeof(char));
|
|
runp = runs = malloc(treesize*sizeof(char));
|
|
memset(freqs, 0, sizeof(freqs));
|
|
cur_run = 1;
|
|
last_len = tree[0].codelength;
|
|
for (i = 1; i <= treesize; i++) {
|
|
if ((i == treesize) || (tree[i].codelength != last_len)) {
|
|
if (last_len == 0) {
|
|
while (cur_run >= 20) {
|
|
excess = cur_run - 20;
|
|
if (excess > 31) excess = 31;
|
|
*codep++ = 18;
|
|
*runp++ = excess;
|
|
cur_run -= excess + 20;
|
|
freqs[18]++;
|
|
}
|
|
while (cur_run >= 4) {
|
|
excess = cur_run - 4;
|
|
if (excess > 15) excess = 15;
|
|
*codep++ = 17;
|
|
*runp++ = excess;
|
|
cur_run -= excess + 4;
|
|
freqs[17]++;
|
|
}
|
|
while (cur_run > 0) {
|
|
*codep = prevlengths[i - cur_run];
|
|
freqs[*codep++]++;
|
|
*runp++ = 0; /* not necessary */
|
|
cur_run--;
|
|
}
|
|
}
|
|
else {
|
|
while (cur_run >= 4) {
|
|
if (cur_run == 4) excess = 0;
|
|
else excess = 1;
|
|
*codep++ = 19;
|
|
*runp++ = excess;
|
|
freqs[19]++;
|
|
/* right, MS lies again. Code is NOT
|
|
prev_len + len (mod 17), it's prev_len - len (mod 17)*/
|
|
*codep = prevlengths[i-cur_run] - last_len;
|
|
if (*codep > 16) *codep += 17;
|
|
freqs[*codep++]++;
|
|
*runp++ = 0; /* not necessary */
|
|
cur_run -= excess+4;
|
|
}
|
|
while (cur_run > 0) {
|
|
*codep = prevlengths[i-cur_run] - last_len;
|
|
if (*codep > 16) *codep += 17;
|
|
*runp++ = 0; /* not necessary */
|
|
cur_run--;
|
|
freqs[*codep++]++;
|
|
}
|
|
}
|
|
if (i != treesize)
|
|
last_len = tree[i].codelength;
|
|
cur_run = 0;
|
|
}
|
|
cur_run++;
|
|
}
|
|
codee = codep;
|
|
#ifdef DEBUG_TREE_COMPRESSION
|
|
*codep++ = 255;
|
|
*runp++ = 255;
|
|
fprintf(stderr, "num: len code run\n");
|
|
for (i = 0; i < treesize; i++) {
|
|
fprintf(stderr, "%3d: %2d %2d %2d\n", i, tree[i].codelength, codes[i], runs[i]);
|
|
}
|
|
#endif
|
|
/* now create the huffman table and write out the pretree */
|
|
build_huffman_tree(LZX_PRETREE_SIZE, 16, freqs, pretree);
|
|
for (i = 0; i < LZX_PRETREE_SIZE; i++) {
|
|
lzx_write_bits(lzxd, 4, pretree[i].codelength);
|
|
}
|
|
codep = codes;
|
|
runp = runs;
|
|
cur_run = 0;
|
|
while (codep < codee) {
|
|
cur_code = *codep++;
|
|
lzx_write_bits(lzxd, pretree[cur_code].codelength, pretree[cur_code].code);
|
|
if (cur_code == 17) {
|
|
cur_run += *runp + 4;
|
|
lzx_write_bits(lzxd, 4, *runp);
|
|
}
|
|
else if (cur_code == 18) {
|
|
cur_run += *runp + 20;
|
|
lzx_write_bits(lzxd, 5, *runp);
|
|
}
|
|
else if (cur_code == 19) {
|
|
cur_run += *runp + 4;
|
|
lzx_write_bits(lzxd, 1, *runp);
|
|
cur_code = *codep++;
|
|
lzx_write_bits(lzxd, pretree[cur_code].codelength, pretree[cur_code].code);
|
|
runp++;
|
|
}
|
|
else {
|
|
cur_run++;
|
|
}
|
|
runp++;
|
|
}
|
|
free(codes);
|
|
free(runs);
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
lzx_reset(lzx_data *lzxd)
|
|
{
|
|
lzxd->need_1bit_header = 1;
|
|
lzxd->R0 = lzxd->R1 = lzxd->R2 = 1;
|
|
memset(lzxd->prev_main_treelengths, 0, lzxd->main_tree_size * sizeof(uint8_t));
|
|
memset(lzxd->prev_length_treelengths, 0, NUM_SECONDARY_LENGTHS * sizeof(uint8_t));
|
|
lz_reset(lzxd->lzi);
|
|
}
|
|
|
|
int lzx_compress_block(lzx_data *lzxd, int block_size, int subdivide)
|
|
{
|
|
int i;
|
|
uint32_t written_sofar = 0;
|
|
int block_type;
|
|
long uncomp_bits;
|
|
long comp_bits;
|
|
long comp_bits_ovh;
|
|
long uncomp_length;
|
|
|
|
if ((lzxd->block_size != block_size) || (lzxd->block_codes == NULL)) {
|
|
if (lzxd->block_codes != NULL) free(lzxd->block_codes);
|
|
lzxd->block_size = block_size;
|
|
lzxd->block_codes = malloc(block_size * sizeof(uint32_t));
|
|
}
|
|
lzxd->subdivide = subdivide?1:0;
|
|
|
|
lzxd->left_in_block = block_size;
|
|
lzxd->left_in_frame = LZX_FRAME_SIZE;
|
|
lzxd->main_entropy = 0.0;
|
|
lzxd->last_ratio = 9999999.0;
|
|
lzxd->block_codesp = lzxd->block_codes;
|
|
|
|
memset(lzxd->length_freq_table, 0, NUM_SECONDARY_LENGTHS * sizeof(int));
|
|
memset(lzxd->main_freq_table, 0, lzxd->main_tree_size * sizeof(int));
|
|
memset(lzxd->aligned_freq_table, 0, LZX_ALIGNED_SIZE * sizeof(int));
|
|
do {
|
|
lz_compress(lzxd->lzi, lzxd->left_in_block);
|
|
if (lzxd->left_in_frame == 0)
|
|
lzxd->left_in_frame = LZX_FRAME_SIZE;
|
|
|
|
if ((lzxd->subdivide<0) || !lzxd->left_in_block ||
|
|
(!lz_left_to_process(lzxd->lzi) && lzxd->at_eof(lzxd->in_arg))) {
|
|
/* now one block is LZ-analyzed. */
|
|
/* time to write it out */
|
|
uncomp_length = lzxd->block_size - lzxd->left_in_block - written_sofar;
|
|
/* uncomp_length will sometimes be 0 when input length is
|
|
an exact multiple of frame size */
|
|
if (uncomp_length == 0)
|
|
continue;
|
|
if (lzxd->subdivide < 0) {
|
|
#ifdef DEBUG_ENTROPY
|
|
fprintf(stderr, "subdivided\n");
|
|
#endif
|
|
lzxd->subdivide = 1;
|
|
}
|
|
|
|
if (lzxd->need_1bit_header) {
|
|
/* one bit Intel preprocessing header */
|
|
/* always 0 because this implementation doesn't do Intel preprocessing */
|
|
lzx_write_bits(lzxd, 1, 0);
|
|
lzxd->need_1bit_header = 0;
|
|
}
|
|
|
|
/* handle extra bits */
|
|
uncomp_bits = comp_bits = 0;
|
|
build_huffman_tree(LZX_ALIGNED_SIZE, 7, lzxd->aligned_freq_table, lzxd->aligned_tree);
|
|
for (i = 0; i < LZX_ALIGNED_SIZE; i++) {
|
|
uncomp_bits += lzxd->aligned_freq_table[i]* 3;
|
|
comp_bits += lzxd->aligned_freq_table[i]* lzxd->aligned_tree[i].codelength;
|
|
}
|
|
comp_bits_ovh = comp_bits + LZX_ALIGNED_SIZE * 3;
|
|
if (comp_bits_ovh < uncomp_bits)
|
|
block_type = LZX_ALIGNED_OFFSET_BLOCK;
|
|
else
|
|
block_type = LZX_VERBATIM_BLOCK;
|
|
|
|
#ifdef DEBUG_EXTRA_BITS
|
|
fprintf(stderr, "Extra bits uncompressed: %5d compressed: %5d compressed w/overhead %5d gain/loss %5d\n", uncomp_bits, comp_bits, comp_bits_ovh, uncomp_bits - comp_bits_ovh);
|
|
#endif
|
|
|
|
/* block type */
|
|
lzx_write_bits(lzxd, 3, block_type);
|
|
/* uncompressed length */
|
|
lzx_write_bits(lzxd, 24, uncomp_length);
|
|
|
|
written_sofar = lzxd->block_size - lzxd->left_in_block;
|
|
|
|
/* now write out the aligned offset trees if present */
|
|
if (block_type == LZX_ALIGNED_OFFSET_BLOCK) {
|
|
for (i = 0; i < LZX_ALIGNED_SIZE; i++) {
|
|
lzx_write_bits(lzxd, 3, lzxd->aligned_tree[i].codelength);
|
|
}
|
|
}
|
|
/* end extra bits */
|
|
build_huffman_tree(lzxd->main_tree_size, LZX_MAX_CODE_LENGTH,
|
|
lzxd->main_freq_table, lzxd->main_tree);
|
|
build_huffman_tree(NUM_SECONDARY_LENGTHS, 16,
|
|
lzxd->length_freq_table, lzxd->length_tree);
|
|
|
|
|
|
|
|
/* now write the pre-tree and tree for main 1 */
|
|
lzx_write_compressed_tree(lzxd, lzxd->main_tree, lzxd->prev_main_treelengths, NUM_CHARS);
|
|
|
|
/* now write the pre-tree and tree for main 2*/
|
|
lzx_write_compressed_tree(lzxd, lzxd->main_tree + NUM_CHARS,
|
|
lzxd->prev_main_treelengths + NUM_CHARS,
|
|
lzxd->main_tree_size - NUM_CHARS);
|
|
|
|
/* now write the pre tree and tree for length */
|
|
lzx_write_compressed_tree(lzxd, lzxd->length_tree, lzxd->prev_length_treelengths,
|
|
NUM_SECONDARY_LENGTHS);
|
|
|
|
/* now write literals */
|
|
lzx_write_compressed_literals(lzxd, block_type);
|
|
|
|
/* copy treelengths somewhere safe to do delta compression */
|
|
for (i = 0; i < lzxd->main_tree_size; i++) {
|
|
lzxd->prev_main_treelengths[i] = lzxd->main_tree[i].codelength;
|
|
}
|
|
for (i = 0; i < NUM_SECONDARY_LENGTHS; i++) {
|
|
lzxd->prev_length_treelengths[i] = lzxd->length_tree[i].codelength;
|
|
}
|
|
lzxd->main_entropy = 0.0;
|
|
lzxd->last_ratio = 9999999.0;
|
|
lzxd->block_codesp = lzxd->block_codes;
|
|
|
|
memset(lzxd->length_freq_table, 0, NUM_SECONDARY_LENGTHS * sizeof(int));
|
|
memset(lzxd->main_freq_table, 0, lzxd->main_tree_size * sizeof(int));
|
|
memset(lzxd->aligned_freq_table, 0, LZX_ALIGNED_SIZE * sizeof(int));
|
|
}
|
|
}
|
|
while (lzxd->left_in_block && (lz_left_to_process(lzxd->lzi) || !lzxd->at_eof(lzxd->in_arg)));
|
|
return 0;
|
|
}
|
|
|
|
int lzx_init(struct lzx_data **lzxdp, int wsize_code,
|
|
lzx_get_bytes_t get_bytes, void *get_bytes_arg,
|
|
lzx_at_eof_t at_eof,
|
|
lzx_put_bytes_t put_bytes, void *put_bytes_arg,
|
|
lzx_mark_frame_t mark_frame, void *mark_frame_arg)
|
|
{
|
|
int wsize;
|
|
struct lzx_data *lzxd;
|
|
|
|
if ((wsize_code < 15) || (wsize_code > 21)) {
|
|
return -1;
|
|
}
|
|
lzx_init_static();
|
|
|
|
*lzxdp = lzxd = malloc(sizeof(*lzxd));
|
|
if (lzxd == 0)
|
|
return -2;
|
|
|
|
lzxd->in_arg = get_bytes_arg;
|
|
lzxd->out_arg = put_bytes_arg;
|
|
lzxd->mark_frame_arg = mark_frame_arg;
|
|
lzxd->get_bytes = get_bytes;
|
|
lzxd->put_bytes = put_bytes;
|
|
lzxd->at_eof = at_eof;
|
|
lzxd->mark_frame = mark_frame;
|
|
|
|
wsize = 1 << (wsize_code);
|
|
|
|
lzxd->bits_in_buf = 0;
|
|
lzxd->block_codes = NULL;
|
|
lzxd->num_position_slots = num_position_slots[wsize_code-15];
|
|
lzxd->main_tree_size = (NUM_CHARS + 8 * lzxd->num_position_slots);
|
|
|
|
lzxd->main_freq_table = malloc(sizeof(int) * lzxd->main_tree_size);
|
|
lzxd->main_tree = malloc(sizeof(huff_entry)* lzxd->main_tree_size);
|
|
lzxd->prev_main_treelengths = malloc(sizeof(uint8_t)*lzxd->main_tree_size);
|
|
|
|
lzxd->lzi = malloc(sizeof (*lzxd->lzi));
|
|
/* the -3 prevents matches at wsize, wsize-1, wsize-2, all of which are illegal */
|
|
lz_init(lzxd->lzi, wsize, wsize - 3, MAX_MATCH, MIN_MATCH, LZX_FRAME_SIZE,
|
|
lzx_get_chars, lzx_output_match, lzx_output_literal,lzxd);
|
|
lzxd->len_uncompressed_input = 0;
|
|
lzxd->len_compressed_output = 0;
|
|
lzx_reset(lzxd);
|
|
return 0;
|
|
}
|
|
|
|
int lzx_finish(struct lzx_data *lzxd, struct lzx_results *lzxr)
|
|
{
|
|
/* lzx_align_output(lzxd); Not needed as long as frame padding is in place */
|
|
if (lzxr) {
|
|
lzxr->len_compressed_output = lzxd->len_compressed_output;
|
|
lzxr->len_uncompressed_input = lzxd->len_uncompressed_input;
|
|
}
|
|
lz_release(lzxd->lzi);
|
|
free(lzxd->lzi);
|
|
free(lzxd->prev_main_treelengths);
|
|
free(lzxd->main_tree);
|
|
free(lzxd->main_freq_table);
|
|
free(lzxd);
|
|
return 0;
|
|
}
|
|
|