aboutsummaryrefslogtreecommitdiffstats
path: root/tools/libxutil/lzo_stream.c
diff options
context:
space:
mode:
Diffstat (limited to 'tools/libxutil/lzo_stream.c')
-rw-r--r--tools/libxutil/lzo_stream.c596
1 files changed, 596 insertions, 0 deletions
diff --git a/tools/libxutil/lzo_stream.c b/tools/libxutil/lzo_stream.c
new file mode 100644
index 0000000000..bf7c348471
--- /dev/null
+++ b/tools/libxutil/lzo_stream.c
@@ -0,0 +1,596 @@
+/* $Id: lzo_stream.c,v 1.4 2003/09/30 15:22:53 mjw Exp $ */
+#define __FILE_ID_INFO "$Id: lzo_stream.c,v 1.4 2003/09/30 15:22:53 mjw Exp $"
+#include <what.h>
+static char __rcsid[] __attribute__((unused)) = WHAT_ID __FILE_ID_INFO;
+/*
+ * Copyright (C) 2003 Hewlett-Packard Company.
+ *
+ * This library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+/** @file
+ * An IOStream implementation using LZO to provide compression and decompression.
+ * This is designed to provide reasonable compression without output latency.
+ * Flushing an LZO stream flushes all pending data to the underlying stream.
+ * This is essential for stream-based (e.g. networked) applications.
+ *
+ * A compressed data stream is a sequence of blocks.
+ * Each block except the last is the plain data size followed by the compressed data size
+ * and the compressed data. The last block has plain data size zero and omits the rest.
+ * Sizes are 4-byte unsigned in network order. If the compressed size is smaller than
+ * the plain size the block data is compressed, otherwise it is plain (uncompressed).
+ *
+ * This format allows compressed data to be read from a stream without reading
+ * past the logical end of compressed data.
+ *
+ * @author Mike Wray <mike.wray@hpl.hp.com>
+ */
+#ifndef __KERNEL__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "lzo1x.h"
+
+#include "allocate.h"
+#include "lzo_stream.h"
+#include "file_stream.h"
+#include "marshal.h"
+
+#define dprintf(fmt, args...) fprintf(stdout, "[DEBUG] LZO>%s" fmt, __FUNCTION__, ##args)
+#define wprintf(fmt, args...) fprintf(stderr, "[WARN] LZO>%s" fmt, __FUNCTION__, ##args)
+#define iprintf(fmt, args...) fprintf(stdout, "[INFO] LZO>%s" fmt, __FUNCTION__, ##args)
+#define eprintf(fmt, args...) fprintf(stderr, "[ERROR] LZO>%s" fmt, __FUNCTION__, ##args)
+
+static int lzo_read(IOStream *s, void *buf, size_t size, size_t count);
+static int lzo_write(IOStream *s, const void *buf, size_t size, size_t count);
+static int lzo_print(IOStream *s, const char *msg, va_list args);
+static int lzo_getc(IOStream *s);
+static int lzo_error(IOStream *s);
+static int lzo_close(IOStream *s);
+static void lzo_free(IOStream *s);
+static int lzo_flush(IOStream *s);
+
+enum {
+ LZO_WRITE = 1,
+ LZO_READ = 2,
+};
+
+/** Methods used by a gzFile* IOStream. */
+static const IOMethods lzo_methods = {
+ read: lzo_read,
+ write: lzo_write,
+ print: lzo_print,
+ getc: lzo_getc,
+ error: lzo_error,
+ close: lzo_close,
+ free: lzo_free,
+ flush: lzo_flush,
+};
+
+//#define PLAIN_SIZE (64 * 1024)
+//#define PLAIN_SIZE (128 * 1024)
+#define PLAIN_SIZE (512 * 1024)
+
+//#define NOCOMPRESS
+
+typedef struct LZOState {
+ /** Flags. */
+ int flags;
+ /** Error indicator. */
+ int error;
+ /** Underlying stream for I/O. */
+ IOStream *io;
+ /** Working memory (only needed for compression, not decompression). */
+ lzo_byte *memory;
+ /** Buffer for plain (uncompressed) data. */
+ lzo_byte *plain;
+ /** Size of the plain buffer. */
+ lzo_uint plain_size;
+ /** Pointer into the plain buffer. */
+ lzo_byte *plain_ptr;
+ /** Number of bytes of plain data available. */
+ lzo_uint plain_n;
+ /** Buffer for compressed data. */
+ lzo_byte *comp;
+ /** Size of the compressed buffer. */
+ lzo_uint comp_size;
+
+ int plain_bytes;
+ int comp_bytes;
+} LZOState;
+
+void LZOState_free(LZOState *z){
+ if(!z) return;
+ deallocate(z->memory);
+ deallocate(z->plain);
+ deallocate(z->comp);
+ deallocate(z);
+}
+
+/** Maximum size of compressed data for the given plain data size.
+ *
+ * @param plain_size size of plain data
+ * @return maximum size of compressed data
+ */
+static int comp_size(int plain_size){
+ return plain_size + (plain_size / 64) + 16 + 3;
+}
+
+static int mode_flags(const char *mode, int *flags){
+ int err = 0;
+ int r=0, w=0;
+ if(!mode){
+ err = -EINVAL;
+ goto exit;
+ }
+ for(; *mode; mode++){
+ if(*mode == 'w') w = 1;
+ if(*mode == 'r') r = 1;
+ }
+ if(r + w != 1){
+ err = -EINVAL;
+ goto exit;
+ }
+ if(r) *flags |= LZO_READ;
+ if(w) *flags |= LZO_WRITE;
+ exit:
+ return err;
+}
+
+/** Get the stream state.
+ *
+ * @param s lzo stream
+ * @return stream state.
+ */
+static inline LZOState * lzo_state(IOStream *s){
+ return s->data;
+}
+
+IOStream *lzo_stream_io(IOStream *s){
+ LZOState *state = lzo_state(s);
+ return state->io;
+}
+
+static inline void set_error(LZOState *state, int err){
+ if(err < 0 && !state->error){
+ state->error = err;
+ }
+}
+
+int lzo_stream_plain_bytes(IOStream *s){
+ LZOState *state = lzo_state(s);
+ return state->plain_bytes;
+}
+
+int lzo_stream_comp_bytes(IOStream *s){
+ LZOState *state = lzo_state(s);
+ return state->comp_bytes;
+}
+
+float lzo_stream_ratio(IOStream *s){
+ LZOState *state = lzo_state(s);
+ float ratio = 0.0;
+ if(state->comp_bytes){
+ ratio = ((float) state->comp_bytes)/((float) state->plain_bytes);
+ }
+ return ratio;
+}
+
+static inline int LZOState_writeable(LZOState *state){
+ return (state->flags & LZO_WRITE) != 0;
+}
+
+static inline int LZOState_readable(LZOState *state){
+ return (state->flags & LZO_READ) != 0;
+}
+
+LZOState * LZOState_new(IOStream *io, int flags){
+ int err = -ENOMEM;
+ LZOState *z = ALLOCATE(LZOState);
+ //dprintf(">\n");
+ if(!z) goto exit;
+ z->io = io;
+ z->flags = flags;
+ if(LZOState_writeable(z)){
+ z->memory = allocate(LZO1X_1_MEM_COMPRESS);
+ if(!z->memory) goto exit;
+ }
+ z->plain_size = PLAIN_SIZE;
+ z->plain = allocate(z->plain_size);
+ if(!z->plain) goto exit;
+ z->plain_ptr = z->plain;
+ z->comp_size = comp_size(z->plain_size);
+ z->comp = allocate(z->comp_size);
+ if(!z->comp) goto exit;
+ err = 0;
+ exit:
+ if(err){
+ LZOState_free(z);
+ z = NULL;
+ }
+ //dprintf("< z=%p\n", z);
+ return z;
+}
+
+static int lzo_compress(LZOState *state){
+ int err = 0;
+ int k, comp_n;
+ //dprintf(">\n");
+ //dprintf(">plain=%p plain_n=%d comp=%p memory=%p\n", state->plain, state->plain_n, state->comp, state->memory);
+ // Compress the plain buffer.
+ err = lzo1x_1_compress(state->plain, state->plain_n,
+ state->comp, &comp_n,
+ state->memory);
+ //dprintf("> err=%d plain_n=%d comp_n=%d\n", err, state->plain_n, comp_n);
+ // Write plain size, compressed size.
+ err = marshal_uint32(state->io, state->plain_n);
+ if(err) goto exit;
+ err = marshal_uint32(state->io, comp_n);
+ if(err) goto exit;
+ //dprintf("> write data...\n");
+ // Write the smaller of the compressed and plain data.
+ if(state->plain_n < comp_n){
+ k = state->plain_n;
+ err = marshal_bytes(state->io, state->plain, state->plain_n);
+ } else {
+ k = comp_n;
+ err = marshal_bytes(state->io, state->comp, comp_n);
+ }
+ if(err) goto exit;
+ // Total output bytes.
+ k+= 8;
+ //dprintf("> wrote %d bytes\n", k);
+ state->plain_bytes += state->plain_n;
+ state->comp_bytes += k;
+ //dprintf("> plain=%d, comp=%d, ratio=%3.2f\n",
+ // state->plain_bytes, state->comp_bytes,
+ // ((float)state->comp_bytes)/((float)state->plain_bytes));
+ // Reset the plain buffer.
+ state->plain_ptr = state->plain;
+ state->plain_n = 0;
+ err = k;
+ exit:
+ //dprintf("< err=%d\n", err);
+ return err;
+}
+
+static int lzo_decompress(LZOState *state){
+ int plain_n, comp_n;
+ int err, k;
+ //dprintf(">\n");
+ err = unmarshal_uint32(state->io, &plain_n);
+ //dprintf("> err=%d plain_n=%d\n", err, plain_n);
+ if(err) goto exit;
+ state->comp_bytes += 4;
+ if(plain_n == 0) goto exit;
+ err = unmarshal_uint32(state->io, &comp_n);
+ //dprintf("> err=%d comp_n=%d\n", err, comp_n);
+ if(err) goto exit;
+ state->comp_bytes += 4;
+ if(plain_n > state->plain_size){
+ err = -EINVAL;
+ goto exit;
+ }
+ if(comp_n > plain_n){
+ //dprintf("> reading plain data %d...\n", plain_n);
+ k = plain_n;
+ err = unmarshal_bytes(state->io, state->plain, plain_n);
+ state->plain_n = plain_n;
+ } else {
+ //dprintf("> reading comp data %d...\n", comp_n);
+ k = comp_n;
+ err = unmarshal_bytes(state->io, state->comp, comp_n);
+ //dprintf("> decompress comp_n=%d\n", comp_n);
+ err = lzo1x_decompress(state->comp, comp_n,
+ state->plain, &state->plain_n,
+ state->memory);
+ //dprintf("> err=%d plain=%d state->plain_n=%d\n", err, plain_n, state->plain_n);
+ if(err != LZO_E_OK || state->plain_n != plain_n){
+ // Bad. Corrupted input.
+ err = -EINVAL;
+ eprintf("> Corrupted!\n");
+ goto exit;
+ }
+ }
+ state->comp_bytes += k;
+ state->plain_bytes += state->plain_n;
+ state->plain_ptr = state->plain;
+ err = k;
+ exit:
+ //dprintf("< err=%d\n", err);
+ return err;
+}
+
+/** Write to the underlying stream using fwrite();
+ *
+ * @param stream destination
+ * @param buf data
+ * @param size size of data elements
+ * @param count number of data elements to write
+ * @return number of data elements written
+ */
+static int lzo_write(IOStream *s, const void *buf, size_t size, size_t count){
+ int err = 0;
+ int n = size * count; // Total number of bytes to write.
+ int chunk; // Size of chunk to write.
+ int remaining; // Number of bytes remaining to write.
+ int space; // Amount of space left in plain buffer.
+ LZOState *state = lzo_state(s);
+#ifdef NOCOMPRESS
+ //dprintf("> buf=%p size=%d count=%d\n", buf, size, count);
+ err = IOStream_write(state->io, buf, size, count);
+ //dprintf("< err=%d\n", err);
+#else
+ //dprintf("> buf=%p size=%d count=%d n=%d\n", buf, size, count, n);
+ remaining = n;
+ space = state->plain_size - state->plain_n;
+ //dprintf("> plain=%p plain_ptr=%p plain_n=%d space=%d\n",
+ // state->plain, state->plain_ptr, state->plain_n, space);
+ while(remaining){
+ chunk = remaining;
+ if(chunk > space) chunk = space;
+ //dprintf("> memcpy %p %p %d\n", state->plain_ptr, buf, chunk);
+ memcpy(state->plain_ptr, buf, chunk);
+ remaining -= chunk;
+ space -= chunk;
+ state->plain_ptr += chunk;
+ state->plain_n += chunk;
+ if(space == 0){
+ // Input buffer is full. Compress and write it.
+ err = lzo_compress(state);
+ if(err < 0) goto exit;
+ space = state->plain_size - state->plain_n;
+ }
+ }
+ err = (size > 1 ? n / size : n);
+ exit:
+ set_error(state, err);
+#endif
+ return err;
+}
+
+
+/** Read from the underlying stream.
+ *
+ * @param stream input
+ * @param buf where to put input
+ * @param size size of data elements
+ * @param count number of data elements to read
+ * @return number of data elements read
+ */
+static int lzo_read(IOStream *s, void *buf, size_t size, size_t count){
+ int err = 0;
+ int k = 0; // Number of (plain) bytes read.
+ int remaining = size * count; // Number of bytes remaining to read.
+ int chunk; // Size of chunk to read.
+ LZOState *state = lzo_state(s);
+#ifdef NOCOMPRESS
+ //dprintf("> buf=%p size=%d count=%d\n", buf, size, count);
+ err = IOStream_read(state->io, buf, size, count);
+ //dprintf("< err=%d\n", err);
+#else
+ if(!(state->flags & LZO_READ)){
+ err = -EINVAL;
+ goto exit;
+ }
+ while(remaining){
+ if(state->plain_n == 0){
+ // No more plain input, decompress some more.
+ err = lzo_decompress(state);
+ if(err < 0) goto exit;
+ // Stop reading if there is no more input.
+ if(err == 0 || state->plain_n == 0) break;
+ }
+ chunk = remaining;
+ if(chunk > state->plain_n) chunk = state->plain_n;
+ memcpy(buf, state->plain_ptr, chunk);
+ k += chunk;
+ buf += chunk;
+ state->plain_ptr += chunk;
+ state->plain_n -= chunk;
+ remaining -= chunk;
+ }
+ err = k;
+ exit:
+ set_error(state, err);
+#endif
+ return err;
+}
+
+/** Print to the underlying stream.
+ * Returns 0 if the formatted output is too big for the internal buffer.
+ *
+ * @param s lzo stream
+ * @param msg format to use
+ * @param args arguments
+ * @return result of the print
+ */
+static int lzo_print(IOStream *s, const char *msg, va_list args){
+ char buf[1024];
+ int buf_n = sizeof(buf);
+ int n;
+ LZOState *state = lzo_state(s);
+ if(!LZOState_writeable(state)){
+ n = -EINVAL;
+ goto exit;
+ }
+ n = vsnprintf(buf, buf_n, (char*)msg, args);
+ if(n < 0) goto exit;
+ if(n > buf_n){
+ n = 0;
+ } else {
+ n = lzo_write(s, buf, 1, n);
+ }
+ exit:
+ return n;
+}
+
+/** Read a character from the underlying stream
+ *
+ * @param s lzo stream
+ * @return character read, IOSTREAM_EOF on end of file (or error)
+ */
+static int lzo_getc(IOStream *s){
+ int err;
+ char c;
+ err = lzo_read(s, &c, 1, 1);
+ if(err < 1) c = EOF;
+ err = (c==EOF ? IOSTREAM_EOF : c);
+ return err;
+}
+
+/** Flush any pending input to the underlying stream.
+ *
+ * @param s lzo stream
+ * @return 0 on success, error code otherwise
+ */
+static int lzo_flush(IOStream *s){
+ int err = 0;
+ LZOState *state = lzo_state(s);
+ //dprintf(">\n");
+#ifdef NOCOMPRESS
+ err = IOStream_flush(state->io);
+#else
+ if(!LZOState_writeable(state)){
+ err = -EINVAL;
+ goto exit;
+ }
+ if(state->plain_n){
+ err = lzo_compress(state);
+ if(err < 0) goto exit;
+ }
+ err = IOStream_flush(state->io);
+ exit:
+ set_error(state, err);
+#endif
+ //dprintf("< err=%d\n", err);
+ return (err < 0 ? err : 0);
+}
+
+/** Check if a stream has an error.
+ *
+ * @param s lzo stream
+ * @return code if has an error, 0 otherwise
+ */
+static int lzo_error(IOStream *s){
+ int err = 0;
+ LZOState *state = lzo_state(s);
+ err = state->error;
+ if(err) goto exit;
+ err = IOStream_error(state->io);
+ exit:
+ return err;
+}
+
+int lzo_stream_finish(IOStream *s){
+ int err = 0;
+ LZOState *state = lzo_state(s);
+ if(!LZOState_writeable(state)){
+ err = -EINVAL;
+ goto exit;
+ }
+ err = lzo_flush(s);
+ if(err < 0) goto exit;
+ err = marshal_int32(state->io, 0);
+ exit:
+ return err;
+}
+
+/** Close an lzo stream.
+ *
+ * @param s lzo stream to close
+ * @return result of the close
+ */
+static int lzo_close(IOStream *s){
+ int err = 0;
+ LZOState *state = lzo_state(s);
+#ifdef NOCOMPRESS
+ err = IOStream_close(state->io);
+#else
+ if(LZOState_writeable(state)){
+ err = lzo_stream_finish(s);
+ }
+ err = IOStream_close(state->io);
+ set_error(state, err);
+#endif
+ return err;
+}
+
+/** Free an lzo stream.
+ *
+ * @param s lzo stream
+ */
+static void lzo_free(IOStream *s){
+ LZOState *state = lzo_state(s);
+ IOStream_free(state->io);
+ LZOState_free(state);
+ s->data = NULL;
+}
+
+/** Create an lzo stream for an IOStream.
+ *
+ * @param io stream to wrap
+ * @return new IOStream using f for i/o
+ */
+IOStream *lzo_stream_new(IOStream *io, const char *mode){
+ int err = -ENOMEM;
+ int flags = 0;
+ IOStream *zio = NULL;
+ LZOState *state = NULL;
+
+ zio = ALLOCATE(IOStream);
+ if(!zio) goto exit;
+ err = mode_flags(mode, &flags);
+ if(err) goto exit;
+ state = LZOState_new(io, flags);
+ if(!state) goto exit;
+ err = 0;
+ zio->data = state;
+ zio->methods = &lzo_methods;
+ exit:
+ if(err){
+ if(state) LZOState_free(state);
+ if(zio) deallocate(zio);
+ zio = NULL;
+ }
+ return zio;
+}
+
+/** IOStream version of fdopen().
+ *
+ * @param fd file descriptor
+ * @param flags giving the mode to open in (as for fdopen())
+ * @return new stream for the open file, or NULL if failed
+ */
+IOStream *lzo_stream_fdopen(int fd, const char *mode){
+ int err = -ENOMEM;
+ IOStream *io = NULL, *zio = NULL;
+ io = file_stream_fdopen(fd, mode);
+ if(!io) goto exit;
+ zio = lzo_stream_new(io, mode);
+ if(!io) goto exit;
+ err = 0;
+ exit:
+ if(err){
+ IOStream_free(io);
+ IOStream_free(zio);
+ zio = NULL;
+ }
+ return zio;
+}
+#endif
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-02 00:09:39 +0000