--- a/libbb/unarchive.c +++ b/libbb/unarchive.c @@ -28,6 +28,7 @@ #include #include "libbb.h" +#include "gzip.h" #define CONFIG_FEATURE_TAR_OLDGNU_COMPATABILITY 1 #define CONFIG_FEATURE_TAR_GNU_EXTENSIONS @@ -39,38 +40,15 @@ static char *linkname = NULL; off_t archive_offset; -#define SEEK_BUF 4096 static ssize_t -seek_by_read(FILE* fd, size_t len) -{ - ssize_t cc, total = 0; - char buf[SEEK_BUF]; - - while (len) { - cc = fread(buf, sizeof(buf[0]), - len > SEEK_BUF ? SEEK_BUF : len, - fd); - - total += cc; - len -= cc; - - if(feof(fd) || ferror(fd)) - break; - } - return total; -} - -static void -seek_sub_file(FILE *fd, const int count) +seek_forward(struct gzip_handle *zh, ssize_t len) { - archive_offset += count; + ssize_t slen = gzip_seek(zh, len); - /* Do not use fseek() on a pipe. It may fail with ESPIPE, leaving the - * stream at an undefined location. - */ - seek_by_read(fd, count); + if (slen == len) + archive_offset += len; - return; + return slen; } @@ -87,7 +65,7 @@ seek_sub_file(FILE *fd, const int count) * trailing '/' or else the last dir will be assumed to be the file prefix */ static char * -extract_archive(FILE *src_stream, FILE *out_stream, +extract_archive(struct gzip_handle *src_stream, FILE *out_stream, const file_header_t *file_entry, const int function, const char *prefix, int *err) @@ -129,14 +107,14 @@ extract_archive(FILE *src_stream, FILE * if (function & extract_to_stream) { if (S_ISREG(file_entry->mode)) { - *err = copy_file_chunk(src_stream, out_stream, file_entry->size); + *err = gzip_copy(src_stream, out_stream, file_entry->size); archive_offset += file_entry->size; } } else if (function & extract_one_to_buffer) { if (S_ISREG(file_entry->mode)) { buffer = (char *) xmalloc(file_entry->size + 1); - fread(buffer, 1, file_entry->size, src_stream); + gzip_read(src_stream, buffer, file_entry->size); buffer[file_entry->size] = '\0'; archive_offset += file_entry->size; goto cleanup; @@ -156,7 +134,7 @@ extract_archive(FILE *src_stream, FILE * *err = -1; error_msg("%s not created: newer or same age file exists", file_entry->name); } - seek_sub_file(src_stream, file_entry->size); + seek_forward(src_stream, file_entry->size); goto cleanup; } } @@ -185,11 +163,11 @@ extract_archive(FILE *src_stream, FILE * } else { if ((dst_stream = wfopen(full_name, "w")) == NULL) { *err = -1; - seek_sub_file(src_stream, file_entry->size); + seek_forward(src_stream, file_entry->size); goto cleanup; } archive_offset += file_entry->size; - *err = copy_file_chunk(src_stream, dst_stream, file_entry->size); + *err = gzip_copy(src_stream, dst_stream, file_entry->size); fclose(dst_stream); } break; @@ -250,7 +228,7 @@ extract_archive(FILE *src_stream, FILE * /* If we arent extracting data we have to skip it, * if data size is 0 then then just do it anyway * (saves testing for it) */ - seek_sub_file(src_stream, file_entry->size); + seek_forward(src_stream, file_entry->size); } /* extract_list and extract_verbose_list can be used in conjunction @@ -274,8 +252,8 @@ cleanup: } static char * -unarchive(FILE *src_stream, FILE *out_stream, - file_header_t *(*get_headers)(FILE *), +unarchive(struct gzip_handle *src_stream, FILE *out_stream, + file_header_t *(*get_headers)(struct gzip_handle *), void (*free_headers)(file_header_t *), const int extract_function, const char *prefix, @@ -329,7 +307,7 @@ unarchive(FILE *src_stream, FILE *out_st } } else { /* seek past the data entry */ - seek_sub_file(src_stream, file_entry->size); + seek_forward(src_stream, file_entry->size); } free_headers(file_entry); } @@ -337,108 +315,9 @@ unarchive(FILE *src_stream, FILE *out_st return buffer; } -static file_header_t * -get_header_ar(FILE *src_stream) -{ - file_header_t *typed; - union { - char raw[60]; - struct { - char name[16]; - char date[12]; - char uid[6]; - char gid[6]; - char mode[8]; - char size[10]; - char magic[2]; - } formated; - } ar; - static char *ar_long_names; - - if (fread(ar.raw, 1, 60, src_stream) != 60) { - return(NULL); - } - archive_offset += 60; - /* align the headers based on the header magic */ - if ((ar.formated.magic[0] != '`') || (ar.formated.magic[1] != '\n')) { - /* some version of ar, have an extra '\n' after each data entry, - * this puts the next header out by 1 */ - if (ar.formated.magic[1] != '`') { - error_msg("Invalid magic"); - return(NULL); - } - /* read the next char out of what would be the data section, - * if its a '\n' then it is a valid header offset by 1*/ - archive_offset++; - if (fgetc(src_stream) != '\n') { - error_msg("Invalid magic"); - return(NULL); - } - /* fix up the header, we started reading 1 byte too early */ - /* raw_header[60] wont be '\n' as it should, but it doesnt matter */ - memmove(ar.raw, &ar.raw[1], 59); - } - - typed = (file_header_t *) xcalloc(1, sizeof(file_header_t)); - - typed->size = (size_t) atoi(ar.formated.size); - /* long filenames have '/' as the first character */ - if (ar.formated.name[0] == '/') { - if (ar.formated.name[1] == '/') { - /* If the second char is a '/' then this entries data section - * stores long filename for multiple entries, they are stored - * in static variable long_names for use in future entries */ - ar_long_names = (char *) xrealloc(ar_long_names, typed->size); - fread(ar_long_names, 1, typed->size, src_stream); - archive_offset += typed->size; - /* This ar entries data section only contained filenames for other records - * they are stored in the static ar_long_names for future reference */ - return (get_header_ar(src_stream)); /* Return next header */ - } else if (ar.formated.name[1] == ' ') { - /* This is the index of symbols in the file for compilers */ - seek_sub_file(src_stream, typed->size); - return (get_header_ar(src_stream)); /* Return next header */ - } else { - /* The number after the '/' indicates the offset in the ar data section - (saved in variable long_name) that conatains the real filename */ - if (!ar_long_names) { - error_msg("Cannot resolve long file name"); - return (NULL); - } - typed->name = xstrdup(ar_long_names + atoi(&ar.formated.name[1])); - } - } else { - /* short filenames */ - typed->name = xcalloc(1, 16); - strncpy(typed->name, ar.formated.name, 16); - } - typed->name[strcspn(typed->name, " /")]='\0'; - - /* convert the rest of the now valid char header to its typed struct */ - parse_mode(ar.formated.mode, &typed->mode); - typed->mtime = atoi(ar.formated.date); - typed->uid = atoi(ar.formated.uid); - typed->gid = atoi(ar.formated.gid); - - return(typed); -} - -static void -free_header_ar(file_header_t *ar_entry) -{ - if (ar_entry == NULL) - return; - - free(ar_entry->name); - if (ar_entry->link_name) - free(ar_entry->link_name); - - free(ar_entry); -} - static file_header_t * -get_header_tar(FILE *tar_stream) +get_header_tar(struct gzip_handle *tar_stream) { union { unsigned char raw[512]; @@ -467,10 +346,10 @@ get_header_tar(FILE *tar_stream) long sum = 0; if (archive_offset % 512 != 0) { - seek_sub_file(tar_stream, 512 - (archive_offset % 512)); + seek_forward(tar_stream, 512 - (archive_offset % 512)); } - if (fread(tar.raw, 1, 512, tar_stream) != 512) { + if (gzip_read(tar_stream, tar.raw, 512) != 512) { /* Unfortunately its common for tar files to have all sorts of * trailing garbage, fail silently */ // error_msg("Couldnt read header"); @@ -557,7 +436,7 @@ get_header_tar(FILE *tar_stream) # ifdef CONFIG_FEATURE_TAR_GNU_EXTENSIONS case 'L': { longname = xmalloc(tar_entry->size + 1); - if(fread(longname, tar_entry->size, 1, tar_stream) != 1) + if(gzip_read(tar_stream, longname, tar_entry->size) != tar_entry->size) return NULL; longname[tar_entry->size] = '\0'; archive_offset += tar_entry->size; @@ -566,7 +445,7 @@ get_header_tar(FILE *tar_stream) } case 'K': { linkname = xmalloc(tar_entry->size + 1); - if(fread(linkname, tar_entry->size, 1, tar_stream) != 1) + if(gzip_read(tar_stream, linkname, tar_entry->size) != tar_entry->size) return NULL; linkname[tar_entry->size] = '\0'; archive_offset += tar_entry->size; @@ -642,6 +521,9 @@ deb_extract(const char *package_filename char *ared_file = NULL; char ar_magic[8]; int gz_err; + struct gzip_handle tar_outer, tar_inner; + file_header_t *tar_header; + ssize_t len; *err = 0; @@ -672,111 +554,44 @@ deb_extract(const char *package_filename /* set the buffer size */ setvbuf(deb_stream, NULL, _IOFBF, 0x8000); - /* check ar magic */ - fread(ar_magic, 1, 8, deb_stream); - - if (strncmp(ar_magic,"!",7) == 0) { - archive_offset = 8; + memset(&tar_outer, 0, sizeof(tar_outer)); + tar_outer.file = deb_stream; + gzip_exec(&tar_outer, NULL); - while ((ar_header = get_header_ar(deb_stream)) != NULL) { - if (strcmp(ared_file, ar_header->name) == 0) { - int gunzip_pid = 0; - FILE *uncompressed_stream; - /* open a stream of decompressed data */ - uncompressed_stream = gz_open(deb_stream, &gunzip_pid); - if (uncompressed_stream == NULL) { - *err = -1; - goto cleanup; - } + /* walk through outer tar file to find ared_file */ + while ((tar_header = get_header_tar(&tar_outer)) != NULL) { + int name_offset = 0; + if (strncmp(tar_header->name, "./", 2) == 0) + name_offset = 2; - archive_offset = 0; - output_buffer = unarchive(uncompressed_stream, - out_stream, get_header_tar, - free_header_tar, - extract_function, prefix, - file_list, err); - fclose(uncompressed_stream); - gz_err = gz_close(gunzip_pid); - if (gz_err) - *err = -1; - free_header_ar(ar_header); - break; - } - if (fseek(deb_stream, ar_header->size, SEEK_CUR) == -1) { - opkg_perror(ERROR, "Couldn't fseek into %s", package_filename); - *err = -1; - free_header_ar(ar_header); - goto cleanup; - } - free_header_ar(ar_header); - } - goto cleanup; - } else if (strncmp(ar_magic, "\037\213", 2) == 0) { - /* it's a gz file, let's assume it's an opkg */ - int unzipped_opkg_pid; - FILE *unzipped_opkg_stream; - file_header_t *tar_header; - archive_offset = 0; - if (fseek(deb_stream, 0, SEEK_SET) == -1) { - opkg_perror(ERROR, "Couldn't fseek into %s", package_filename); - *err = -1; - goto cleanup; - } - unzipped_opkg_stream = gz_open(deb_stream, &unzipped_opkg_pid); - if (unzipped_opkg_stream == NULL) { - *err = -1; - goto cleanup; - } + if (strcmp(ared_file, tar_header->name+name_offset) == 0) { + memset(&tar_inner, 0, sizeof(tar_inner)); + tar_inner.gzip = &tar_outer; + gzip_exec(&tar_inner, NULL); - /* walk through outer tar file to find ared_file */ - while ((tar_header = get_header_tar(unzipped_opkg_stream)) != NULL) { - int name_offset = 0; - if (strncmp(tar_header->name, "./", 2) == 0) - name_offset = 2; - if (strcmp(ared_file, tar_header->name+name_offset) == 0) { - int gunzip_pid = 0; - FILE *uncompressed_stream; - /* open a stream of decompressed data */ - uncompressed_stream = gz_open(unzipped_opkg_stream, &gunzip_pid); - if (uncompressed_stream == NULL) { - *err = -1; - goto cleanup; - } - archive_offset = 0; + archive_offset = 0; - output_buffer = unarchive(uncompressed_stream, - out_stream, - get_header_tar, - free_header_tar, - extract_function, - prefix, - file_list, - err); + output_buffer = unarchive(&tar_inner, + out_stream, + get_header_tar, + free_header_tar, + extract_function, + prefix, + file_list, + err); - free_header_tar(tar_header); - fclose(uncompressed_stream); - gz_err = gz_close(gunzip_pid); - if (gz_err) - *err = -1; - break; - } - seek_sub_file(unzipped_opkg_stream, tar_header->size); free_header_tar(tar_header); + gzip_close(&tar_inner); + break; } - fclose(unzipped_opkg_stream); - gz_err = gz_close(unzipped_opkg_pid); - if (gz_err) - *err = -1; - goto cleanup; - } else { - *err = -1; - error_msg("%s: invalid magic", package_filename); + seek_forward(&tar_outer, tar_header->size); + free_header_tar(tar_header); } cleanup: - if (deb_stream) - fclose(deb_stream); + gzip_close(&tar_outer); + if (file_list) free(file_list); --- /dev/null +++ b/libbb/gzip.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2016 Jo-Philipp Wich + * + * Zlib decrompression utility routines. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Library General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#include +#include +#include + +struct gzip_handle { + FILE *file; + struct gzip_handle *gzip; + + pid_t pid; + int rfd, wfd; + struct sigaction pipe_sa; + pthread_t thread; +}; + +int gzip_exec(struct gzip_handle *zh, const char *filename); +ssize_t gzip_read(struct gzip_handle *zh, char *buf, ssize_t len); +ssize_t gzip_copy(struct gzip_handle *zh, FILE *out, ssize_t len); +int gzip_close(struct gzip_handle *zh); +FILE *gzip_fdopen(struct gzip_handle *zh, const char *filename); + +#define gzip_seek(zh, len) gzip_copy(zh, NULL, len) --- a/libbb/Makefile.am +++ b/libbb/Makefile.am @@ -4,9 +4,8 @@ ALL_CFLAGS=-g -O -Wall -DHOST_CPU_STR=\" noinst_LIBRARIES = libbb.a -libbb_a_SOURCES = gz_open.c \ +libbb_a_SOURCES = \ libbb.h \ - unzip.c \ wfopen.c \ unarchive.c \ copy_file.c \ @@ -20,7 +19,8 @@ libbb_a_SOURCES = gz_open.c \ parse_mode.c \ time_string.c \ all_read.c \ - mode_string.c + mode_string.c \ + gzip.c libbb_la_CFLAGS = $(ALL_CFLAGS) #libbb_la_LDFLAGS = -static --- /dev/null +++ b/libbb/gzip.c @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2016 Jo-Philipp Wich + * Copyright (C) 2016 Felix Fietkau + * + * Zlib decrompression utility routines. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Library General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gzip.h" + +static void +to_devnull(int fd) +{ + int devnull = open("/dev/null", fd ? O_WRONLY : O_RDONLY); + + if (devnull >= 0) + dup2(devnull, fd); + + if (devnull > STDERR_FILENO) + close(devnull); +} + +void * +gzip_thread(void *ptr) +{ + struct gzip_handle *zh = ptr; + char buf[4096]; + int len, ret; + + while (1) { + if (zh->file) + len = fread(buf, 1, sizeof(buf), zh->file); + else if (zh->gzip) + len = gzip_read(zh->gzip, buf, sizeof(buf)); + + if (len <= 0) + break; + + do { + ret = write(zh->wfd, buf, len); + } while (ret == -1 && errno == EINTR); + } + + close(zh->wfd); + zh->wfd = -1; +} + +int +gzip_exec(struct gzip_handle *zh, const char *filename) +{ + int rpipe[2] = { -1, -1 }, wpipe[2] = { -1, -1 }; + struct sigaction pipe_sa = { .sa_handler = SIG_IGN }; + + zh->rfd = -1; + zh->wfd = -1; + + if (sigaction(SIGPIPE, &pipe_sa, &zh->pipe_sa) < 0) + return -1; + + if (pipe(rpipe) < 0) + return -1; + + if (!filename && pipe(wpipe) < 0) { + close(rpipe[0]); + close(rpipe[1]); + return -1; + } + + zh->pid = vfork(); + + switch (zh->pid) { + case -1: + return -1; + + case 0: + to_devnull(STDERR_FILENO); + + if (filename) { + to_devnull(STDIN_FILENO); + } + else { + dup2(wpipe[0], STDIN_FILENO); + close(wpipe[0]); + close(wpipe[1]); + } + + dup2(rpipe[1], STDOUT_FILENO); + close(rpipe[0]); + close(rpipe[1]); + + execlp("gzip", "gzip", "-d", filename ? filename : "-c", NULL); + exit(-1); + + default: + zh->rfd = rpipe[0]; + zh->wfd = wpipe[1]; + + fcntl(zh->rfd, F_SETFD, fcntl(zh->rfd, F_GETFD) | FD_CLOEXEC); + close(rpipe[1]); + + if (zh->wfd >= 0) { + fcntl(zh->wfd, F_SETFD, fcntl(zh->wfd, F_GETFD) | FD_CLOEXEC); + close(wpipe[0]); + pthread_create(&zh->thread, NULL, gzip_thread, zh); + } + } + + return 0; +} + +ssize_t +gzip_read(struct gzip_handle *zh, char *buf, ssize_t len) +{ + ssize_t ret; + + do { + ret = read(zh->rfd, buf, len); + } while (ret == -1 && errno != EINTR); + + return ret; +} + +ssize_t +gzip_copy(struct gzip_handle *zh, FILE *out, ssize_t len) +{ + char buf[4096]; + ssize_t rlen, total = 0; + + while (len > 0) { + rlen = gzip_read(zh, buf, + (len > sizeof(buf)) ? sizeof(buf) : len); + + if (rlen <= 0) + break; + + if (out != NULL) { + if (fwrite(buf, 1, rlen, out) != rlen) + break; + } + + len -= rlen; + total += rlen; + } + + return total; +} + +FILE * +gzip_fdopen(struct gzip_handle *zh, const char *filename) +{ + memset(zh, 0, sizeof(*zh)); + + if (!filename || gzip_exec(zh, filename) < 0) + return NULL; + + fcntl(zh->rfd, F_SETFL, fcntl(zh->rfd, F_GETFL) & ~O_NONBLOCK); + + return fdopen(zh->rfd, "r"); +} + +int +gzip_close(struct gzip_handle *zh) +{ + int code = -1; + + if (zh->rfd >= 0) + close(zh->rfd); + + if (zh->wfd >= 0) + close(zh->wfd); + + if (zh->pid > 0) { + kill(zh->pid, SIGKILL); + waitpid(zh->pid, &code, 0); + } + + if (zh->file) + fclose(zh->file); + + if (zh->thread) + pthread_join(zh->thread, NULL); + + sigaction(SIGPIPE, &zh->pipe_sa, NULL); + + return WIFEXITED(code) ? WEXITSTATUS(code) : -1; +} --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,4 +3,4 @@ bin_PROGRAMS = opkg-cl opkg_cl_SOURCES = opkg-cl.c opkg_cl_LDADD = $(top_builddir)/libopkg/libopkg.a \ - $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS) + $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS) -lpthread --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -16,7 +16,7 @@ noinst_PROGRAMS = libopkg_test #opkg_active_list_test_SOURCES = opkg_active_list_test.c #opkg_active_list_test_CFLAGS = $(ALL_CFLAGS) -I$(top_srcdir) -libopkg_test_LDADD = $(top_builddir)/libopkg/libopkg.a $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS) +libopkg_test_LDADD = $(top_builddir)/libopkg/libopkg.a $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS) -lpthread libopkg_test_SOURCE = libopkg_test.c libopkg_test_LDFLAGS = -static