From 71478fd62d8483483abb34609cdabb7f9cbadfd6 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 19 Dec 2015 14:18:43 +0000 Subject: Add hostTools from https://github.com/Noltari/cfe_bcm63xx --- hostTools/Makefile | 139 ++ hostTools/addvtoken.c | 117 ++ hostTools/bcmImageBuilder.c | 488 +++++ hostTools/cmplzma.cpp | 431 +++++ hostTools/cramfs/mkcramfs.cpp | 1049 +++++++++++ hostTools/createimg.c | 481 +++++ hostTools/lzma/compress/7z.h | 12 + hostTools/lzma/compress/7zapi.cpp | 19 + hostTools/lzma/compress/7zapi.h | 16 + hostTools/lzma/compress/7zlzma.cpp | 79 + hostTools/lzma/compress/AriBitCoder.cpp | 21 + hostTools/lzma/compress/AriBitCoder.h | 101 + hostTools/lzma/compress/AriConst.h | 17 + hostTools/lzma/compress/AriPrice.h | 12 + hostTools/lzma/compress/BinTree.h | 64 + hostTools/lzma/compress/BinTree2.h | 13 + hostTools/lzma/compress/BinTree2Main.h | 13 + hostTools/lzma/compress/BinTree3.h | 17 + hostTools/lzma/compress/BinTree3Main.h | 17 + hostTools/lzma/compress/BinTree3Z.h | 14 + hostTools/lzma/compress/BinTree3ZMain.h | 14 + hostTools/lzma/compress/BinTree4.h | 19 + hostTools/lzma/compress/BinTree4Main.h | 19 + hostTools/lzma/compress/BinTree4b.h | 21 + hostTools/lzma/compress/BinTree4bMain.h | 21 + hostTools/lzma/compress/BinTreeMF.h | 19 + hostTools/lzma/compress/BinTreeMFMain.h | 30 + hostTools/lzma/compress/BinTreeMain.h | 502 +++++ hostTools/lzma/compress/BitTreeCoder.h | 290 +++ hostTools/lzma/compress/CRC.h | 27 + hostTools/lzma/compress/Const.h | 92 + hostTools/lzma/compress/IInOutStreams.cpp | 25 + hostTools/lzma/compress/IInOutStreams.h | 31 + hostTools/lzma/compress/InByte.cpp | 41 + hostTools/lzma/compress/InByte.h | 58 + hostTools/lzma/compress/LZMA.cpp | 23 + hostTools/lzma/compress/LZMA.h | 105 ++ hostTools/lzma/compress/LZMADecoder.cpp | 276 +++ hostTools/lzma/compress/LZMADecoder.h | 64 + hostTools/lzma/compress/LZMAEncoder.cpp | 981 ++++++++++ hostTools/lzma/compress/LZMAEncoder.h | 228 +++ hostTools/lzma/compress/LenCoder.cpp | 73 + hostTools/lzma/compress/LenCoder.h | 122 ++ hostTools/lzma/compress/LiteralCoder.cpp | 66 + hostTools/lzma/compress/LiteralCoder.h | 160 ++ hostTools/lzma/compress/OutByte.cpp | 45 + hostTools/lzma/compress/OutByte.h | 42 + hostTools/lzma/compress/Portable.h | 48 + hostTools/lzma/compress/RCDefs.h | 42 + hostTools/lzma/compress/RangeCoder.h | 232 +++ hostTools/lzma/compress/StdAfx.cpp | 8 + hostTools/lzma/compress/StdAfx.h | 22 + hostTools/lzma/compress/WindowIn.cpp | 97 + hostTools/lzma/compress/WindowIn.h | 91 + hostTools/lzma/compress/WindowOut.cpp | 71 + hostTools/lzma/compress/WindowOut.h | 71 + hostTools/lzma/compress/lzDecomp.cpp | 45 + hostTools/lzma/decompress/7z.h | 16 + hostTools/lzma/decompress/7zlzma.c | 57 + hostTools/lzma/decompress/AriBitCoder.h | 51 + hostTools/lzma/decompress/BitTreeCoder.h | 160 ++ hostTools/lzma/decompress/IInOutStreams.c | 38 + hostTools/lzma/decompress/IInOutStreams.h | 62 + hostTools/lzma/decompress/LZMA.h | 83 + hostTools/lzma/decompress/LZMADecoder.c | 398 ++++ hostTools/lzma/decompress/LZMADecoder.h | 60 + hostTools/lzma/decompress/LenCoder.h | 75 + hostTools/lzma/decompress/LiteralCoder.h | 146 ++ hostTools/lzma/decompress/Makefile | 6 + hostTools/lzma/decompress/Portable.h | 59 + hostTools/lzma/decompress/RCDefs.h | 43 + hostTools/lzma/decompress/RangeCoder.h | 56 + hostTools/lzma/decompress/WindowOut.h | 47 + hostTools/lzma/decompress/vxTypesOld.h | 289 +++ hostTools/mkfs.jffs2 | Bin 0 -> 42087 bytes hostTools/nvramcrc.c | 216 +++ hostTools/nvramembed.c | 416 +++++ hostTools/scripts/Menuconfig | 1601 ++++++++++++++++ hostTools/scripts/defconfig-bcm.template | 1066 +++++++++++ hostTools/scripts/gendefconfig | 398 ++++ .../voice/ccLoadRules/CommEngine/cxcLoadRules.txt | 10 + .../voice/ccLoadRules/dslx_common/cxcLoadRules.txt | 108 ++ .../dslx_common_ldx197/cxcLoadRules.txt | 89 + .../scripts/nightlybuild/voice/cxc_nb_ce_getsrc.sh | 133 ++ .../scripts/nightlybuild/voice/cxc_nb_getsrc.sh | 133 ++ .../scripts/nightlybuild/voice/cxc_nbrt_app.bat | 13 + .../scripts/nightlybuild/voice/cxc_nbrt_build.bat | 33 + .../nightlybuild/voice/cxc_nbrt_ce_postprocess.sh | 129 ++ .../scripts/nightlybuild/voice/cxc_nbrt_cfg.bat | 117 ++ .../nightlybuild/voice/cxc_nbrt_genlabel.sh | 34 + .../scripts/nightlybuild/voice/cxc_nbrt_main.bat | 37 + .../nightlybuild/voice/cxc_nbrt_main_test.bat | 12 + .../nightlybuild/voice/cxc_nbrt_postprocess.sh | 125 ++ .../nightlybuild/voice/cxc_nbrt_prebuild.bat | 140 ++ .../scripts/nightlybuild/voice/cxc_nbrt_setenv.bat | 65 + .../scripts/nightlybuild/voice/cxc_nbrt_start.bat | 28 + .../scripts/nightlybuild/voice/cxc_nbrt_test.bat | 62 + .../nightlybuild/voice/cxc_nbrt_updatebaseline.sh | 18 + .../scripts/nightlybuild/voice/cxwarnerrorgrep.sh | 37 + .../nightlybuild/voice/lnx_nbrt_basic_cmds.exp | 115 ++ .../scripts/nightlybuild/voice/lnx_nbrt_build.sh | 9 + .../scripts/nightlybuild/voice/lnx_nbrt_prepare.sh | 9 + .../nightlybuild/voice/lnx_nbrt_view_build.exp | 53 + .../nightlybuild/voice/lnx_nbrt_view_prepare.exp | 31 + hostTools/scripts/nightlybuild/voice/readme.txt | 19 + hostTools/squashfs/Makefile | 12 + hostTools/squashfs/mksquashfs.c | 1971 ++++++++++++++++++++ hostTools/squashfs/mksquashfs.h | 47 + hostTools/squashfs/read_fs.c | 498 +++++ hostTools/squashfs/read_fs.h | 48 + hostTools/squashfs/sort.c | 304 +++ hostTools/squashfs/squashfs_fs.h | 474 +++++ hostTools/squashfs/squashfs_fs_i.h | 34 + hostTools/squashfs/squashfs_fs_sb.h | 65 + 114 files changed, 17596 insertions(+) create mode 100644 hostTools/Makefile create mode 100644 hostTools/addvtoken.c create mode 100644 hostTools/bcmImageBuilder.c create mode 100644 hostTools/cmplzma.cpp create mode 100644 hostTools/cramfs/mkcramfs.cpp create mode 100644 hostTools/createimg.c create mode 100644 hostTools/lzma/compress/7z.h create mode 100644 hostTools/lzma/compress/7zapi.cpp create mode 100644 hostTools/lzma/compress/7zapi.h create mode 100644 hostTools/lzma/compress/7zlzma.cpp create mode 100644 hostTools/lzma/compress/AriBitCoder.cpp create mode 100644 hostTools/lzma/compress/AriBitCoder.h create mode 100644 hostTools/lzma/compress/AriConst.h create mode 100644 hostTools/lzma/compress/AriPrice.h create mode 100644 hostTools/lzma/compress/BinTree.h create mode 100644 hostTools/lzma/compress/BinTree2.h create mode 100644 hostTools/lzma/compress/BinTree2Main.h create mode 100644 hostTools/lzma/compress/BinTree3.h create mode 100644 hostTools/lzma/compress/BinTree3Main.h create mode 100644 hostTools/lzma/compress/BinTree3Z.h create mode 100644 hostTools/lzma/compress/BinTree3ZMain.h create mode 100644 hostTools/lzma/compress/BinTree4.h create mode 100644 hostTools/lzma/compress/BinTree4Main.h create mode 100644 hostTools/lzma/compress/BinTree4b.h create mode 100644 hostTools/lzma/compress/BinTree4bMain.h create mode 100644 hostTools/lzma/compress/BinTreeMF.h create mode 100644 hostTools/lzma/compress/BinTreeMFMain.h create mode 100644 hostTools/lzma/compress/BinTreeMain.h create mode 100644 hostTools/lzma/compress/BitTreeCoder.h create mode 100644 hostTools/lzma/compress/CRC.h create mode 100644 hostTools/lzma/compress/Const.h create mode 100644 hostTools/lzma/compress/IInOutStreams.cpp create mode 100644 hostTools/lzma/compress/IInOutStreams.h create mode 100644 hostTools/lzma/compress/InByte.cpp create mode 100644 hostTools/lzma/compress/InByte.h create mode 100644 hostTools/lzma/compress/LZMA.cpp create mode 100644 hostTools/lzma/compress/LZMA.h create mode 100644 hostTools/lzma/compress/LZMADecoder.cpp create mode 100644 hostTools/lzma/compress/LZMADecoder.h create mode 100644 hostTools/lzma/compress/LZMAEncoder.cpp create mode 100644 hostTools/lzma/compress/LZMAEncoder.h create mode 100644 hostTools/lzma/compress/LenCoder.cpp create mode 100644 hostTools/lzma/compress/LenCoder.h create mode 100644 hostTools/lzma/compress/LiteralCoder.cpp create mode 100644 hostTools/lzma/compress/LiteralCoder.h create mode 100644 hostTools/lzma/compress/OutByte.cpp create mode 100644 hostTools/lzma/compress/OutByte.h create mode 100644 hostTools/lzma/compress/Portable.h create mode 100644 hostTools/lzma/compress/RCDefs.h create mode 100644 hostTools/lzma/compress/RangeCoder.h create mode 100644 hostTools/lzma/compress/StdAfx.cpp create mode 100644 hostTools/lzma/compress/StdAfx.h create mode 100644 hostTools/lzma/compress/WindowIn.cpp create mode 100644 hostTools/lzma/compress/WindowIn.h create mode 100644 hostTools/lzma/compress/WindowOut.cpp create mode 100644 hostTools/lzma/compress/WindowOut.h create mode 100644 hostTools/lzma/compress/lzDecomp.cpp create mode 100644 hostTools/lzma/decompress/7z.h create mode 100644 hostTools/lzma/decompress/7zlzma.c create mode 100644 hostTools/lzma/decompress/AriBitCoder.h create mode 100644 hostTools/lzma/decompress/BitTreeCoder.h create mode 100644 hostTools/lzma/decompress/IInOutStreams.c create mode 100644 hostTools/lzma/decompress/IInOutStreams.h create mode 100644 hostTools/lzma/decompress/LZMA.h create mode 100644 hostTools/lzma/decompress/LZMADecoder.c create mode 100644 hostTools/lzma/decompress/LZMADecoder.h create mode 100644 hostTools/lzma/decompress/LenCoder.h create mode 100644 hostTools/lzma/decompress/LiteralCoder.h create mode 100644 hostTools/lzma/decompress/Makefile create mode 100644 hostTools/lzma/decompress/Portable.h create mode 100644 hostTools/lzma/decompress/RCDefs.h create mode 100644 hostTools/lzma/decompress/RangeCoder.h create mode 100644 hostTools/lzma/decompress/WindowOut.h create mode 100644 hostTools/lzma/decompress/vxTypesOld.h create mode 100644 hostTools/mkfs.jffs2 create mode 100644 hostTools/nvramcrc.c create mode 100644 hostTools/nvramembed.c create mode 100644 hostTools/scripts/Menuconfig create mode 100644 hostTools/scripts/defconfig-bcm.template create mode 100644 hostTools/scripts/gendefconfig create mode 100644 hostTools/scripts/nightlybuild/voice/ccLoadRules/CommEngine/cxcLoadRules.txt create mode 100644 hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common/cxcLoadRules.txt create mode 100644 hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common_ldx197/cxcLoadRules.txt create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nb_ce_getsrc.sh create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nb_getsrc.sh create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_app.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_build.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_ce_postprocess.sh create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_cfg.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_genlabel.sh create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_main.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_main_test.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_postprocess.sh create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_prebuild.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_setenv.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_start.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_test.bat create mode 100644 hostTools/scripts/nightlybuild/voice/cxc_nbrt_updatebaseline.sh create mode 100644 hostTools/scripts/nightlybuild/voice/cxwarnerrorgrep.sh create mode 100644 hostTools/scripts/nightlybuild/voice/lnx_nbrt_basic_cmds.exp create mode 100644 hostTools/scripts/nightlybuild/voice/lnx_nbrt_build.sh create mode 100644 hostTools/scripts/nightlybuild/voice/lnx_nbrt_prepare.sh create mode 100644 hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_build.exp create mode 100644 hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_prepare.exp create mode 100644 hostTools/scripts/nightlybuild/voice/readme.txt create mode 100644 hostTools/squashfs/Makefile create mode 100644 hostTools/squashfs/mksquashfs.c create mode 100644 hostTools/squashfs/mksquashfs.h create mode 100644 hostTools/squashfs/read_fs.c create mode 100644 hostTools/squashfs/read_fs.h create mode 100644 hostTools/squashfs/sort.c create mode 100644 hostTools/squashfs/squashfs_fs.h create mode 100644 hostTools/squashfs/squashfs_fs_i.h create mode 100644 hostTools/squashfs/squashfs_fs_sb.h diff --git a/hostTools/Makefile b/hostTools/Makefile new file mode 100644 index 0000000..491c82f --- /dev/null +++ b/hostTools/Makefile @@ -0,0 +1,139 @@ +ifndef KERNEL_DIR +KERNEL_DIR = ../kernel/linux +endif +ifndef BRCM_BOARD +BRCM_BOARD = bcm963xx +endif +ifndef TARGETS_DIR +TARGETS_DIR = ../targets +endif +ifndef SHARED_DIR +SHARED_DIR = ../shared +endif +ifndef INC_BRCMBOARDPARMS_PATH +INC_BRCMBOARDPARMS_PATH = $(SHARED_DIR)/opensource/boardparms +endif +LZMADIR = ./lzma/compress +INCLUDEDIR = -I$(INC_BRCMDRIVER_PUB_PATH)/$(BRCM_BOARD) -I$(INC_BRCMSHARED_PUB_PATH)/$(BRCM_BOARD) -I. -I$(LZMADIR) +LZMAOBJS = \ + 7zapi.o \ + 7zlzma.o \ + AriBitCoder.o \ + IInOutStreams.o \ + LenCoder.o \ + LiteralCoder.o \ + LZMA.o \ + LZMAEncoder.o \ + OutByte.o \ + WindowIn.o + +CONFIG_SHELL := $(shell if [ -x "$$BASH" ]; then echo $$BASH; \ + else if [ -x /bin/bash ]; then echo /bin/bash; \ + else echo sh; fi ; fi) +TOPDIR := $(shell /bin/pwd) +SSHD_DIR = $(OPENSOURCE_DIR)/sshd +GENKEY_DIR = $(SSHD_DIR)/genkey +HOST_GENKEY = $(TOPDIR)/host_genkey +CC = gcc +STRIP = strip +RSA_KEY = $(TARGETS_DIR)/fs.src/etc/rsa_host_key +###DSS_KEY = $(TARGETS_DIR)/fs.src/etc/dss_host_key +export SSHD_DIR HOST_GENKEY CC STRIP + +#WARNINGS= -Wall + +#export DEFS = -DDEBUG +export CFLAGS = -m32 $(DEFS) $(WARNINGS) -O2 -DGNU $(INCLUDEDIR) + +vpath %.cpp $(LZMADIR) + +ifeq ($(strip $(BRCM_KERNEL_ROOTFS)),nfs) +all: +else +all: build_mkfs build_cmplzma build_hostgenkey build_imageutil +endif + +build_imageutil: bcmImageBuilder addvtoken createimg + +bcmImageBuilder: bcmImageBuilder.c + gcc $(CFLAGS) -o bcmImageBuilder bcmImageBuilder.c +addvtoken: addvtoken.c + gcc $(CFLAGS) -o addvtoken addvtoken.c +createimg: createimg.c $(INC_BRCMBOARDPARMS_PATH)/$(BRCM_BOARD)/boardparms.c + gcc $(CFLAGS) -DCONFIG_BCM9$(BRCM_CHIP) -o createimg createimg.c $(INC_BRCMBOARDPARMS_PATH)/$(BRCM_BOARD)/boardparms.c + +ifeq ($(strip $(BRCM_KERNEL_ROOTFS)),squashfs) +build_mkfs: mksquashfs $(LZMAOBJS) +else +ifeq ($(strip $(BRCM_KERNEL_ROOTFS)),cramfs) +build_mkfs: mkcramfs $(LZMAOBJS) +endif +endif + +mksquashfs: squashfs/*.c squashfs/*.h $(LZMAOBJS) + @if [ ! -e /usr/include/zlib.h ]; then \ + echo *****You need to install zlib-devel package to build mksquashfs! ; \ + echo *****You may find it in your Linux distribution CDs. ; \ + exit ;\ + fi + gcc $(CFLAGS) -Isquashfs -c squashfs/*.c + g++ -m32 -o mksquashfs mksquashfs.o read_fs.o sort.o $(LZMAOBJS) -lz + +mkcramfs: cramfs/mkcramfs.cpp $(LZMAOBJS) + @if [ ! -e /usr/include/zlib.h ]; then \ + echo *****You need to install zlib-devel package to build mkcramfs! ; \ + echo *****You may find it in your Linux distribution CDs. ; \ + exit ;\ + fi + g++ $(CFLAGS) -I$(KERNEL_DIR)/include -c cramfs/mkcramfs.cpp + g++ -m32 -o mkcramfs mkcramfs.o $(LZMAOBJS) -lz -lm + +build_cmplzma: cmplzma $(LZMAOBJS) + +cmplzma: cmplzma.cpp $(LZMAOBJS) + g++ $(CFLAGS) -c cmplzma.cpp + g++ -m32 -o cmplzma cmplzma.o $(LZMAOBJS) -lm + + +ifeq ($(strip $(BUILD_CFM_SSHD)),y) +ifeq ($(strip $(BUILD_SSHD_MIPS_GENKEY)),y) +build_hostgenkey: + @if [ -e $(RSA_KEY) ]; then \ + rm -fr $(RSA_KEY); \ + exit ;\ + fi +### @if [ -e $(DSS_KEY) ]; then \ +### rm -fr $(DSS_KEY); \ +### exit ;\ +### fi +else +build_hostgenkey: + @if [ ! -e $(RSA_KEY) ]; then \ + cd $(GENKEY_DIR) && $(MAKE); \ + $(HOST_GENKEY) -t rsa -f $(RSA_KEY); \ + exit ;\ + fi +### @if [ ! -e $(DSS_KEY) ]; then \ +### $(HOST_GENKEY) -t dss -f $(DSS_KEY); \ +### exit ;\ +### fi +endif +else +build_hostgenkey: + @if [ -e $(RSA_KEY) ]; then \ + rm -fr $(RSA_KEY); \ + exit ;\ + fi +### @if [ -e $(DSS_KEY) ]; then \ +### rm -fr $(DSS_KEY); \ +### exit ;\ +### fi +endif + +$(LZMAOBJS): %.o: %.cpp + g++ -c $(CFLAGS) $^ -o $@ + +clean: + rm -f bcmImageBuilder addvtoken createimg mksquashfs mkcramfs cmplzma $(HOST_GENKEY) + rm -fr *.o core + rm -fr $(LZMADIR)/*.o $(GENKEY_DIR)/*.o diff --git a/hostTools/addvtoken.c b/hostTools/addvtoken.c new file mode 100644 index 0000000..de0a96a --- /dev/null +++ b/hostTools/addvtoken.c @@ -0,0 +1,117 @@ +//************************************************************************************* +// Broadcom Corp. Confidential +// Copyright 2000, 2001, 2002 Broadcom Corp. All Rights Reserved. +// +// THIS SOFTWARE MAY ONLY BE USED SUBJECT TO AN EXECUTED +// SOFTWARE LICENSE AGREEMENT BETWEEN THE USER AND BROADCOM. +// YOU HAVE NO RIGHT TO USE OR EXPLOIT THIS MATERIAL EXCEPT +// SUBJECT TO THE TERMS OF SUCH AN AGREEMENT. +// +//************************************************************************************** +// File Name : addvtoken.c +// +// Description: Add validation token - 20 bytes, to the firmware image file to +// be uploaded by CFE 'w' command. For now, just 4 byte crc in +// network byte order +// +// Created : 08/18/2002 seanl +//************************************************************************************** + +#include +#include +#include +#include +#include + +typedef unsigned char byte; +typedef unsigned int UINT32; +#define BUF_SIZE 100 * 1024 + +#define BCMTAG_EXE_USE +#include "bcmTag.h" + +byte buffer[BUF_SIZE]; +byte vToken[TOKEN_LEN]; + +/*************************************************************************** +// Function Name: getCrc32 +// Description : caculate the CRC 32 of the given data. +// Parameters : pdata - array of data. +// size - number of input data bytes. +// crc - either CRC32_INIT_VALUE or previous return value. +// Returns : crc. +****************************************************************************/ +UINT32 getCrc32(byte *pdata, UINT32 size, UINT32 crc) +{ + while (size-- > 0) + crc = (crc >> 8) ^ Crc32_table[(crc ^ *pdata++) & 0xff]; + + return crc; +} + + +int main(int argc, char **argv) +{ + FILE *stream_in, *stream_out; + int total = 0, count, numWritten = 0; + UINT32 imageCrc; + + if (argc != 3) + { + printf("Usage:\n"); + printf("addcrc input-file tag-output-file\n"); + exit(1); + } + if( (stream_in = fopen(argv[1], "rb")) == NULL) + { + printf("failed on open input file: %s\n", argv[1]); + exit(1); + } + if( (stream_out = fopen(argv[2], "wb+")) == NULL) + { + printf("failed on open output file: %s\n", argv[2]); + exit(1); + } + + total = count = 0; + imageCrc = CRC32_INIT_VALUE; + + while( !feof( stream_in ) ) + { + count = fread( buffer, sizeof( char ), BUF_SIZE, stream_in ); + if( ferror( stream_in ) ) + { + perror( "Read error" ); + exit(1); + } + + imageCrc = getCrc32(buffer, count, imageCrc); + numWritten = fwrite(buffer, sizeof(char), count, stream_out); + if (ferror(stream_out)) + { + perror("addcrc: Write image file error"); + exit(1); + } + total += count; + } + + // *** assume it is always in network byte order (big endia) + imageCrc = htonl(imageCrc); + memset(vToken, 0, TOKEN_LEN); + memcpy(vToken, (byte*)&imageCrc, CRC_LEN); + + // write the crc to the end of the output file + numWritten = fwrite(vToken, sizeof(char), TOKEN_LEN, stream_out); + if (ferror(stream_out)) + { + perror("addcrc: Write image file error"); + exit(1); + } + + fclose(stream_in); + fclose(stream_out); + + printf( "addvtoken: Output file size = %d with image crc = 0x%x\n", total+TOKEN_LEN, imageCrc); + + exit(0); +} diff --git a/hostTools/bcmImageBuilder.c b/hostTools/bcmImageBuilder.c new file mode 100644 index 0000000..2e5b768 --- /dev/null +++ b/hostTools/bcmImageBuilder.c @@ -0,0 +1,488 @@ +//************************************************************************************* +// Broadcom Corp. Confidential +// Copyright 2000, 2001, 2002, 2003 Broadcom Corp. All Rights Reserved. +// +// THIS SOFTWARE MAY ONLY BE USED SUBJECT TO AN EXECUTED +// SOFTWARE LICENSE AGREEMENT BETWEEN THE USER AND BROADCOM. +// YOU HAVE NO RIGHT TO USE OR EXPLOIT THIS MATERIAL EXCEPT +// SUBJECT TO THE TERMS OF SUCH AN AGREEMENT. +// +//************************************************************************************** +// File Name : bcmImageBuilder.c +// +// Description: Build a image with a header (tag), a compressed kernel and +// a compress JFFS2 root file system +// +// Created : 04/23/2002 songw +// +// Modified : 05/18/2002 seanl +// Redefined the tag info. Add imageType and getting all the flash address +// from "board.h" which is shared by CFE and web uploading code. +// +// : 04/18/2003 seanl +// add the support for cramfs and new flash image layout: +// 1). minicfe : 64k +// 2). TAG: 256b +// 3). cramfs: +// 4). kernel: +// 5). psi: 16k (default) +// +// +//************************************************************************************** +#include +#include +#include +#include +#include +#include + +#define BCMTAG_EXE_USE +#include "bcmTag.h" + +/* The BCM963xx flash memory map contains five sections: boot loader, + * file system, kernel, nvram storage, persistent storage. This program + * defines the starting address for the boot loader, file system and kernel. + * The run-time flash driver determines the starting address of the nvram + * and persistent storage sections based on the flash part size. + */ +#include "board.h" + +/* Typedefs.h */ +typedef unsigned char byte; +typedef unsigned int UINT32; + +#define BUF_SIZE 100 * 1024 +static byte buffer[BUF_SIZE]; + +/* + struct option { + char *name; + int has_arg; + int *flag; + int val; + }; + + The name field should contain the option name without the leading double + dash. + The has_arg field should be one of: + no_argument no argument to the option is expect. + required_argument an argument to the option is required. + optional_argument an argument to the option may be presented. + If flag is non-NULL, then the integer pointed to by it will be set to the + value in the val field. If the flag field is NULL, then the val field + will be returned. Setting flag to NULL and setting val to the correspond- + ing short option will make this function act just like getopt(3). +*/ +static struct option longopts[] = { + { "help", 0, 0, 'h' }, + { "version", 0, 0, 'v' }, + { "littleendian", 0, 0, 'l'}, + { "chip", 1, 0, 0}, + { "board", 1, 0, 0}, + { "blocksize", 1, 0, 0}, + { "output", 1, 0, 0}, + { "cfefile", 1, 0, 0}, + { "rootfsfile", 1, 0, 0}, + { "kernelfile", 1, 0, 0}, + { "include-cfe", 0, 0, 'i'}, + { 0, 0, 0, 0 } +}; + +static char version[] = "Broadcom Image Builder version 0.3"; + +static char usage[] = "Usage:\t%s {-h|--help|-v|--version}\n" + "\t[--littleendian]\n" + "\t--chip -- chip id {6338|6348|6358}\n" + "\t--board --blocksize {64|128}\n" + "\t--output --cfefile \n" + "\t[--rootfsfile --kernelfile [-i|--include-cfe]]\n"; + +static char *progname; + +/*************************************************************************** +// Function Name: getCrc32 +// Description : caculate the CRC 32 of the given data. +// Parameters : pdata - array of data. +// size - number of input data bytes. +// crc - either CRC32_INIT_VALUE or previous return value. +// Returns : crc. +****************************************************************************/ +UINT32 getCrc32(byte *pdata, UINT32 size, UINT32 crc) +{ + while (size-- > 0) + crc = (crc >> 8) ^ Crc32_table[(crc ^ *pdata++) & 0xff]; + + return crc; +} + +static void build_image(int littleendian, char *chipid, char *board, unsigned int blksize, + char *outputfile, char *cfefile, char *rootfsfile, char *kernelfile, int includecfe) +{ + FILE *stream_cfe = NULL; + FILE *stream_rootfs = NULL; + FILE *stream_kernel = NULL; + FILE *stream_output = NULL; + unsigned int total_size, cfe_size, rootfs_size, kernel_size, count, numWritten; + unsigned int cfeaddr = IMAGE_BASE; + unsigned int fskerneladdr; + FILE_TAG tag; + UINT32 imageCrc, tagCrc, fsCrc, kernelCrc; + + if ((stream_cfe = fopen(cfefile, "rb")) == NULL) { + printf("bcmImageBuilder: Failed to open cfe file: %s\n", cfefile); + fclose(stream_cfe); + exit(1); + } + + if (strlen(rootfsfile) != 0 && strlen(kernelfile) != 0) { + if ((stream_rootfs = fopen(rootfsfile, "rb")) == NULL) { + printf("bcmImageBuilder: Failed to open root file system file: %s\n", rootfsfile); + fclose(stream_cfe); + exit(1); + } + if ((stream_kernel = fopen(kernelfile, "rb")) == NULL) { + printf("bcmImageBuilder: Filed to open kernel file: %s\n", kernelfile); + fclose(stream_cfe); + fclose(stream_rootfs); + exit(1); + } + } + + if((stream_output = fopen(outputfile, "wb")) == NULL) { + printf("bcmImageBuilder: Failed to create output file: %s\n", outputfile); + fclose(stream_cfe); + fclose(stream_kernel); + fclose(stream_rootfs); + exit(1); + } + + total_size = cfe_size = rootfs_size = kernel_size = count = 0; + imageCrc = CRC32_INIT_VALUE; + fsCrc = CRC32_INIT_VALUE; + kernelCrc = CRC32_INIT_VALUE; + + while (!feof( stream_cfe )) { + count = fread( buffer, sizeof(byte), BUF_SIZE, stream_cfe); + if (ferror( stream_cfe)) { + perror( "bcmImageBuilder: Read error" ); + exit(1); + } + imageCrc = getCrc32(buffer, count, imageCrc); + /* Total up actual bytes read */ + total_size += count; + cfe_size += count; + } + + /* Calculate address after cfe in flash; Tag is before fs now */ + fskerneladdr = cfeaddr + ((cfe_size + (blksize - 1)) & ~(blksize - 1)); + + if (stream_rootfs && stream_kernel) { + if (!includecfe) { + /* CFE is not included in this image */ + total_size = 0; + imageCrc = CRC32_INIT_VALUE; + fclose(stream_cfe); + stream_cfe = NULL; + } + + while (!feof( stream_rootfs )) { + count = fread( buffer, sizeof(byte), BUF_SIZE, stream_rootfs); + if (ferror( stream_rootfs)) { + perror( "bcmImageBuilder: Read error" ); + exit(1); + } + imageCrc = getCrc32(buffer, count, imageCrc); + fsCrc = getCrc32(buffer, count, fsCrc); + /* Total up actual bytes read */ + total_size += count; + rootfs_size += count; + } + + while (!feof( stream_kernel )) { + count = fread( buffer, sizeof(byte), BUF_SIZE, stream_kernel ); + if (ferror( stream_kernel)) { + perror( "bcmImageBuilder: Read error" ); + exit(1); + } + imageCrc = getCrc32(buffer, count, imageCrc); + kernelCrc = getCrc32(buffer, count, kernelCrc); + /* Total up actual bytes read */ + total_size += count; + kernel_size += count; + } + } + + if (!stream_cfe) { + cfeaddr = 0; + cfe_size = 0; + } + + if (!stream_rootfs || !stream_kernel) { + fskerneladdr = 0; + rootfs_size = 0; + kernel_size = 0; + } + + // fill the tag structure + memset(&tag, 0, sizeof(FILE_TAG)); + + strcpy(tag.tagVersion, BCM_TAG_VER); + strncpy(tag.signiture_1, BCM_SIG_1, SIG_LEN -1); + strncpy(tag.signiture_2, BCM_SIG_2, SIG_LEN_2 -1); + strncpy(tag.chipId, chipid, CHIP_ID_LEN - 1); + strncpy(tag.boardId, board, BOARD_ID_LEN -1); + if (littleendian) + strcpy(tag.bigEndian, "0"); + else + strcpy(tag.bigEndian, "1"); + + sprintf(tag.totalImageLen,"%d",total_size); + + sprintf(tag.cfeAddress, "%u", cfeaddr); + sprintf(tag.cfeLen,"%d",cfe_size); + + if (!stream_rootfs || !stream_kernel) + { + sprintf(tag.rootfsAddress,"%lu",(unsigned long) 0); + sprintf(tag.rootfsLen,"%d",rootfs_size); + sprintf(tag.kernelAddress,"%u", 0); + sprintf(tag.kernelLen,"%d",kernel_size); + } + else + { + sprintf(tag.rootfsAddress,"%lu",(unsigned long) (fskerneladdr+TAG_LEN)); + sprintf(tag.rootfsLen,"%d",rootfs_size); + sprintf(tag.kernelAddress,"%u", fskerneladdr + rootfs_size + TAG_LEN); + sprintf(tag.kernelLen,"%d",kernel_size); + } + if (!littleendian) { + imageCrc = htonl(imageCrc); + fsCrc = htonl(fsCrc); + kernelCrc = htonl(kernelCrc); + } + memcpy(tag.imageValidationToken, (byte*)&imageCrc, CRC_LEN); + memcpy(tag.imageValidationToken + CRC_LEN, (byte*)&fsCrc, CRC_LEN); + memcpy(tag.imageValidationToken + (CRC_LEN * 2), (byte*)&kernelCrc, CRC_LEN); + + // get tag crc + tagCrc = CRC32_INIT_VALUE; + tagCrc = getCrc32((byte*)&tag, TAG_LEN-TOKEN_LEN, tagCrc); + if (!littleendian) + tagCrc = htonl(tagCrc); + memcpy(tag.tagValidationToken, (byte*)&tagCrc, CRC_LEN); + +#ifdef DEBUG + printf ("tagVersion %s\n", tag.tagVersion); + printf ("signiture_1 %s\n", tag.signiture_1); + printf ("signiture_2 %s\n", tag.signiture_2); + printf ("chipId %s\n", tag.chipId); + printf ("boardId %s\n", tag.boardId); + printf ("bigEndian %s\n", tag.bigEndian); + printf ("totalImageLen %s\n", tag.totalImageLen); + printf ("cfeAddress %s, 0x%08X\n", tag.cfeAddress, (unsigned int)cfeaddr); + printf ("cfeLen %s\n", tag.cfeLen); + printf ("rootfsAddress %s, 0x%08X\n", tag.rootfsAddress, (unsigned int) fskerneladdr); + printf ("rootfsLen %s\n", tag.rootfsLen); + printf ("kernelAddress %s, 0x%08X\n", tag.kernelAddress, (unsigned int) (fskerneladdr + rootfs_size)); + printf ("kernelLen %s\n", tag.kernelLen); +#endif + + //----------------Start to write image file--------------------------- + //Write tag first + numWritten = fwrite((byte*)&tag, sizeof(byte), TAG_LEN, stream_output); + if (numWritten != TAG_LEN) { + printf("bcmImageBuilder: Failed to write tag: byte written: %d byte\n", numWritten); + exit(1); + } + + //Write the cfe if exist + if (stream_cfe) { + rewind(stream_cfe); + while (!feof( stream_cfe )) { + count = fread( buffer, sizeof(byte), BUF_SIZE, stream_cfe ); + if (ferror(stream_cfe)) { + perror("bcmImageBuilder: Read cfe file error"); + exit(1); + } + numWritten = fwrite(buffer, sizeof(byte), count, stream_output); + if (ferror(stream_output)) { + perror("bcmImageBuilder: Write image file error"); + exit(1); + } + } + fclose(stream_cfe); + } + + //Write the rootfs + if (stream_rootfs) { + rewind(stream_rootfs); + while (!feof( stream_rootfs )) { + count = fread( buffer, sizeof(byte), BUF_SIZE, stream_rootfs ); + if (ferror(stream_rootfs)) { + perror("bcmImageBuilder: Read root file system file error"); + exit(1); + } + numWritten = fwrite(buffer, sizeof(byte), count, stream_output); + if (ferror(stream_output)) { + perror("bcmImageBuilder: Write image file error"); + exit(1); + } + } + fclose(stream_rootfs); + } + + //Write the kernel + if (stream_kernel) { + rewind(stream_kernel); + while (!feof( stream_kernel )) { + count = fread( buffer, sizeof(byte), BUF_SIZE, stream_kernel ); + if (ferror(stream_kernel)) { + perror("bcmImageBuilder: Read kerenl file error"); + exit(1); + } + numWritten = fwrite(buffer, sizeof(byte), count, stream_output); + if (ferror(stream_output)) { + perror("bcmImageBuilder: Write image file error"); + exit(1); + } + } + fclose(stream_kernel); + } + + fclose(stream_output); + + printf("bcmImageBuilder\n"); + + if (stream_cfe) + printf("\tCFE image size : %u\n", cfe_size); + printf("\tFile tag size : %d\n", TAG_LEN); + if (stream_rootfs) + printf("\tRoot filesystem image size : %u\n", rootfs_size); + if (stream_kernel) + printf("\tKernel image size : %u\n", kernel_size); + + printf("\tCombined image file size : %u\n\n", total_size+TAG_LEN); +} + + +int main (int argc, char **argv) +{ + int optc; + int h = 0, v = 0, littleendian = 0, includecfe = 0, lose = 0; + int option_index = 0; + char cfefile[256], rootfsfile[256], kernelfile[256], outputfile[256]; + char boardid[BOARD_ID_LEN]; + char chipid[CHIP_ID_LEN]; + unsigned int blksize = 0; + + progname = argv[0]; + chipid[0] = boardid[0] = cfefile[0] = kernelfile[0] = rootfsfile[0] = outputfile[0] = '\0'; + + while ((optc = getopt_long (argc, argv, "hvli", longopts, &option_index)) != EOF) { + switch (optc){ + case 0: +#ifdef DEBUG + printf ("option %s", longopts[option_index].name); + if (optarg) + printf (" with arg %s", optarg); + printf ("\n"); +#endif + if (strcmp(longopts[option_index].name, "cfefile") == 0) + strcpy(cfefile, optarg); + else if (strcmp(longopts[option_index].name, "rootfsfile") == 0) + strcpy(rootfsfile, optarg); + else if (strcmp(longopts[option_index].name, "kernelfile") == 0) + strcpy(kernelfile, optarg); + else if (strcmp(longopts[option_index].name, "output") == 0) + strcpy(outputfile, optarg); + else if (strcmp(longopts[option_index].name, "chip") == 0) + strcpy(chipid, optarg); + else if (strcmp(longopts[option_index].name, "board") == 0) + strcpy(boardid, optarg); + else if (strcmp(longopts[option_index].name, "blocksize") == 0) + blksize = atoi(optarg) * 1024; + break; + case 'v': + v = 1; + break; + case 'h': + h = 1; + break; + case 'l': + littleendian = 1; + break; + case 'i': + includecfe = 1; + break; + default: + lose = 1; + break; + } + } + + if (lose || optind < argc || argc < 2) + { + /* Print error message and exit. */ + fprintf (stderr, usage, progname); + exit (1); + } + + if (v) { + /* Print version number. */ + fprintf (stderr, "%s\n", version); + if (! h) + exit (0); + } + + if (h) { + /* Print help info and exit. */ + fprintf (stderr, "%s\n", version); + fprintf (stderr, usage, progname); + fputs ("\n", stderr); + fputs (" -h, --help Print a summary of the options\n", stderr); + fputs (" -v, --version Print the version number\n", stderr); + fputs (" -l, --littleendian Build little endian image\n", stderr); + fputs (" --chip Chip id\n", stderr); + fputs (" --board Board name\n", stderr); + fputs (" --blocksize Flash memory block size in KBytes\n", stderr); + fputs (" --output Output image file name\n", stderr); + fputs (" --cfefile CFE imgage name\n", stderr); + fputs (" --kernelfile Kernel image name\n", stderr); + fputs (" --rootfsfile Root file system image name\n", stderr); + fputs (" -i, --include-cfe Add CFE to kernel and rootfs image\n", stderr); + exit (0); + } + + + if (strlen(chipid) == 0 || strlen(boardid) == 0 || strlen(outputfile) == 0 + || strlen(cfefile) == 0 || (blksize != 64*1024 && blksize !=128*1024)) + { + fprintf (stderr, "Required input parameters are missing\n\n"); + fprintf (stderr, usage, progname); + exit (1); + } + if ((strlen(rootfsfile) == 0 && strlen(kernelfile) != 0) || + (strlen(rootfsfile) != 0 && strlen(kernelfile) == 0) ) + { + fprintf (stderr, "Options --rootfsfile and --kernelfile must be used together\n\n"); + fprintf (stderr, usage, progname); + exit (1); + } + if (strlen(chipid) > (CHIP_ID_LEN - 1)) + { + fprintf (stderr, "Maximum chip id length is %d characters\n\n", (CHIP_ID_LEN - 1)); + fprintf (stderr, usage, progname); + exit (1); + } + if (strlen(boardid) > 15) + { + fprintf (stderr, "Maximum board name length is 15 characters\n\n"); + fprintf (stderr, usage, progname); + exit (1); + } + + build_image(littleendian, chipid, boardid, blksize, outputfile, cfefile, rootfsfile, kernelfile, includecfe); + + exit(0); +} + diff --git a/hostTools/cmplzma.cpp b/hostTools/cmplzma.cpp new file mode 100644 index 0000000..29fe406 --- /dev/null +++ b/hostTools/cmplzma.cpp @@ -0,0 +1,431 @@ +/* +<:copyright-broadcom + + Copyright (c) 2002 Broadcom Corporation + All Rights Reserved + No portions of this material may be reproduced in any form without the + written permission of: + Broadcom Corporation + 16215 Alton Parkway + Irvine, California 92619 + All information contained in this document is Broadcom Corporation + company private, proprietary, and trade secret. + +:> +*/ +/*************************************************************************** + * File Name : cmplzma.c + * + * Description: This program compress binary file with lzma compression method + * It requires big endian mips32 ELF and the bin files as the input + * for CFE and vmlinux. + * + * For CFE RAM compress: + * Command: cmplzma [-t] -c -2 cfe cfe.bin flashimg.S + * where cfe is elf, cfe.bin is + * binary file to be compressed and flashimg.S is Asm data array for + * the compressed binary to be linked in as data. + * + * For vmlinux: + * Command: cmplzma [-t] -k -2 vmlinux vmlinux.bin vmlinux.lz + * where vmlinux is the elf file, vmliux.bin the binary file + * and vmlinux.lz is the compressed output + * + * Updates : 04/08/2003 seanl. Created. + * + ***************************************************************************/ + +/* Includes. */ +#include +#include +#include +#include +#include +#include +#include + +#include "7z.h" + + +// elf structs and defines from CFE + +/* p_type */ +#define PT_NULL 0 /* Program header table entry unused */ +#define PT_LOAD 1 /* Loadable program segment */ +#define PT_DYNAMIC 2 /* Dynamic linking information */ +#define PT_INTERP 3 /* Program interpreter */ +#define PT_NOTE 4 /* Auxiliary information */ +#define PT_SHLIB 5 /* Reserved, unspecified semantics */ +#define PT_PHDR 6 /* Entry for header table itself */ +#define PT_LOPROC 0x70000000 /* Processor-specific */ +#define PT_HIPROC 0x7FFFFFFF /* Processor-specific */ + +#define CFE_ERR_NOTELF -11 +#define CFE_ERR_NOT32BIT -12 +#define CFE_ERR_WRONGENDIAN -13 +#define CFE_ERR_BADELFVERS -14 +#define CFE_ERR_NOTMIPS -15 +#define CFE_ERR_BADELFFMT -16 +#define CFE_ERR_BADADDR -17 + +/* e_indent */ +#define EI_MAG0 0 /* File identification byte 0 index */ +#define EI_MAG1 1 /* File identification byte 1 index */ +#define EI_MAG2 2 /* File identification byte 2 index */ +#define EI_MAG3 3 /* File identification byte 3 index */ +#define EI_CLASS 4 /* File class */ + +#define ELFCLASSNONE 0 /* Invalid class */ +#define ELFCLASS32 1 /* 32-bit objects */ +#define ELFCLASS64 2 /* 64-bit objects */ +#define EI_DATA 5 /* Data encoding */ + +#define ELFDATANONE 0 /* Invalid data encoding */ +#define ELFDATA2LSB 1 /* 2's complement, little endian */ +#define ELFDATA2MSB 2 /* 2's complement, big endian */ +#define EI_VERSION 6 /* File version */ +#define EI_PAD 7 /* Start of padding bytes */ + +#define ELFMAG0 0x7F /* Magic number byte 0 */ +#define ELFMAG1 'E' /* Magic number byte 1 */ +#define ELFMAG2 'L' /* Magic number byte 2 */ +#define ELFMAG3 'F' /* Magic number byte 3 */ + +typedef unsigned short Elf32_Half; +typedef unsigned int Elf32_Word; +typedef signed int Elf32_Sword; +typedef unsigned int Elf32_Off; +typedef unsigned int Elf32_Addr; +typedef unsigned char Elf_Char; +/* + * ELF File Header + */ +#define EI_NIDENT 16 +typedef struct { + Elf_Char e_ident[EI_NIDENT]; + Elf32_Half e_type; + Elf32_Half e_machine; + Elf32_Word e_version; + Elf32_Addr e_entry; + Elf32_Off e_phoff; + Elf32_Off e_shoff; + Elf32_Word e_flags; + Elf32_Half e_ehsize; + Elf32_Half e_phentsize; + Elf32_Half e_phnum; + Elf32_Half e_shentsize; + Elf32_Half e_shnum; + Elf32_Half e_shstrndx; +} Elf32_Ehdr; + +/* + * Program Header + */ +typedef struct { + Elf32_Word p_type; /* Identifies program segment type */ + Elf32_Off p_offset; /* Segment file offset */ + Elf32_Addr p_vaddr; /* Segment virtual address */ + Elf32_Addr p_paddr; /* Segment physical address */ + Elf32_Word p_filesz; /* Segment size in file */ + Elf32_Word p_memsz; /* Segment size in memory */ + Elf32_Word p_flags; /* Segment flags */ + Elf32_Word p_align; /* Segment alignment, file & memory */ +} Elf32_Phdr; + +int getElfInfo(char *elfFile, Elf32_Addr *eEntry, Elf32_Addr *pVaddr); + +// Check the elf file validity and extract the program entry and text +// start address +int getElfInfo(char *elfFile, Elf32_Addr *eEntry, Elf32_Addr *pVaddr) +{ + Elf32_Ehdr *ep; + Elf32_Phdr *phtab = 0; + unsigned int nbytes; + int i; + Elf32_Ehdr ehdr; + FILE *hInput; + + if ((hInput = fopen(elfFile, "rb")) == NULL) + { + printf("Error open file: %s\n", elfFile); + return -1; + } + + if (fread((char *) &ehdr, sizeof(char), sizeof(ehdr), hInput) != sizeof(ehdr)) + { + printf("Error reading file: %s\n", elfFile); + return -1; + } + + ep = &ehdr; + + *eEntry = ep->e_entry; + + /* check header validity */ + if (ep->e_ident[EI_MAG0] != ELFMAG0 || + ep->e_ident[EI_MAG1] != ELFMAG1 || + ep->e_ident[EI_MAG2] != ELFMAG2 || + ep->e_ident[EI_MAG3] != ELFMAG3) + { + printf("Not ELF file\n"); + return CFE_ERR_NOTELF; + } + + if (ep->e_ident[EI_CLASS] != ELFCLASS32) + { + printf("Not 32 bit elf\n"); + return CFE_ERR_NOT32BIT; + } + + if (ep->e_ident[EI_DATA] != ELFDATA2MSB) + { + printf("Not BIG Endian\n"); + return CFE_ERR_WRONGENDIAN; /* big endian */ + } + + /* Is there a program header? */ + if (ep->e_phoff == 0 || ep->e_phnum == 0) + { + printf("No program header? Wrong elf file\n"); + return CFE_ERR_BADELFFMT; + } + + /* Load program header */ + ep->e_phnum = htons(ep->e_phnum); + ep->e_phoff = htonl(ep->e_phoff); + nbytes = ep->e_phnum * sizeof(Elf32_Phdr); + phtab = (Elf32_Phdr *) malloc(nbytes); + if (!phtab) + { + printf("Failed to malloc memory!\n"); + return -1; + } + + if (fseek(hInput, ep->e_phoff, SEEK_SET)!= 0) + { + free(phtab); + printf("File seek error\n"); + return -1; + } + if (fread((unsigned char *)phtab, sizeof(char), nbytes, hInput) != nbytes) + { + free(phtab); + printf("File read error\n"); + return -1; + } + + for (i = 0; i < ep->e_phnum; i++) + { + Elf32_Off lowest_offset = ~0; + Elf32_Phdr *ph = 0; + ph = &phtab[i]; + phtab[i].p_offset = htonl(phtab[i].p_offset); + phtab[i].p_type = htonl(phtab[i].p_type); + if ((phtab[i].p_type == PT_LOAD) && (phtab[i].p_offset < lowest_offset)) + { + ph = &phtab[i]; + lowest_offset = ph->p_offset; + *pVaddr = ph->p_vaddr; // found the text start address + return 0; + } + } + printf("No text start address found! Wrong elf file ?\n"); + return -1; +} + +void usage(char *pName) +{ + printf("Example:\n"); + printf("To compress CFE ram : %s -c -2 inputElfFile inputBinFile outputAsmFile\n", pName); + printf("To compress linux Kernel: %s -k -2 inputElfFile inputBinFile outputCompressedFile\n\n", pName); + printf("NOTE: -2 is the default compression level. Allowable levels are -1 through -3\n"); + printf("where -3 may yield better compression ratio but slower. For faster compression, use -1\n"); + exit(-1); +} + +/************************************************************* + * Function Name: main + * Description : Program entry point that parses command line parameters + * and calls a function to create the image. + * Returns : 0 + ***************************************************************************/ +int main (int argc, char **argv) +{ + FILE *hInput = NULL, *hOutput = NULL; + struct stat StatBuf; + char inputElfFile[256], inputBinFile[256], outputFile[256], segment[6]; + unsigned char *inData, *outData; + bool status; + unsigned long outLen, inLen; + int cmpKernel = 0; + Elf32_Addr entryPoint; + Elf32_Addr textAddr; + unsigned lzma_algo; + unsigned lzma_dictsize; + unsigned lzma_fastbytes; + int lzma_compression_level; + int ret; + char *pgmName = argv[0]; + + if (argc == 7 && strcmp(argv[1], "-t") == 0) + { + strcpy(segment, ".text"); + argc--; + argv++; + } + else + strcpy(segment, ".data"); + + if (argc != 6) + usage(pgmName); + + if (strcmp(argv[1], "-k") == 0) + cmpKernel = 1; + else if (strcmp(argv[1], "-c") != 0) + usage(pgmName); + + if (strcmp(argv[2], "-1") == 0) + lzma_compression_level = 1; + else if (strcmp(argv[2], "-2") == 0) + lzma_compression_level = 2; + else if (strcmp(argv[2], "-3") == 0) + lzma_compression_level = 3; + else + usage(pgmName); + + strcpy(inputElfFile, argv[3]); + strcpy(inputBinFile, argv[4]); + strcpy(outputFile, argv[5]); + + if ((ret = getElfInfo(inputElfFile, &entryPoint, &textAddr)) != 0) + return -1; + + printf("Code text starts: textAddr=0x%08X Program entry point: 0x%08X,\n", + (unsigned int)(htonl(textAddr)), (unsigned int)(htonl(entryPoint))); + + if (stat(inputBinFile, &StatBuf ) == 0 && (hInput = fopen(inputBinFile, "rb" )) == NULL) + { + printf( "Error opening input file %s.\n\n", inputBinFile); + return -1; + } + + inData = (unsigned char *) malloc(StatBuf.st_size); + outData = (unsigned char *) malloc((StatBuf.st_size+2) * 5); // make sure we have enough output buf + + if (!inData || !outData) + { + printf( "Memory allocation error, in=0x%8.8lx, out=0x%8.8lx.\n\n", + (unsigned long) inData, (unsigned long) outData); + fclose( hInput ); + return -1; + } + + if (fread(inData, sizeof(char), StatBuf.st_size, hInput) != StatBuf.st_size) + { + printf( "Error read input file %s.\n\n", inputBinFile); + return -1; + } + + inLen = StatBuf.st_size; + + // compression function + switch (lzma_compression_level) + { + case 1 : + lzma_algo = 1; + lzma_dictsize = 1 << 20; + lzma_fastbytes = 64; + break; + case 2 : + lzma_algo = 2; + lzma_dictsize = 1 << 22; + lzma_fastbytes = 128; + break; + case 3 : + lzma_algo = 2; + lzma_dictsize = 1 << 24; + lzma_fastbytes = 255; + break; + default : + printf("Invalid LZMA compression level."); + } + outLen = 23*1024*1024; + status = compress_lzma_7z((const unsigned char*) inData, + (unsigned) inLen, + (unsigned char*) outData, + (unsigned &) outLen, + lzma_algo, + lzma_dictsize, + lzma_fastbytes); + if (status != true) + { + /* this should NEVER happen */ + printf("Internal error - LZMA compression failed.\n"); + return false; + } + + /* Open output file. */ + if ((hOutput = fopen(outputFile, "w+" )) == NULL) + { + printf ("Error opening output file %s.\n\n", outputFile); + return -1; + } + + if (cmpKernel) + { + unsigned long swapedOutLen = htonl(outLen); //little Endia on build host and big Endia on target + if (fwrite(&textAddr, sizeof(Elf32_Addr), 1, hOutput) != 1 || + fwrite(&entryPoint, sizeof(Elf32_Addr), 1, hOutput) != 1 || + fwrite(&swapedOutLen, sizeof(unsigned long), 1, hOutput) != 1 || + fwrite(outData, sizeof(char), outLen, hOutput) != outLen) + printf( "Error writing to output file.\n\n" ); + } + else // write the asm file for CFE + { + struct tm *newtime; + time_t aclock; + int i; + unsigned char *curPtr = outData, *endPtr = outData + outLen; + unsigned char *entryPtr = (unsigned char*) &entryPoint; + + time( &aclock ); /* Get time in seconds */ + newtime = localtime( &aclock ); /* Convert time to struct */ + fprintf(hOutput, "/* Convert binary to asm\n Input file : %s\n Output file: %s\n Date : %s*/\n" + , inputBinFile, outputFile, asctime(newtime)); + fprintf(hOutput, "%s", "\t.globl _binArrayStart\n"); + fprintf(hOutput, "%s", "\t.globl _binArrayEnd\n"); + fprintf(hOutput, "\t%s\n\n", segment); + fprintf(hOutput, "%s", "_binArrayStart:"); + // write 4 bytes of entry point first + fprintf(hOutput, "%s%04o", "\n\t\t.byte ", (unsigned int) *entryPtr++); + fprintf(hOutput, ",%04o", (unsigned int) *entryPtr++); + fprintf(hOutput, ",%04o", (unsigned int) *entryPtr++); + fprintf(hOutput, ",%04o", (unsigned int) *entryPtr); + i = 4; + for (; curPtr < endPtr; curPtr++) + { + if (i == 0) + fprintf(hOutput, "%s%04o", "\n\t\t.byte ", (unsigned int) *curPtr); + else + fprintf(hOutput, ",%04o", (unsigned int) *curPtr); + if (++i == 16) + i = 0; + } + fprintf(hOutput, "%s", "\n_binArrayEnd:\n"); + } + + fclose( hOutput ); + + printf("Before compression: %ld After compression (level=%d): %ld\n\r", inLen, lzma_compression_level, outLen); + printf("Percent Compression = %.2f\n\r", (float)((float)(inLen - outLen)/(float)inLen)*(float)100); + + if(inData) + free(inData); + if(outData) + free(outData); + fclose(hInput); + + return(0); +} + + diff --git a/hostTools/cramfs/mkcramfs.cpp b/hostTools/cramfs/mkcramfs.cpp new file mode 100644 index 0000000..614bcce --- /dev/null +++ b/hostTools/cramfs/mkcramfs.cpp @@ -0,0 +1,1049 @@ +/* + * mkcramfs - make a cramfs file system + * + * Copyright (C) 1999-2002 Transmeta Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * If you change the disk format of cramfs, please update fs/cramfs/README. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* BRCM Modification start */ +#include "7z.h" +/* BRCM Modification end */ + +/* Exit codes used by mkfs-type programs */ +#define MKFS_OK 0 /* No errors */ +#define MKFS_ERROR 8 /* Operational error */ +#define MKFS_USAGE 16 /* Usage or syntax error */ + +/* The kernel only supports PAD_SIZE of 0 and 512. */ +#define PAD_SIZE 512 + +/* The kernel assumes PAGE_CACHE_SIZE as block size. */ +#define PAGE_CACHE_SIZE (4096) + +/* + * The longest filename component to allow for in the input directory tree. + * ext2fs (and many others) allow up to 255 bytes. A couple of filesystems + * allow longer (e.g. smbfs 1024), but there isn't much use in supporting + * >255-byte names in the input directory tree given that such names get + * truncated to CRAMFS_MAXPATHLEN (252 bytes) when written to cramfs. + * + * Old versions of mkcramfs generated corrupted filesystems if any input + * filenames exceeded CRAMFS_MAXPATHLEN (252 bytes), however old + * versions of cramfsck seem to have been able to detect the corruption. + */ +#define MAX_INPUT_NAMELEN 255 + +/* + * Maximum size fs you can create is roughly 256MB. (The last file's + * data must begin within 256MB boundary but can extend beyond that.) + * + * Note that if you want it to fit in a ROM then you're limited to what the + * hardware and kernel can support. + */ +#define MAXFSLEN ((((1 << CRAMFS_OFFSET_WIDTH) - 1) << 2) /* offset */ \ + + (1 << CRAMFS_SIZE_WIDTH) - 1 /* filesize */ \ + + (1 << CRAMFS_SIZE_WIDTH) * 4 / PAGE_CACHE_SIZE /* block pointers */ ) + +static const char *progname = "mkcramfs"; +static unsigned int blksize = PAGE_CACHE_SIZE; +static long total_blocks = 0, total_nodes = 1; /* pre-count the root node */ +static int image_length = 0; + +/* + * If opt_holes is set, then mkcramfs can create explicit holes in the + * data, which saves 26 bytes per hole (which is a lot smaller a + * saving than most most filesystems). + * + * Note that kernels up to at least 2.3.39 don't support cramfs holes, + * which is why this is turned off by default. + * + * If opt_verbose is 1, be verbose. If it is higher, be even more verbose. + */ +static u32 opt_edition = 0; +static int opt_errors = 0; +static int opt_holes = 0; +static int opt_pad = 0; +static int opt_verbose = 0; +static char *opt_image = NULL; +static char *opt_name = NULL; +static int swap_endian = 0; +/* BRCM Modification start */ +static int opt_gzip = 0; +static int opt_compression_level = 1; +/* BRCM Modification end */ + + +static int warn_dev, warn_gid, warn_namelen, warn_skip, warn_size, warn_uid; + +/* In-core version of inode / directory entry. */ +struct entry { + /* stats */ + unsigned char *name; + unsigned int mode, size, uid, gid; + + /* these are only used for non-empty files */ + char *path; /* always null except non-empty files */ + int fd; /* temporarily open files while mmapped */ + + /* FS data */ + void *uncompressed; + /* points to other identical file */ + struct entry *same; + unsigned int offset; /* pointer to compressed data in archive */ + unsigned int dir_offset; /* Where in the archive is the directory entry? */ + + /* organization */ + struct entry *child; /* null for non-directories and empty directories */ + struct entry *next; +}; + +/* Input status of 0 to print help and exit without an error. */ +static void usage(int status) +{ + FILE *stream = status ? stderr : stdout; + + fprintf(stream, "usage: %s [-h] [-e edition] [-i file] [-n name] dirname outfile\n" + " -h print this help\n" + " -E make all warnings errors (non-zero exit status)\n" + " -e edition set edition number (part of fsid)\n" + " -g use gzip compression\n" + " -i file insert a file image into the filesystem (requires >= 2.4.0)\n" + " -l level set compression level\n" + " -n name set name of cramfs filesystem\n" + " -p pad by %d bytes for boot code\n" + " -r reverse endian-ness of filesystem\n" + " -s sort directory entries (old option, ignored)\n" + " -v be more verbose\n" + " -z make explicit holes (requires >= 2.3.39)\n" + " dirname root of the directory tree to be compressed\n" + " outfile output file\n", progname, PAD_SIZE); + + exit(status); +} + +static void die(int status, int syserr, const char *fmt, ...) +{ + va_list arg_ptr; + int save = errno; + + fflush(0); + va_start(arg_ptr, fmt); + fprintf(stderr, "%s: ", progname); + vfprintf(stderr, fmt, arg_ptr); + if (syserr) { + fprintf(stderr, ": %s", strerror(save)); + } + fprintf(stderr, "\n"); + va_end(arg_ptr); + exit(status); +} + +static void map_entry(struct entry *entry) +{ + if (entry->path) { + entry->fd = open(entry->path, O_RDONLY); + if (entry->fd < 0) { + die(MKFS_ERROR, 1, "open failed: %s", entry->path); + } + entry->uncompressed = mmap(NULL, entry->size, PROT_READ, MAP_PRIVATE, entry->fd, 0); + if (entry->uncompressed == MAP_FAILED) { + die(MKFS_ERROR, 1, "mmap failed: %s", entry->path); + } + } +} + +static void unmap_entry(struct entry *entry) +{ + if (entry->path) { + if (munmap(entry->uncompressed, entry->size) < 0) { + die(MKFS_ERROR, 1, "munmap failed: %s", entry->path); + } + close(entry->fd); + } +} + +static int find_identical_file(struct entry *orig, struct entry *newfile) +{ + if (orig == newfile) + return 1; + if (!orig) + return 0; + if (orig->size == newfile->size && (orig->path || orig->uncompressed)) + { + map_entry(orig); + map_entry(newfile); + if (!memcmp(orig->uncompressed, newfile->uncompressed, orig->size)) + { + newfile->same = orig; + unmap_entry(newfile); + unmap_entry(orig); + return 1; + } + unmap_entry(newfile); + unmap_entry(orig); + } + return (find_identical_file(orig->child, newfile) || + find_identical_file(orig->next, newfile)); +} + +static void eliminate_doubles(struct entry *root, struct entry *orig) { + if (orig) { + if (orig->size && (orig->path || orig->uncompressed)) + find_identical_file(root, orig); + eliminate_doubles(root, orig->child); + eliminate_doubles(root, orig->next); + } +} + +/* + * We define our own sorting function instead of using alphasort which + * uses strcoll and changes ordering based on locale information. + */ +static int cramsort (const void *a, const void *b) +{ + return strcmp ((*(const struct dirent **) a)->d_name, + (*(const struct dirent **) b)->d_name); +} + +static unsigned int parse_directory(struct entry *root_entry, const char *name, struct entry **prev, loff_t *fslen_ub) +{ + struct dirent **dirlist; + int totalsize = 0, dircount, dirindex; + char *path, *endpath; + size_t len = strlen(name); + + /* Set up the path. */ + /* TODO: Reuse the parent's buffer to save memcpy'ing and duplication. */ + path = (char *)malloc(len + 1 + MAX_INPUT_NAMELEN + 1); + if (!path) { + die(MKFS_ERROR, 1, "malloc failed"); + } + memcpy(path, name, len); + endpath = path + len; + *endpath = '/'; + endpath++; + + /* read in the directory and sort */ + dircount = scandir(name, &dirlist, 0, cramsort); + + if (dircount < 0) { + die(MKFS_ERROR, 1, "scandir failed: %s", name); + } + + /* process directory */ + for (dirindex = 0; dirindex < dircount; dirindex++) { + struct dirent *dirent; + struct entry *entry; + struct stat st; + int size; + size_t namelen; + + dirent = dirlist[dirindex]; + + /* Ignore "." and ".." - we won't be adding them to the archive */ + if (dirent->d_name[0] == '.') { + if (dirent->d_name[1] == '\0') + continue; + if (dirent->d_name[1] == '.') { + if (dirent->d_name[2] == '\0') + continue; + } + } + namelen = strlen(dirent->d_name); + if (namelen > MAX_INPUT_NAMELEN) { + die(MKFS_ERROR, 0, + "very long (%u bytes) filename found: %s\n" + "please increase MAX_INPUT_NAMELEN in mkcramfs.c and recompile", + namelen, dirent->d_name); + } + memcpy(endpath, dirent->d_name, namelen + 1); + + if (lstat(path, &st) < 0) { + warn_skip = 1; + continue; + } + entry = (struct entry *)calloc(1, sizeof(struct entry)); + if (!entry) { + die(MKFS_ERROR, 1, "calloc failed"); + } + entry->name = (unsigned char *)strdup(dirent->d_name); + if (!entry->name) { + die(MKFS_ERROR, 1, "strdup failed"); + } + /* truncate multi-byte UTF-8 filenames on character boundary */ + if (namelen > CRAMFS_MAXPATHLEN) { + namelen = CRAMFS_MAXPATHLEN; + warn_namelen = 1; + /* the first lost byte must not be a trail byte */ + while ((entry->name[namelen] & 0xc0) == 0x80) { + namelen--; + /* are we reasonably certain it was UTF-8 ? */ + if (entry->name[namelen] < 0x80 || !namelen) { + die(MKFS_ERROR, 0, "cannot truncate filenames not encoded in UTF-8"); + } + } + entry->name[namelen] = '\0'; + } + entry->mode = st.st_mode; + entry->size = st.st_size; + entry->uid = st.st_uid; + if (entry->uid >= 1 << CRAMFS_UID_WIDTH) + warn_uid = 1; + entry->gid = st.st_gid; + if (entry->gid >= 1 << CRAMFS_GID_WIDTH) + /* TODO: We ought to replace with a default + gid instead of truncating; otherwise there + are security problems. Maybe mode should + be &= ~070. Same goes for uid once Linux + supports >16-bit uids. */ + warn_gid = 1; + size = sizeof(struct cramfs_inode) + ((namelen + 3) & ~3); + *fslen_ub += size; + if (S_ISDIR(st.st_mode)) { + entry->size = parse_directory(root_entry, path, &entry->child, fslen_ub); + } else if (S_ISREG(st.st_mode)) { + if (entry->size) { + if (access(path, R_OK) < 0) { + warn_skip = 1; + continue; + } + entry->path = strdup(path); + if (!entry->path) { + die(MKFS_ERROR, 1, "strdup failed"); + } + if ((entry->size >= 1 << CRAMFS_SIZE_WIDTH)) { + warn_size = 1; + entry->size = (1 << CRAMFS_SIZE_WIDTH) - 1; + } + } + } else if (S_ISLNK(st.st_mode)) { + entry->uncompressed = malloc(entry->size); + if (!entry->uncompressed) { + die(MKFS_ERROR, 1, "malloc failed"); + } + if (readlink(path, (char *)entry->uncompressed, entry->size) < 0) { + warn_skip = 1; + continue; + } + } else if (S_ISFIFO(st.st_mode) || S_ISSOCK(st.st_mode)) { + /* maybe we should skip sockets */ + entry->size = 0; + } else if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode)) { + entry->size = st.st_rdev; + if (entry->size & -(1<name); + } + + if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + int blocks = ((entry->size - 1) / blksize + 1); + + /* block pointers & data expansion allowance + data */ + if (entry->size) + *fslen_ub += (4+26)*blocks + entry->size + 3; + } + + /* Link it into the list */ + *prev = entry; + prev = &entry->next; + totalsize += size; + } + free(path); + free(dirlist); /* allocated by scandir() with malloc() */ + return totalsize; +} + +/* routines to swap endianness/bitfields in inode/superblock block data */ +static void fix_inode(struct cramfs_inode *inode) +{ +#define wswap(x) (((x)>>24) | (((x)>>8)&0xff00) | (((x)&0xff00)<<8) | (((x)&0xff)<<24)) + /* attempt #2 */ + inode->mode = (inode->mode >> 8) | ((inode->mode&0xff)<<8); + inode->uid = (inode->uid >> 8) | ((inode->uid&0xff)<<8); + inode->size = (inode->size >> 16) | (inode->size&0xff00) | + ((inode->size&0xff)<<16); + ((u32*)inode)[2] = wswap(inode->offset | (inode->namelen<<26)); +} + +static void fix_offset(struct cramfs_inode *inode, u32 offset) +{ + u32 tmp = wswap(((u32*)inode)[2]); + ((u32*)inode)[2] = wswap((offset >> 2) | (tmp&0xfc000000)); +} + +static void fix_block_pointer(u32 *p) +{ + *p = wswap(*p); +} + +static void fix_super(struct cramfs_super *super) +{ + u32 *p = (u32*)super; + + /* fix superblock fields */ + p[0] = wswap(p[0]); /* magic */ + p[1] = wswap(p[1]); /* size */ + p[2] = wswap(p[2]); /* flags */ + p[3] = wswap(p[3]); /* future */ + + /* fix filesystem info fields */ + p = (u32*)&super->fsid; + p[0] = wswap(p[0]); /* crc */ + p[1] = wswap(p[1]); /* edition */ + p[2] = wswap(p[2]); /* blocks */ + p[3] = wswap(p[3]); /* files */ + + fix_inode(&super->root); +#undef wswap +} + +/* Returns sizeof(struct cramfs_super), which includes the root inode. */ +static unsigned int write_superblock(struct entry *root, char *base, int size) +{ + struct cramfs_super *super = (struct cramfs_super *) base; + unsigned int offset = sizeof(struct cramfs_super) + image_length; + + offset += opt_pad; /* 0 if no padding */ + + super->magic = CRAMFS_MAGIC; + super->flags = CRAMFS_FLAG_FSID_VERSION_2 | CRAMFS_FLAG_SORTED_DIRS; + if (opt_holes) + super->flags |= CRAMFS_FLAG_HOLES; + if (image_length > 0) + super->flags |= CRAMFS_FLAG_SHIFTED_ROOT_OFFSET; + super->size = size; + memcpy(super->signature, CRAMFS_SIGNATURE, sizeof(super->signature)); + + super->fsid.crc = crc32(0L, Z_NULL, 0); + super->fsid.edition = opt_edition; + super->fsid.blocks = total_blocks; + super->fsid.files = total_nodes; + + memset(super->name, 0x00, sizeof(super->name)); + if (opt_name) + strncpy((char *)super->name, opt_name, sizeof(super->name)); + else + strncpy((char *)super->name, "Compressed", sizeof(super->name)); + + super->root.mode = root->mode; + super->root.uid = root->uid; + super->root.gid = root->gid; + super->root.size = root->size; + super->root.offset = offset >> 2; + if (swap_endian) fix_super(super); + + return offset; +} + +static void set_data_offset(struct entry *entry, char *base, unsigned long offset) +{ + struct cramfs_inode *inode = (struct cramfs_inode *) (base + entry->dir_offset); + + if ((offset & 3) != 0) { + die(MKFS_ERROR, 0, "illegal offset of %lu bytes", offset); + } + if (offset >= (1 << (2 + CRAMFS_OFFSET_WIDTH))) { + die(MKFS_ERROR, 0, "filesystem too big"); + } + if (swap_endian) + fix_offset(inode, offset); + else + inode->offset = (offset >> 2); +} + +/* + * TODO: Does this work for chars >= 0x80? Most filesystems use UTF-8 + * encoding for filenames, whereas the console is a single-byte + * character set like iso-latin-1. + */ +static void print_node(struct entry *e) +{ + char info[10]; + char type = '?'; + + if (S_ISREG(e->mode)) type = 'f'; + else if (S_ISDIR(e->mode)) type = 'd'; + else if (S_ISLNK(e->mode)) type = 'l'; + else if (S_ISCHR(e->mode)) type = 'c'; + else if (S_ISBLK(e->mode)) type = 'b'; + else if (S_ISFIFO(e->mode)) type = 'p'; + else if (S_ISSOCK(e->mode)) type = 's'; + + if (S_ISCHR(e->mode) || (S_ISBLK(e->mode))) { + /* major/minor numbers can be as high as 2^12 or 4096 */ + snprintf(info, 10, "%4d,%4d", major(e->size), minor(e->size)); + } + else { + /* size be as high as 2^24 or 16777216 */ + snprintf(info, 10, "%9d", e->size); + } + + printf("%c %04o %s %5d:%-3d %s\n", + type, e->mode & ~S_IFMT, info, e->uid, e->gid, e->name); +} + +/* + * We do a width-first printout of the directory + * entries, using a stack to remember the directories + * we've seen. + */ +static unsigned int write_directory_structure(struct entry *entry, char *base, unsigned int offset) +{ + int stack_entries = 0; + int stack_size = 64; + struct entry **entry_stack; + + entry_stack = (struct entry **)malloc(stack_size * sizeof(struct entry *)); + if (!entry_stack) { + die(MKFS_ERROR, 1, "malloc failed"); + } + + if (opt_verbose) { + printf("root:\n"); + } + + for (;;) { + int dir_start = stack_entries; + while (entry) { + struct cramfs_inode *inode = (struct cramfs_inode *) (base + offset); + size_t len = strlen((const char *)entry->name); + + entry->dir_offset = offset; + + inode->mode = entry->mode; + inode->uid = entry->uid; + inode->gid = entry->gid; + inode->size = entry->size; + inode->offset = 0; + /* Non-empty directories, regfiles and symlinks will + write over inode->offset later. */ + + offset += sizeof(struct cramfs_inode); + total_nodes++; /* another node */ + memcpy(base + offset, entry->name, len); + /* Pad up the name to a 4-byte boundary */ + while (len & 3) { + *(base + offset + len) = '\0'; + len++; + } + inode->namelen = len >> 2; + offset += len; + + if (opt_verbose) + print_node(entry); + + if (entry->child) { + if (stack_entries >= stack_size) { + stack_size *= 2; + entry_stack = (struct entry **)realloc(entry_stack, stack_size * sizeof(struct entry *)); + if (!entry_stack) { + die(MKFS_ERROR, 1, "realloc failed"); + } + } + entry_stack[stack_entries] = entry; + stack_entries++; + } + entry = entry->next; + if (swap_endian) fix_inode(inode); + } + + /* + * Reverse the order the stack entries pushed during + * this directory, for a small optimization of disk + * access in the created fs. This change makes things + * `ls -UR' order. + */ + { + struct entry **lo = entry_stack + dir_start; + struct entry **hi = entry_stack + stack_entries; + struct entry *tmp; + + while (lo < --hi) { + tmp = *lo; + *lo++ = *hi; + *hi = tmp; + } + } + + /* Pop a subdirectory entry from the stack, and recurse. */ + if (!stack_entries) + break; + stack_entries--; + entry = entry_stack[stack_entries]; + + set_data_offset(entry, base, offset); + if (opt_verbose) { + printf("%s:\n", entry->name); + } + entry = entry->child; + } + free(entry_stack); + return offset; +} + +static int is_zero(char const *begin, unsigned len) +{ + /* Returns non-zero iff the first LEN bytes from BEGIN are all NULs. */ + return (len-- == 0 || + (begin[0] == '\0' && + (len-- == 0 || + (begin[1] == '\0' && + (len-- == 0 || + (begin[2] == '\0' && + (len-- == 0 || + (begin[3] == '\0' && + memcmp(begin, begin + 4, len) == 0)))))))); +} + +/* BRCM Modification start */ +static void dumpHex(unsigned char *start, int len) +{ + unsigned char *ptr = start, + *end = start + len; + int i; + + while (ptr < end) { + for (i=0;i<16 && ptr < end;i++) + printf("%4x", *ptr++); + printf("\n"); + } +} +/* BRCM Modification end */ + +/* + * One 4-byte pointer per block and then the actual blocked + * output. The first block does not need an offset pointer, + * as it will start immediately after the pointer block; + * so the i'th pointer points to the end of the i'th block + * (i.e. the start of the (i+1)'th block or past EOF). + * + * Note that size > 0, as a zero-sized file wouldn't ever + * have gotten here in the first place. + */ +static unsigned int do_compress(char *base, unsigned int offset, char const *name, char *uncompressed, unsigned int size) +{ + unsigned long original_size = size; + unsigned long original_offset = offset; + unsigned long new_size; + unsigned long blocks = (size - 1) / blksize + 1; + unsigned long curr = offset + 4 * blocks; + int change; + static int one = 0; + + total_blocks += blocks; + + do { + unsigned long len = 2 * blksize; + unsigned int input = size; + int err; + + if (input > blksize) + input = blksize; + size -= input; + if (!(opt_holes && is_zero (uncompressed, input))) { + /* BRCM Modification start */ + if (opt_gzip) { + err = compress2((Bytef *)base + curr, &len, (const Bytef *)uncompressed, input, Z_BEST_COMPRESSION); + if (err != Z_OK) { + die(MKFS_ERROR, 0, "compression error: %s", zError(err)); + } + } else { + unsigned lzma_algo; + unsigned lzma_dictsize; + unsigned lzma_fastbytes; + switch (opt_compression_level) { + case 1 : + lzma_algo = 1; + lzma_dictsize = 1 << 20; + lzma_fastbytes = 64; + break; + case 2 : + lzma_algo = 2; + lzma_dictsize = 1 << 22; + lzma_fastbytes = 128; + break; + case 3 : + lzma_algo = 2; + lzma_dictsize = 1 << 24; + lzma_fastbytes = 255; + break; + default : + die(MKFS_ERROR, 0, "Invalid LZMA compression level. Must be 1,2,3."); + } + len = 8*PAGE_CACHE_SIZE; +#if 0 + if (one < 2) { + printf("Input data, size =%d\n",input); + dumpHex((unsigned char *)uncompressed,input); + } +#endif + err = compress_lzma_7z((const unsigned char*) uncompressed, + (unsigned) input, + (unsigned char*) (base+curr), + (unsigned &) len, + lzma_algo, + lzma_dictsize, + lzma_fastbytes); + if (!err) { + /* this should NEVER happen */ + die(MKFS_ERROR, 0, "Internal error - LZMA compression failed.\n"); + } +#if 0 + if (one < 2 ) { + printf("Output data, size =%d\n",len); + dumpHex((unsigned char *)(base+curr),len); + } + if (one < 2) { + printf("Input data, size =%d\n",input); + dumpHex((unsigned char *)uncompressed,input); + } + + one++; +#endif +#if 0 + printf("verify...\n"); + unsigned char verify[4096]; + memset((void *)verify,0,4096); + err = decompress_lzma_7z((unsigned char*) (base+curr), len, verify, 23*1024*1024); + if (err != 0) { + die(MKFS_ERROR, 0, "Internal error - LZMA decompression failed.\n"); + } + printf("Verifying...input=%d,",input); + if (memcmp(verify,uncompressed,input) != 0) { + die(MKFS_ERROR, 0, "Internal error - LZMA compression/decompression not matching.\n"); + } +#endif + } + if (opt_verbose) { + printf("(%ld) ",len); + } else { + printf("."); + } + /* BRCM Modification end*/ + curr += len; + } + uncompressed += input; + + if (len > blksize*2) { + /* (I don't think this can happen with zlib.) */ + die(MKFS_ERROR, 0, "AIEEE: block \"compressed\" to > 2*blocklength (%ld)", len); + } + + *(u32 *) (base + offset) = curr; + if (swap_endian) fix_block_pointer((u32*)(base + offset)); + offset += 4; + } while (size); + + curr = (curr + 3) & ~3; + new_size = curr - original_offset; + /* TODO: Arguably, original_size in these 2 lines should be + st_blocks * 512. But if you say that then perhaps + administrative data should also be included in both. */ + change = new_size - original_size; + if (opt_verbose > 1) { + printf("%6.2f%% (%+d bytes)\t%s\n", + (change * 100) / (double) original_size, change, name); + } + + return curr; +} + + +/* + * Traverse the entry tree, writing data for every item that has + * non-null entry->path (i.e. every non-empty regfile) and non-null + * entry->uncompressed (i.e. every symlink). + */ +static unsigned int write_data(struct entry *entry, char *base, unsigned int offset) +{ + do { + if (entry->path || entry->uncompressed) { + if (entry->same) { + set_data_offset(entry, base, entry->same->offset); + entry->offset = entry->same->offset; + } + else { + set_data_offset(entry, base, offset); + entry->offset = offset; + map_entry(entry); + offset = do_compress(base, offset, (const char *)entry->name, (char *)entry->uncompressed, entry->size); + unmap_entry(entry); + } + } + else if (entry->child) + offset = write_data(entry->child, base, offset); + entry=entry->next; + } while (entry); + return offset; +} + +static unsigned int write_file(char *file, char *base, unsigned int offset) +{ + int fd; + char *buf; + + fd = open(file, O_RDONLY); + if (fd < 0) { + die(MKFS_ERROR, 1, "open failed: %s", file); + } + buf = (char *)mmap(NULL, image_length, PROT_READ, MAP_PRIVATE, fd, 0); + if (buf == MAP_FAILED) { + die(MKFS_ERROR, 1, "mmap failed"); + } + memcpy(base + offset, buf, image_length); + munmap(buf, image_length); + close (fd); + /* Pad up the image_length to a 4-byte boundary */ + while (image_length & 3) { + *(base + offset + image_length) = '\0'; + image_length++; + } + return (offset + image_length); +} + +int main(int argc, char **argv) +{ + struct stat st; /* used twice... */ + struct entry *root_entry; + char *rom_image; + ssize_t offset, written; + int fd; + /* initial guess (upper-bound) of required filesystem size */ + loff_t fslen_ub = sizeof(struct cramfs_super); + char const *dirname, *outfile; + u32 crc; + int c; /* for getopt */ + char *ep; /* for strtoul */ + + total_blocks = 0; + + if (argc) + progname = argv[0]; + + /* command line options */ + while ((c = getopt(argc, argv, "hEe:i:l:n:gprsvz")) != EOF) { + switch (c) { + case 'h': + usage(MKFS_OK); + case 'E': + opt_errors = 1; + break; + case 'e': + errno = 0; + opt_edition = strtoul(optarg, &ep, 10); + if (errno || optarg[0] == '\0' || *ep != '\0') + usage(MKFS_USAGE); + break; + case 'g': + opt_gzip = 1; + printf("Using gzip to compress instead of default lzma\n"); + break; + case 'i': + opt_image = optarg; + if (lstat(opt_image, &st) < 0) { + die(MKFS_ERROR, 1, "lstat failed: %s", opt_image); + } + image_length = st.st_size; /* may be padded later */ + fslen_ub += (image_length + 3); /* 3 is for padding */ + break; + case 'l': + errno = 0; + opt_compression_level = strtoul(optarg, &ep, 10); + if (errno || optarg[0] == '\0' || *ep != '\0') + usage(MKFS_USAGE); + break; + case 'n': + opt_name = optarg; + break; + case 'p': + opt_pad = PAD_SIZE; + fslen_ub += PAD_SIZE; + break; + case 'r': + swap_endian = 1; + printf("Swapping filesystem endian-ness\n"); + break; + case 's': + /* old option, ignored */ + break; + case 'v': + opt_verbose++; + break; + case 'z': + opt_holes = 1; + break; + } + } + + if ((argc - optind) != 2) + usage(MKFS_USAGE); + dirname = argv[optind]; + outfile = argv[optind + 1]; + + if (stat(dirname, &st) < 0) { + die(MKFS_USAGE, 1, "stat failed: %s", dirname); + } + fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) { + die(MKFS_USAGE, 1, "open failed: %s", outfile); + } + + root_entry = (struct entry *)calloc(1, sizeof(struct entry)); + if (!root_entry) { + die(MKFS_ERROR, 1, "calloc failed"); + } + root_entry->mode = st.st_mode; + root_entry->uid = st.st_uid; + root_entry->gid = st.st_gid; + + root_entry->size = parse_directory(root_entry, dirname, &root_entry->child, &fslen_ub); + + /* always allocate a multiple of blksize bytes because that's + what we're going to write later on */ + fslen_ub = ((fslen_ub - 1) | (blksize - 1)) + 1; + + if (fslen_ub > MAXFSLEN) { + fprintf(stderr, + "warning: estimate of required size (upper bound) is %LdMB, but maximum image size is %uMB, we might die prematurely\n", + fslen_ub >> 20, + MAXFSLEN >> 20); + fslen_ub = MAXFSLEN; + } + + /* find duplicate files. TODO: uses the most inefficient algorithm + possible. */ + eliminate_doubles(root_entry, root_entry); + + /* TODO: Why do we use a private/anonymous mapping here + followed by a write below, instead of just a shared mapping + and a couple of ftruncate calls? Is it just to save us + having to deal with removing the file afterwards? If we + really need this huge anonymous mapping, we ought to mmap + in smaller chunks, so that the user doesn't need nn MB of + RAM free. If the reason is to be able to write to + un-mmappable block devices, then we could try shared mmap + and revert to anonymous mmap if the shared mmap fails. */ + rom_image = (char *)mmap(NULL, fslen_ub?fslen_ub:1, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (rom_image == MAP_FAILED) { + die(MKFS_ERROR, 1, "mmap failed"); + } + + /* Skip the first opt_pad bytes for boot loader code */ + offset = opt_pad; + memset(rom_image, 0x00, opt_pad); + + /* Skip the superblock and come back to write it later. */ + offset += sizeof(struct cramfs_super); + + /* Insert a file image. */ + if (opt_image) { + printf("Including: %s\n", opt_image); + offset = write_file(opt_image, rom_image, offset); + } + + offset = write_directory_structure(root_entry->child, rom_image, offset); + printf("Directory data: %d bytes\n", offset); + + printf("Compressing directory and files...\n"); + offset = write_data(root_entry, rom_image, offset); + + /* We always write a multiple of blksize bytes, so that + losetup works. */ + offset = ((offset - 1) | (blksize - 1)) + 1; + printf("\nEverything: %d kilobytes\n", offset >> 10); + + /* Write the superblock now that we can fill in all of the fields. */ + write_superblock(root_entry, rom_image+opt_pad, offset); + printf("Super block: %d bytes\n", sizeof(struct cramfs_super)); + + /* Put the checksum in. */ + crc = crc32(0L, Z_NULL, 0); + crc = crc32(crc, (const Bytef *)(rom_image+opt_pad), (offset-opt_pad)); + ((struct cramfs_super *) (rom_image+opt_pad))->fsid.crc = crc; + printf("CRC: %x\n", crc); + + /* Check to make sure we allocated enough space. */ + if (fslen_ub < offset) { + die(MKFS_ERROR, 0, "not enough space allocated for ROM image (%Ld allocated, %d used)", fslen_ub, offset); + } + + written = write(fd, rom_image, offset); + if (written < 0) { + die(MKFS_ERROR, 1, "write failed"); + } + if (offset != written) { + die(MKFS_ERROR, 0, "ROM image write failed (wrote %d of %d bytes)", written, offset); + } + + /* (These warnings used to come at the start, but they scroll off the + screen too quickly.) */ + if (warn_namelen) + fprintf(stderr, /* bytes, not chars: think UTF-8. */ + "warning: filenames truncated to %d bytes (possibly less if multi-byte UTF-8)\n", + CRAMFS_MAXPATHLEN); + if (warn_skip) + fprintf(stderr, "warning: files were skipped due to errors\n"); + if (warn_size) + fprintf(stderr, + "warning: file sizes truncated to %luMB (minus 1 byte)\n", + 1L << (CRAMFS_SIZE_WIDTH - 20)); + if (warn_uid) /* (not possible with current Linux versions) */ + fprintf(stderr, + "warning: uids truncated to %u bits (this may be a security concern)\n", + CRAMFS_UID_WIDTH); + /* + if (warn_gid) + fprintf(stderr, + "warning: gids truncated to %u bits (this may be a security concern)\n", + CRAMFS_GID_WIDTH); + */ + if (warn_dev) + fprintf(stderr, + "WARNING: device numbers truncated to %u bits (this almost certainly means\n" + "that some device files will be wrong)\n", + CRAMFS_OFFSET_WIDTH); + if (opt_errors && + (warn_namelen||warn_skip||warn_size||warn_uid||warn_gid||warn_dev)) + exit(MKFS_ERROR); + + exit(MKFS_OK); +} + +/* + * Local variables: + * c-file-style: "linux" + * End: + */ diff --git a/hostTools/createimg.c b/hostTools/createimg.c new file mode 100644 index 0000000..1ec5e0e --- /dev/null +++ b/hostTools/createimg.c @@ -0,0 +1,481 @@ +/* +<:copyright-broadcom + + Copyright (c) 2002 Broadcom Corporation + All Rights Reserved + No portions of this material may be reproduced in any form without the + written permission of: + Broadcom Corporation + 16215 Alton Parkway + Irvine, California 92619 + All information contained in this document is Broadcom Corporation + company private, proprietary, and trade secret. + +:> +*/ +/*************************************************************************** + * File Name : createimg.c + * + * Description: This program creates a complete flash image for a BCM963xx + * board from an input file that is created by the + * bcmImageBuilder utility. + * + * Build : $ gcc -o createimg createimg.c boardparms.c + * + * Example : createimg -b 96348GW2 -n 4 -m 02:10:18:18:12:01 -f 4 + * -i bcm96348GW_cfe_fs_kernel -o bcm96348GW_cfe_fs_kernel.img + * where: + * -b = board id + * -n = number of mac address + * -m = base mac address + * -s = sdram size in MB + * -i = input file name + * -o = output file name + * + * Updates : 06/05/2002 lat. Created. + * 08/18/2002 seanl. Move the image tag to the front of the kernel (was in nvram). + * 12/08/2002 seanl. Add support for 2MG 96345R and 4MG 96345GW flash image + * + ***************************************************************************/ + +/* Includes. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef unsigned char byte; +typedef unsigned int UINT32; +#define BCMTAG_EXE_USE +#include "bcmTag.h" +#include "boardparms.h" +#define MAX_MAC_STR_LEN 19 // mac address string 18+1 in regular format - 02:18:10:11:aa:bb + +#include "board.h" + + +static struct option longopts[] = +{ + { "boardid", 1, 0, 'b'}, + { "numbermac", 1, 0, 'n'}, + { "macaddr", 1, 0, 'm'}, + { "sdramsize", 1, 0, 's'}, + { "inputfile", 1, 0, 'i'}, + { "outputfile", 1, 0, 'o'}, + { "help", 0, 0, 'h'}, + { "version", 0, 0, 'v'}, + { 0, 0, 0, 0 } +}; + +static char version[] = "Broadcom Flash Image Creator version 0.3"; + +static char usage[] = +"Usage:\t%s [-Ibhvnmpsfio] [--help] [--version]\n" +"\texample:\n" +"\tcreateimg -b 96348GW2 -n 12 -m 02:10:18:18:12:01 -f 4 -e y " +"-i cfe_fs_kernel -o cfe_fs_kernel.img\n" +"\twhere:\n" +"\t[-b] board id name\n" +"\t[-n] number of mac address\n" +"\t[-m] base mac address\n" +"\t[-s] memory size in MB\n" +"\t[-i] input image name\n" +"\t[-o] ouput image name\n"; + +static char *progname; + +/*************************************************************************** +// Function Name: getCrc32 +// Description : caculate the CRC 32 of the given data. +// Parameters : pdata - array of data. +// size - number of input data bytes. +// crc - either CRC32_INIT_VALUE or previous return value. +// Returns : crc. +****************************************************************************/ +UINT32 getCrc32(byte *pdata, UINT32 size, UINT32 crc) +{ + while (size-- > 0) + crc = (crc >> 8) ^ Crc32_table[(crc ^ *pdata++) & 0xff]; + + return crc; +} + +/*************************************************************************** +// Function Name: parsexdigit +// Description : parse hex digit +// Parameters : input char +// Returns : hex int +****************************************************************************/ +static int parsexdigit(char str) +{ + int digit; + + if ((str >= '0') && (str <= '9')) + digit = str - '0'; + else if ((str >= 'a') && (str <= 'f')) + digit = str - 'a' + 10; + else if ((str >= 'A') && (str <= 'F')) + digit = str - 'A' + 10; + else + return -1; + + return digit; +} + +/*************************************************************************** +// Function Name: parsehwaddr +// Description : convert in = "255.255.255.0" to fffffff00 +// Parameters : str = "255.255.255.0" ; +// Return : if not -1, hwaddr = fffffff00 +****************************************************************************/ +int parsehwaddr(unsigned char *str, unsigned char *hwaddr) +{ + int digit1,digit2; + int idx = 6; + + if (strlen(str) != MAX_MAC_STR_LEN-2) + return -1; + if (*(str+2) != ':' || *(str+5) != ':' || *(str+8) != ':' || *(str+11) != ':' || *(str+14) != ':') + return -1; + + while (*str && (idx > 0)) { + digit1 = parsexdigit(*str); + if (digit1 < 0) + return -1; + str++; + if (!*str) + return -1; + + if (*str == ':') { + digit2 = digit1; + digit1 = 0; + } + else { + digit2 = parsexdigit(*str); + if (digit2 < 0) + return -1; + str++; + } + + *hwaddr++ = (digit1 << 4) | digit2; + idx--; + + if (*str == ':') + str++; + } + return 0; +} + +/*************************************************************************** + * Function Name: fillInOutputBuffer + * Description : Fills the output buffer with the flash image contents. + * Returns : len of output buffer - OK, -1 - failed. + ***************************************************************************/ +int fillInOutputBuffer(int numOfMac, unsigned char *ucaBaseMac, + int sdramType, char *szBoardId, unsigned long ulOutputSize, char *pszIn, + char *pszOut) +{ + int nLen; + PFILE_TAG pTag = (PFILE_TAG) pszIn; + unsigned long ulCfeOffset = 0; + unsigned long ulFsOffset = 0; + unsigned long ulKernelOffset = 0; + unsigned long ulCfeLen = 0; + unsigned long ulFsLen = 0; + unsigned long ulKernelLen = 0; + NVRAM_DATA nvramData; + UINT32 crc = CRC32_INIT_VALUE; + unsigned long spaceLeft = 0; + unsigned long ulCmtThread = 0; + + ulCfeOffset = strtoul(pTag->cfeAddress, NULL, 10) - IMAGE_BASE; + ulFsOffset = strtoul(pTag->rootfsAddress, NULL, 10) - IMAGE_BASE; + ulKernelOffset = strtoul(pTag->kernelAddress, NULL, 10) - IMAGE_BASE; + ulCfeLen = strtoul(pTag->cfeLen, NULL, 10); + ulFsLen = strtoul(pTag->rootfsLen, NULL, 10); + ulKernelLen = strtoul(pTag->kernelLen, NULL, 10); + + printf("\tImage components offsets\n"); + printf("\tcfe offset : 0x%8.8lx -- Length: %d\n", + ulCfeOffset + IMAGE_BASE, ulCfeLen); + printf("\tfile tag offset : 0x%8.8lx -- Length: %d\n", + ulFsOffset - TAG_LEN + IMAGE_BASE, TAG_LEN); + printf("\trootfs offset : 0x%8.8lx -- Length: %d\n", + ulFsOffset + IMAGE_BASE, ulFsLen); + printf("\tkernel offset : 0x%8.8lx -- Length: %d\n\n", + ulKernelOffset + IMAGE_BASE, ulKernelLen); + + nLen = ulKernelOffset + ulKernelLen + FLASH_RESERVED_AT_END; + + if( nLen > (int) ulOutputSize ) + { + printf( "\nERROR: The image size is greater than the output buffer " + "allocated by this program.\n" ); + return( -1 ); + } + + printf( "\tThe size of the entire flash image is %d bytes.\n", nLen ); + if( nLen < 1 * 1024 * 1024 ) + printf( "\tA 1 MB or greater flash part is needed.\n\n" ); + else if( nLen < 2 * 1024 * 1024 ) + printf( "\tA 2 MB or greater flash part is needed.\n\n" ); + else if( nLen < 4 * 1024 * 1024 ) + printf( "\tA 4 MB or greater flash part is needed.\n\n" ); + else if( nLen < 8 * 1024 * 1024 ) + printf( "\tA 8 MB or greater flash part is needed.\n\n" ); + + printf( "\tThe flash space remaining for a 2 MB flash part: %ld bytes.\n", 2*ONEK*ONEK - nLen); + printf( "\tThe flash space remaining for a 4 MB flash part: %ld bytes.\n\n", 4*ONEK*ONEK - nLen); + + memcpy( pszOut + ulCfeOffset, pszIn + TAG_LEN, ulCfeLen ); + // the tag is before fs now... + memcpy( pszOut + ulFsOffset - TAG_LEN, pszIn, TAG_LEN ); + memcpy( pszOut + ulFsOffset, pszIn + TAG_LEN + ulCfeLen, ulFsLen ); + memcpy( pszOut + ulKernelOffset, pszIn + TAG_LEN + ulCfeLen + ulFsLen, ulKernelLen ); + + // add board thread number in the CFE space + BpGetCMTThread(&ulCmtThread); + memcpy(pszOut + ulCfeOffset + THREAD_NUM_ADDRESS_OFFSET, (char*)&ulCmtThread, sizeof(int)); + + // add memory size type in the CFE space + sdramType = htonl(sdramType); + memcpy(pszOut + ulCfeOffset + SDRAM_TYPE_ADDRESS_OFFSET, (char*)&sdramType, sizeof(int)); + + // create nvram data. No default bootline. CFE will create one on the first boot + memset((char *)&nvramData, 0, sizeof(NVRAM_DATA)); + nvramData.ulVersion = htonl(NVRAM_VERSION_NUMBER); + strncpy(nvramData.szBoardId, szBoardId, NVRAM_BOARD_ID_STRING_LEN); + nvramData.ulNumMacAddrs = htonl(numOfMac); + memcpy(nvramData.ucaBaseMacAddr, ucaBaseMac, NVRAM_MAC_ADDRESS_LEN); + nvramData.ulCheckSum = 0; + crc = getCrc32((char *)&nvramData, (UINT32) sizeof(NVRAM_DATA), crc); + nvramData.ulCheckSum = htonl(crc); + + memcpy(pszOut + NVRAM_DATA_OFFSET, (char *)&nvramData, sizeof(NVRAM_DATA)); + + return nLen; + +} /*fillInOutputBuffer */ + + +/*************************************************************************** + * Function Name: verifyTag + * Description : check on the input file by tag verification + * Returns : 0 - ok, -1 bad input image file + ***************************************************************************/ +int verifyTag(PFILE_TAG pTag) +{ + UINT32 crc; + int status = 0; + + // check tag validate token first + crc = CRC32_INIT_VALUE; + crc = getCrc32((char*) pTag, (UINT32)TAG_LEN-TOKEN_LEN, crc); + crc = htonl(crc); + + if (crc != (UINT32)(*(UINT32*)(pTag->tagValidationToken))) + { + printf("Illegal image ! Tag crc failed.\n"); + status = -1; + } + + return status; +} + +/*************************************************************************** + * Function Name: createImage + * Description : Creates the flash image file. + * Returns : None. + ***************************************************************************/ +void createImage(int numOfMac, unsigned char *ucaBaseMac, + int sdramType, char *szBoardId, char *inputFile, + char *outputFile) +{ + FILE *hInput = NULL; + struct stat StatBuf; + char *pszIn = NULL, *pszOut = NULL; + unsigned long ulOutputSize = 4 * 1024 * 1024; + + if (stat(inputFile, &StatBuf ) == 0 && (hInput = fopen(inputFile, "rb" )) == NULL) + { + printf( "createimg: Error opening input file %s.\n\n", inputFile); + return; + } + + /* Allocate buffers to read the entire input file and to write the + * entire flash. + */ + pszIn = (char *) malloc(StatBuf.st_size); + pszOut = (char *) malloc(ulOutputSize); + if (!pszIn || !pszOut) + { + printf( "Memory allocation error, in=0x%8.8lx, out=0x%8.8lx.\n\n", + (unsigned long) pszIn, (unsigned long) pszOut ); + fclose( hInput ); + return; + } + + /* Read the input file into memory. */ + if (fread( pszIn, sizeof(char), StatBuf.st_size, hInput ) == StatBuf.st_size) + { + /* Verify that FILE_TAG fields are OK. */ + if( verifyTag((PFILE_TAG)pszIn) == 0) + { + FILE *hOutput; + int nLen; + + /* Fill in the output buffer with the flash image. */ + memset( (unsigned char *) pszOut, 0xff, ulOutputSize ); + if ((nLen = fillInOutputBuffer(numOfMac, ucaBaseMac, + sdramType, szBoardId, ulOutputSize, pszIn, pszOut)) > 0) + { + /* Open output file. */ + if ((hOutput = fopen(outputFile, "w+" )) != NULL) + { + /* Write the output buffer to the output file. */ + if (fwrite(pszOut, sizeof(char), nLen, hOutput) == nLen) + printf( "\t%s flash image file is\n\tsuccessfully created.\n\n", outputFile); + else + printf( "createimg: Error writing to output file.\n\n" ); + fclose( hOutput ); + } + else + printf ("createimg: Error opening output file %s.\n\n", outputFile); + } + } + } + + if( pszIn ) + free( pszIn ); + + if( pszOut ) + free( pszOut ); + + fclose( hInput ); + +} /* createImage */ + + + +/************************************************************* + * Function Name: main + * Description : Program entry point that parses command line parameters + * and calls a function to create the image. + * Returns : 0 + ***************************************************************************/ +int main (int argc, char **argv) +{ + int optc; + int option_index = 0; + int h = 0, v = 0, lose = 0; + char inputFile[256], outputFile[256], macString[256]; + char szBoardId[BP_BOARD_ID_LEN] = ""; + int numOfMac = 0, sdramSize = 0, enetExtPhy = 0; + unsigned long sdramType = 0; + unsigned char ucaBaseMacAddr[NVRAM_MAC_ADDRESS_LEN]; + + /* Parse command line options. */ + progname = argv[0]; + inputFile[0] = outputFile[0] = macString[0] = '\0'; + + while ((optc = getopt_long (argc, argv, "hvp:n:m:s:b:i:o:", longopts, &option_index)) != EOF) + { + switch (optc) + { + case 'v': + v = 1; + break; + case 'h': + h = 1; + break; + case 'n': + numOfMac = strtoul(optarg, NULL, 10); + break; + case 'm': + strcpy(macString, optarg); + break; + case 's': + sdramSize = strtoul(optarg, NULL, 10); + break; + case 'i': + strcpy(inputFile, optarg); + break; + case 'o': + strcpy(outputFile, optarg); + break; + case 'b': + strcpy(szBoardId, optarg); + break; + default: + lose = 1; + break; + } + } + if (v) + { + /* Print version number. */ + fprintf (stderr, "%s\n", version); + if (!h) + exit(0); + } + + if (h) + { + /* Print help info and exit. */ + fprintf(stderr, "%s\n", version); + fprintf(stderr, usage, progname); + exit(0); + } + + if (lose || optind < argc || argc < 2 || + inputFile[0] == '\0' || outputFile[0] == '\0' || + (parsehwaddr(macString, ucaBaseMacAddr) == -1)) + { + fprintf (stderr, usage, progname); + exit(1); + } + + if (sdramSize == 0 && szBoardId[0] != '\0') + { + BpSetBoardId( szBoardId ); + if( BpGetSdramSize( (unsigned long *) &sdramType ) == BP_SUCCESS ) + { + switch (sdramType) + { + case BP_MEMORY_8MB_1_CHIP: + sdramSize = 8; + break; + case BP_MEMORY_16MB_1_CHIP: + case BP_MEMORY_16MB_2_CHIP: + sdramSize = 16; + break; + case BP_MEMORY_32MB_1_CHIP: + case BP_MEMORY_32MB_2_CHIP: + sdramSize = 32; + break; + case BP_MEMORY_64MB_2_CHIP: + sdramSize = 64; + break; + } + } + } + + printf("createimg: Creating image with the following inputs:\n" ); + printf("\tBoard id : %s\n", szBoardId); + printf("\tNumber of Mac Addresses : %d\n", numOfMac); + printf("\tBase Mac Address: : %s\n", macString); + printf("\tMemory size : %dMb\n", sdramSize); + printf("\tInput File Name : %s\n", inputFile); + printf("\tOutput File Name : %s\n\n", outputFile); + + createImage(numOfMac, ucaBaseMacAddr, sdramType, szBoardId, + inputFile, outputFile); + + return(0); + +} /* main */ + diff --git a/hostTools/lzma/compress/7z.h b/hostTools/lzma/compress/7z.h new file mode 100644 index 0000000..241a2ea --- /dev/null +++ b/hostTools/lzma/compress/7z.h @@ -0,0 +1,12 @@ +#ifndef __7Z_H +#define __7Z_H + +bool compress_deflate_7z(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned& out_size, unsigned num_passes, unsigned num_fast_bytes) throw (); +bool decompress_deflate_7z(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned out_size) throw (); +bool compress_rfc1950_7z(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned& out_size, unsigned num_passes, unsigned num_fast_bytes) throw (); + +bool compress_lzma_7z(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned& out_size, unsigned algo, unsigned dictionary_size, unsigned num_fast_bytes) throw (); +bool decompress_lzma_7z(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned out_size) throw (); + +#endif + diff --git a/hostTools/lzma/compress/7zapi.cpp b/hostTools/lzma/compress/7zapi.cpp new file mode 100644 index 0000000..cc6c908 --- /dev/null +++ b/hostTools/lzma/compress/7zapi.cpp @@ -0,0 +1,19 @@ +#include "7z.h" +/********************** APIs Definitions ************************************/ + +extern "C" { + +int compress_lzma_7zapi(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned *out_size, unsigned algo, unsigned dictionary_size, unsigned num_fast_bytes); + +} + +int compress_lzma_7zapi(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned *out_size, unsigned algo, unsigned dictionary_size, unsigned num_fast_bytes) +{ + bool ret; + //unsigned outsize = *out_size; + + ret = compress_lzma_7z(in_data, in_size, out_data, *out_size, algo, dictionary_size, num_fast_bytes); + + return (int)ret; +} + diff --git a/hostTools/lzma/compress/7zapi.h b/hostTools/lzma/compress/7zapi.h new file mode 100644 index 0000000..e2d6ac3 --- /dev/null +++ b/hostTools/lzma/compress/7zapi.h @@ -0,0 +1,16 @@ +#ifndef __7ZAPI_H__ +#define __7ZAPI_H__ + +#if defined(__cplusplus) +extern "C" { +#endif + +extern int compress_lzma_7zapi(const unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned *out_size, unsigned algo, unsigned dictionary_size, unsigned num_fast_bytes); + + +#if defined(__cplusplus) +} +#endif + + +#endif diff --git a/hostTools/lzma/compress/7zlzma.cpp b/hostTools/lzma/compress/7zlzma.cpp new file mode 100644 index 0000000..35468c1 --- /dev/null +++ b/hostTools/lzma/compress/7zlzma.cpp @@ -0,0 +1,79 @@ +#include "7z.h" + +#include "LZMAEncoder.h" +#include "LZMADecoder.h" + +bool compress_lzma_7z(const unsigned char* in_data, + unsigned in_size, + unsigned char* out_data, + unsigned& out_size, + unsigned algo, + unsigned dictionary_size, + unsigned num_fast_bytes) throw () { + try { + NCompress::NLZMA::CEncoder cc; + + // reduce the dictionary size if the file is small + while (dictionary_size > 8 && dictionary_size / 2 >= out_size) + dictionary_size /= 2; + + if (cc.SetDictionarySize(dictionary_size) != S_OK) + return false; + + if (cc.SetEncoderNumFastBytes(num_fast_bytes) != S_OK) + return false; + + if (cc.SetEncoderAlgorithm(algo) != S_OK) + return false; + + ISequentialInStream in(reinterpret_cast(in_data), in_size); + ISequentialOutStream out(reinterpret_cast(out_data), out_size); + + UINT64 in_size_l = in_size; + + if (cc.WriteCoderProperties(&out) != S_OK) + return false; + + if (cc.Code(&in, &out, &in_size_l) != S_OK) + return false; + + out_size = out.size_get(); + + if (out.overflow_get()) + return false; + + return true; + } catch (...) { + return false; + } +} +#if 0 +bool decompress_lzma_7z(const unsigned char* in_data, + unsigned in_size, + unsigned char* out_data, + unsigned out_size) throw () { + try { + NCompress::NLZMA::CDecoder cc; + + ISequentialInStream in(reinterpret_cast(in_data), in_size); + ISequentialOutStream out(reinterpret_cast(out_data), out_size); + + UINT64 in_size_l = in_size; + UINT64 out_size_l = out_size; + + if (cc.ReadCoderProperties(&in) != S_OK) + return false; + + if (cc.Code(&in, &out, &in_size_l, &out_size_l) != S_OK) + return false; + + if (out.size_get() != out_size || out.overflow_get()) + return false; + + return true; + } catch (...) { + return false; + } +} +#endif + diff --git a/hostTools/lzma/compress/AriBitCoder.cpp b/hostTools/lzma/compress/AriBitCoder.cpp new file mode 100644 index 0000000..ade1888 --- /dev/null +++ b/hostTools/lzma/compress/AriBitCoder.cpp @@ -0,0 +1,21 @@ +#include "AriBitCoder.h" +#include "AriPrice.h" + +#include + +namespace NCompression { +namespace NArithmetic { + +static const double kDummyMultMid = (1.0 / kBitPrice) / 2; + +CPriceTables::CPriceTables() +{ + double aLn2 = log(2); + double aLnAll = log(kBitModelTotal >> kNumMoveReducingBits); + for(UINT32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++) + m_StatePrices[i] = UINT32((fabs(aLnAll - log(i)) / aLn2 + kDummyMultMid) * kBitPrice); +} + +CPriceTables g_PriceTables; + +}} diff --git a/hostTools/lzma/compress/AriBitCoder.h b/hostTools/lzma/compress/AriBitCoder.h new file mode 100644 index 0000000..76328cd --- /dev/null +++ b/hostTools/lzma/compress/AriBitCoder.h @@ -0,0 +1,101 @@ +#ifndef __COMPRESSION_BITCODER_H +#define __COMPRESSION_BITCODER_H + +#include "RangeCoder.h" + +namespace NCompression { +namespace NArithmetic { + +const int kNumBitModelTotalBits = 11; +const UINT32 kBitModelTotal = (1 << kNumBitModelTotalBits); + +const int kNumMoveReducingBits = 2; + + +class CPriceTables +{ +public: + UINT32 m_StatePrices[kBitModelTotal >> kNumMoveReducingBits]; + CPriceTables(); +}; + +extern CPriceTables g_PriceTables; + + +///////////////////////////// +// CBitModel + +template +class CBitModel +{ +public: + UINT32 m_Probability; + void UpdateModel(UINT32 aSymbol) + { + /* + m_Probability -= (m_Probability + ((aSymbol - 1) & ((1 << aNumMoveBits) - 1))) >> aNumMoveBits; + m_Probability += (1 - aSymbol) << (kNumBitModelTotalBits - aNumMoveBits); + */ + if (aSymbol == 0) + m_Probability += (kBitModelTotal - m_Probability) >> aNumMoveBits; + else + m_Probability -= (m_Probability) >> aNumMoveBits; + } +public: + void Init() { m_Probability = kBitModelTotal / 2; } +}; + +template +class CBitEncoder: public CBitModel +{ +public: + void Encode(CRangeEncoder *aRangeEncoder, UINT32 aSymbol) + { + aRangeEncoder->EncodeBit(this->m_Probability, kNumBitModelTotalBits, aSymbol); + this->UpdateModel(aSymbol); + } + UINT32 GetPrice(UINT32 aSymbol) const + { + return g_PriceTables.m_StatePrices[ + (((this->m_Probability - aSymbol) ^ ((-(int)aSymbol))) & (kBitModelTotal - 1)) >> kNumMoveReducingBits]; + } +}; + + +template +class CBitDecoder: public CBitModel +{ +public: + UINT32 Decode(CRangeDecoder *aRangeDecoder) + { + UINT32 aNewBound = (aRangeDecoder->m_Range >> kNumBitModelTotalBits) * this->m_Probability; + if (aRangeDecoder->m_Code < aNewBound) + { + aRangeDecoder->m_Range = aNewBound; + this->m_Probability += (kBitModelTotal - this->m_Probability) >> aNumMoveBits; + if (aRangeDecoder->m_Range < kTopValue) + { + aRangeDecoder->m_Code = (aRangeDecoder->m_Code << 8) | aRangeDecoder->m_Stream.ReadByte(); + aRangeDecoder->m_Range <<= 8; + } + return 0; + } + else + { + aRangeDecoder->m_Range -= aNewBound; + aRangeDecoder->m_Code -= aNewBound; + this->m_Probability -= (this->m_Probability) >> aNumMoveBits; + if (aRangeDecoder->m_Range < kTopValue) + { + aRangeDecoder->m_Code = (aRangeDecoder->m_Code << 8) | aRangeDecoder->m_Stream.ReadByte(); + aRangeDecoder->m_Range <<= 8; + } + return 1; + } + } +}; + +}} + + +#endif diff --git a/hostTools/lzma/compress/AriConst.h b/hostTools/lzma/compress/AriConst.h new file mode 100644 index 0000000..1e429dd --- /dev/null +++ b/hostTools/lzma/compress/AriConst.h @@ -0,0 +1,17 @@ +#ifndef __ARICONST_H +#define __ARICONST_H + +#include "AriBitCoder.h" + +typedef NCompression::NArithmetic::CRangeEncoder CMyRangeEncoder; +typedef NCompression::NArithmetic::CRangeDecoder CMyRangeDecoder; + +template class CMyBitEncoder: + public NCompression::NArithmetic::CBitEncoder {}; +template class CMyBitDecoder: + public NCompression::NArithmetic::CBitDecoder {}; + +#endif + + + diff --git a/hostTools/lzma/compress/AriPrice.h b/hostTools/lzma/compress/AriPrice.h new file mode 100644 index 0000000..ccc398e --- /dev/null +++ b/hostTools/lzma/compress/AriPrice.h @@ -0,0 +1,12 @@ +#ifndef __COMPRESSION_ARIPRICE_H +#define __COMPRESSION_ARIPRICE_H + +namespace NCompression { +namespace NArithmetic { + +const UINT32 kNumBitPriceShiftBits = 6; +const UINT32 kBitPrice = 1 << kNumBitPriceShiftBits; + +}} + +#endif diff --git a/hostTools/lzma/compress/BinTree.h b/hostTools/lzma/compress/BinTree.h new file mode 100644 index 0000000..29c273d --- /dev/null +++ b/hostTools/lzma/compress/BinTree.h @@ -0,0 +1,64 @@ +#include "Portable.h" +#include "WindowIn.h" + +namespace BT_NAMESPACE { + +typedef UINT32 CIndex; +const UINT32 kMaxValForNormalize = (UINT32(1) << 31) - 1; + +struct CPair +{ + CIndex Left; + CIndex Right; +}; + +class CInTree: public NStream::NWindow::CIn +{ + UINT32 m_HistorySize; + UINT32 m_MatchMaxLen; + + CIndex *m_Hash; + + #ifdef HASH_ARRAY_2 + CIndex *m_Hash2; + #ifdef HASH_ARRAY_3 + CIndex *m_Hash3; + #endif + #endif + + CPair *m_Son; + CPair *m_Base; + + UINT32 m_CutValue; + + void NormalizeLinks(CIndex *anArray, UINT32 aNumItems, UINT32 aSubValue); + void Normalize(); + void FreeMemory(); + +protected: + virtual void AfterMoveBlock(); +public: + CInTree(); + ~CInTree(); + HRESULT Create(UINT32 aSizeHistory, UINT32 aKeepAddBufferBefore, UINT32 aMatchMaxLen, + UINT32 aKeepAddBufferAfter, UINT32 _dwSizeReserv = (1<<17)); + HRESULT Init(ISequentialInStream *aStream); + void SetCutValue(UINT32 aCutValue) { m_CutValue = aCutValue; } + UINT32 GetLongestMatch(UINT32 *aDistances); + void DummyLongestMatch(); + HRESULT MovePos() + { + RETURN_IF_NOT_S_OK(CIn::MovePos()); + if (m_Pos == kMaxValForNormalize) + Normalize(); + return S_OK; + } + void ReduceOffsets(UINT32 aSubValue) + { + CIn::ReduceOffsets(aSubValue); + m_Son += aSubValue; + } +}; + +} + diff --git a/hostTools/lzma/compress/BinTree2.h b/hostTools/lzma/compress/BinTree2.h new file mode 100644 index 0000000..7953d06 --- /dev/null +++ b/hostTools/lzma/compress/BinTree2.h @@ -0,0 +1,13 @@ +#ifndef __BINTREE2__H +#define __BINTREE2__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT2 + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT2 + +#include "BinTreeMF.h" + +#endif + diff --git a/hostTools/lzma/compress/BinTree2Main.h b/hostTools/lzma/compress/BinTree2Main.h new file mode 100644 index 0000000..385166e --- /dev/null +++ b/hostTools/lzma/compress/BinTree2Main.h @@ -0,0 +1,13 @@ +#ifndef __BINTREE2MAIN__H +#define __BINTREE2MAIN__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT2 + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT2 + +#include "BinTreeMFMain.h" + +#endif + diff --git a/hostTools/lzma/compress/BinTree3.h b/hostTools/lzma/compress/BinTree3.h new file mode 100644 index 0000000..320a619 --- /dev/null +++ b/hostTools/lzma/compress/BinTree3.h @@ -0,0 +1,17 @@ +#ifndef __BINTREE3__H +#define __BINTREE3__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT3 + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT3 + +#define HASH_ARRAY_2 + +#include "BinTreeMF.h" + +#undef HASH_ARRAY_2 + +#endif + diff --git a/hostTools/lzma/compress/BinTree3Main.h b/hostTools/lzma/compress/BinTree3Main.h new file mode 100644 index 0000000..6331084 --- /dev/null +++ b/hostTools/lzma/compress/BinTree3Main.h @@ -0,0 +1,17 @@ +#ifndef __BINTREE3MAIN__H +#define __BINTREE3MAIN__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT3 + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT3 + +#define HASH_ARRAY_2 + +#include "BinTreeMFMain.h" + +#undef HASH_ARRAY_2 + +#endif + diff --git a/hostTools/lzma/compress/BinTree3Z.h b/hostTools/lzma/compress/BinTree3Z.h new file mode 100644 index 0000000..cf65c7f --- /dev/null +++ b/hostTools/lzma/compress/BinTree3Z.h @@ -0,0 +1,14 @@ +#ifndef __BINTREE3Z__H +#define __BINTREE3Z__H + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT3Z + +#define HASH_ZIP + +#include "BinTree.h" + +#undef HASH_ZIP + +#endif + diff --git a/hostTools/lzma/compress/BinTree3ZMain.h b/hostTools/lzma/compress/BinTree3ZMain.h new file mode 100644 index 0000000..eb9e080 --- /dev/null +++ b/hostTools/lzma/compress/BinTree3ZMain.h @@ -0,0 +1,14 @@ +#ifndef __BINTREE3ZMAIN__H +#define __BINTREE3ZMAIN__H + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT3Z + +#define HASH_ZIP + +#include "BinTreeMain.h" + +#undef HASH_ZIP + +#endif + diff --git a/hostTools/lzma/compress/BinTree4.h b/hostTools/lzma/compress/BinTree4.h new file mode 100644 index 0000000..6af6978 --- /dev/null +++ b/hostTools/lzma/compress/BinTree4.h @@ -0,0 +1,19 @@ +#ifndef __BINTREE4__H +#define __BINTREE4__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT4 + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT4 + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 + +#include "BinTreeMF.h" + +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#endif + diff --git a/hostTools/lzma/compress/BinTree4Main.h b/hostTools/lzma/compress/BinTree4Main.h new file mode 100644 index 0000000..f8e2e22 --- /dev/null +++ b/hostTools/lzma/compress/BinTree4Main.h @@ -0,0 +1,19 @@ +#ifndef __BINTREEMAIN4__H +#define __BINTREEMAIN4__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT4 + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT4 + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 + +#include "BinTreeMFMain.h" + +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#endif + diff --git a/hostTools/lzma/compress/BinTree4b.h b/hostTools/lzma/compress/BinTree4b.h new file mode 100644 index 0000000..7b68754 --- /dev/null +++ b/hostTools/lzma/compress/BinTree4b.h @@ -0,0 +1,21 @@ +#ifndef __BINTREE4B__H +#define __BINTREE4B__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT4b + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT4b + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 +#define HASH_BIG + +#include "BinTreeMF.h" + +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 +#undef HASH_BIG + +#endif + diff --git a/hostTools/lzma/compress/BinTree4bMain.h b/hostTools/lzma/compress/BinTree4bMain.h new file mode 100644 index 0000000..b707ca1 --- /dev/null +++ b/hostTools/lzma/compress/BinTree4bMain.h @@ -0,0 +1,21 @@ +#ifndef __BINTREE4BMAIN__H +#define __BINTREE4BMAIN__H + +#undef BT_CLSID +#define BT_CLSID CLSID_CMatchFinderBT4b + +#undef BT_NAMESPACE +#define BT_NAMESPACE NBT4b + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 +#define HASH_BIG + +#include "BinTreeMFMain.h" + +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 +#undef HASH_BIG + +#endif + diff --git a/hostTools/lzma/compress/BinTreeMF.h b/hostTools/lzma/compress/BinTreeMF.h new file mode 100644 index 0000000..8019ab5 --- /dev/null +++ b/hostTools/lzma/compress/BinTreeMF.h @@ -0,0 +1,19 @@ +// #ifndef __BINTREEMF_H +// #define __BINTREEMF_H + +#include "BinTree.h" + +namespace BT_NAMESPACE { + +class CMatchFinderBinTree : public CInTree +{ +public: + HRESULT Create(UINT32 aSizeHistory, + UINT32 aKeepAddBufferBefore, UINT32 aMatchMaxLen, + UINT32 aKeepAddBufferAfter); +}; + +} + +// #endif + diff --git a/hostTools/lzma/compress/BinTreeMFMain.h b/hostTools/lzma/compress/BinTreeMFMain.h new file mode 100644 index 0000000..5438093 --- /dev/null +++ b/hostTools/lzma/compress/BinTreeMFMain.h @@ -0,0 +1,30 @@ +#include "BinTreeMain.h" + +namespace BT_NAMESPACE { + +HRESULT CMatchFinderBinTree::Create(UINT32 aSizeHistory, + UINT32 aKeepAddBufferBefore, UINT32 aMatchMaxLen, + UINT32 aKeepAddBufferAfter) +{ + const UINT32 kAlignMask = (1 << 16) - 1; + UINT32 aWindowReservSize = aSizeHistory / 2; + aWindowReservSize += kAlignMask; + aWindowReservSize &= ~(kAlignMask); + + const int kMinDictSize = (1 << 19); + if (aWindowReservSize < kMinDictSize) + aWindowReservSize = kMinDictSize; + aWindowReservSize += 256; + + try + { + return CInTree::Create(aSizeHistory, aKeepAddBufferBefore, aMatchMaxLen, + aKeepAddBufferAfter, aWindowReservSize); + } + catch(...) + { + return E_OUTOFMEMORY; + } +} + +} diff --git a/hostTools/lzma/compress/BinTreeMain.h b/hostTools/lzma/compress/BinTreeMain.h new file mode 100644 index 0000000..f452a94 --- /dev/null +++ b/hostTools/lzma/compress/BinTreeMain.h @@ -0,0 +1,502 @@ +#include "CRC.h" + +namespace BT_NAMESPACE { + +#ifdef HASH_ARRAY_2 + static const UINT32 kHash2Size = 1 << 10; + #ifdef HASH_ARRAY_3 + static const UINT32 kNumHashDirectBytes = 0; + static const UINT32 kNumHashBytes = 4; + static const UINT32 kHash3Size = 1 << 18; + #ifdef HASH_BIG + static const UINT32 kHashSize = 1 << 23; + #else + static const UINT32 kHashSize = 1 << 20; + #endif + #else + static const UINT32 kNumHashDirectBytes = 3; + static const UINT32 kNumHashBytes = 3; + static const UINT32 kHashSize = 1 << (8 * kNumHashBytes); + #endif +#else + #ifdef HASH_ZIP + static const UINT32 kNumHashDirectBytes = 0; + static const UINT32 kNumHashBytes = 3; + static const UINT32 kHashSize = 1 << 16; + #else + static const UINT32 kNumHashDirectBytes = 2; + static const UINT32 kNumHashBytes = 2; + static const UINT32 kHashSize = 1 << (8 * kNumHashBytes); + #endif +#endif + +CInTree::CInTree(): + #ifdef HASH_ARRAY_2 + m_Hash2(0), + #ifdef HASH_ARRAY_3 + m_Hash3(0), + #endif + #endif + m_Hash(0), + m_Base(0), + m_CutValue(0xFF) +{ +} + +void CInTree::FreeMemory() +{ + if (m_Base != 0) + delete [] m_Base; + if (m_Hash != 0) + delete [] m_Hash; + m_Base = 0; + m_Hash = 0; + CIn::Free(); +} + +CInTree::~CInTree() +{ + FreeMemory(); +} + +HRESULT CInTree::Create(UINT32 aSizeHistory, UINT32 aKeepAddBufferBefore, + UINT32 aMatchMaxLen, UINT32 aKeepAddBufferAfter, UINT32 aSizeReserv) +{ + FreeMemory(); + CIn::Create(aSizeHistory + aKeepAddBufferBefore, + aMatchMaxLen + aKeepAddBufferAfter, aSizeReserv); + + if (m_BlockSize + 256 > kMaxValForNormalize) + return E_INVALIDARG; + + m_HistorySize = aSizeHistory; + m_MatchMaxLen = aMatchMaxLen; + + + UINT32 aSize = kHashSize; + #ifdef HASH_ARRAY_2 + aSize += kHash2Size; + #ifdef HASH_ARRAY_3 + aSize += kHash3Size; + #endif + #endif + + m_Base = new CPair[m_BlockSize + 1]; + if (m_Base == 0) + return E_OUTOFMEMORY; + m_Hash = new CIndex[aSize + 1]; + if (m_Hash == 0) + return E_OUTOFMEMORY; + + #ifdef HASH_ARRAY_2 + m_Hash2 = &m_Hash[kHashSize]; + #ifdef HASH_ARRAY_3 + m_Hash3 = &m_Hash2[kHash2Size]; + #endif + #endif + return S_OK; +} + +static const UINT32 kEmptyHashValue = 0; + +HRESULT CInTree::Init(ISequentialInStream *aStream) +{ + RETURN_IF_NOT_S_OK(CIn::Init(aStream)); + unsigned i; + for(i = 0; i < kHashSize; i++) + m_Hash[i] = kEmptyHashValue; + + #ifdef HASH_ARRAY_2 + for(i = 0; i < kHash2Size; i++) + m_Hash2[i] = kEmptyHashValue; + #ifdef HASH_ARRAY_3 + for(i = 0; i < kHash3Size; i++) + m_Hash3[i] = kEmptyHashValue; + #endif + #endif + + m_Son = m_Base; + + ReduceOffsets(0 - 1); + return S_OK; +} + + +#ifdef HASH_ARRAY_2 +#ifdef HASH_ARRAY_3 +inline UINT32 Hash(const BYTE *aPointer, UINT32 &aHash2Value, UINT32 &aHash3Value) +{ + UINT32 aTemp = CCRC::m_Table[aPointer[0]] ^ aPointer[1]; + aHash2Value = aTemp & (kHash2Size - 1); + aHash3Value = (aTemp ^ (UINT32(aPointer[2]) << 8)) & (kHash3Size - 1); + return (aTemp ^ (UINT32(aPointer[2]) << 8) ^ (CCRC::m_Table[aPointer[3]] << 5)) & + (kHashSize - 1); +} +#else // no HASH_ARRAY_3 +inline UINT32 Hash(const BYTE *aPointer, UINT32 &aHash2Value) +{ + aHash2Value = (CCRC::m_Table[aPointer[0]] ^ aPointer[1]) & (kHash2Size - 1); + return (*((const UINT32 *)aPointer)) & 0xFFFFFF; +} +#endif // HASH_ARRAY_3 +#else // no HASH_ARRAY_2 +#ifdef HASH_ZIP +inline UINT32 Hash(const BYTE *aPointer) +{ + return ((UINT32(aPointer[0]) << 8) ^ + CCRC::m_Table[aPointer[1]] ^ aPointer[2]) & (kHashSize - 1); +} +#else // no HASH_ZIP +inline UINT32 Hash(const BYTE *aPointer) +{ + return aPointer[0] ^ (UINT32(aPointer[1]) << 8); +} +#endif // HASH_ZIP +#endif // HASH_ARRAY_2 + +UINT32 CInTree::GetLongestMatch(UINT32 *aDistances) +{ + UINT32 aCurrentLimit; + if (m_Pos + m_MatchMaxLen <= m_StreamPos) + aCurrentLimit = m_MatchMaxLen; + else + { + aCurrentLimit = m_StreamPos - m_Pos; + if(aCurrentLimit < kNumHashBytes) + return 0; + } + + UINT32 aMatchMinPos = (m_Pos > m_HistorySize) ? (m_Pos - m_HistorySize) : 1; + BYTE *aCur = m_Buffer + m_Pos; + + UINT32 aMatchHashLenMax = 0; + + #ifdef HASH_ARRAY_2 + UINT32 aHash2Value; + #ifdef HASH_ARRAY_3 + UINT32 aHash3Value; + UINT32 aHashValue = Hash(aCur, aHash2Value, aHash3Value); + #else + UINT32 aHashValue = Hash(aCur, aHash2Value); + #endif + #else + UINT32 aHashValue = Hash(aCur); + #endif + + UINT32 aCurMatch = m_Hash[aHashValue]; + #ifdef HASH_ARRAY_2 + UINT32 aCurMatch2 = m_Hash2[aHash2Value]; + #ifdef HASH_ARRAY_3 + UINT32 aCurMatch3 = m_Hash3[aHash3Value]; + #endif + m_Hash2[aHash2Value] = m_Pos; + bool aMatchLen2Exist = false; + UINT32 aLen2Distance = 0; + if(aCurMatch2 >= aMatchMinPos) + { + if (m_Buffer[aCurMatch2] == aCur[0]) + { + aLen2Distance = m_Pos - aCurMatch2 - 1; + aMatchHashLenMax = 2; + aMatchLen2Exist = true; + } + } + + #ifdef HASH_ARRAY_3 + m_Hash3[aHash3Value] = m_Pos; + UINT32 aMatchLen3Exist = false; + UINT32 aLen3Distance = 0; + if(aCurMatch3 >= aMatchMinPos) + { + if (m_Buffer[aCurMatch3] == aCur[0]) + { + aLen3Distance = m_Pos - aCurMatch3 - 1; + aMatchHashLenMax = 3; + aMatchLen3Exist = true; + if (aMatchLen2Exist) + { + if (aLen3Distance < aLen2Distance) + aLen2Distance = aLen3Distance; + } + else + { + aLen2Distance = aLen3Distance; + aMatchLen2Exist = true; + } + } + } + #endif + #endif + + m_Hash[aHashValue] = m_Pos; + if(aCurMatch < aMatchMinPos) + { + m_Son[m_Pos].Left = kEmptyHashValue; + m_Son[m_Pos].Right = kEmptyHashValue; + + #ifdef HASH_ARRAY_2 + aDistances[2] = aLen2Distance; + #ifdef HASH_ARRAY_3 + aDistances[3] = aLen3Distance; + #endif + #endif + + return aMatchHashLenMax; + } + CIndex *aPtrLeft = &m_Son[m_Pos].Right; + CIndex *aPtrRight = &m_Son[m_Pos].Left; + + UINT32 aMax, aMinSameLeft, aMinSameRight, aMinSame; + aMax = aMinSameLeft = aMinSameRight = aMinSame = kNumHashDirectBytes; + + #ifdef HASH_ARRAY_2 + #ifndef HASH_ARRAY_3 + if (aMatchLen2Exist) + aDistances[2] = aLen2Distance; + else + if (kNumHashDirectBytes >= 2) + aDistances[2] = m_Pos - aCurMatch - 1; + #endif + #endif + + aDistances[aMax] = m_Pos - aCurMatch - 1; + + for(UINT32 aCount = m_CutValue; aCount > 0; aCount--) + { + BYTE *pby1 = m_Buffer + aCurMatch; + // CIndex aLeft = m_Son[aCurMatch].Left; // it's prefetch + UINT32 aCurrentLen; + for(aCurrentLen = aMinSame; aCurrentLen < aCurrentLimit; aCurrentLen++/*, dwComps++*/) + if (pby1[aCurrentLen] != aCur[aCurrentLen]) + break; + while (aCurrentLen > aMax) + aDistances[++aMax] = m_Pos - aCurMatch - 1; + if (aCurrentLen != aCurrentLimit) + { + if (pby1[aCurrentLen] < aCur[aCurrentLen]) + { + *aPtrRight = aCurMatch; + aPtrRight = &m_Son[aCurMatch].Right; + aCurMatch = m_Son[aCurMatch].Right; + if(aCurrentLen > aMinSameLeft) + { + aMinSameLeft = aCurrentLen; + aMinSame = MyMin(aMinSameLeft, aMinSameRight); + } + } + else + { + *aPtrLeft = aCurMatch; + aPtrLeft = &m_Son[aCurMatch].Left; + // aCurMatch = aLeft; + aCurMatch = m_Son[aCurMatch].Left; + if(aCurrentLen > aMinSameRight) + { + aMinSameRight = aCurrentLen; + aMinSame = MyMin(aMinSameLeft, aMinSameRight); + } + } + } + else + { + if(aCurrentLen < m_MatchMaxLen) + { + *aPtrLeft = aCurMatch; + aPtrLeft = &m_Son[aCurMatch].Left; + aCurMatch = m_Son[aCurMatch].Left; + if(aCurrentLen > aMinSameRight) + { + aMinSameRight = aCurrentLen; + aMinSame = MyMin(aMinSameLeft, aMinSameRight); + } + } + else + { + *aPtrLeft = m_Son[aCurMatch].Right; + *aPtrRight = m_Son[aCurMatch].Left; + + #ifdef HASH_ARRAY_2 + if (aMatchLen2Exist && aLen2Distance < aDistances[2]) + aDistances[2] = aLen2Distance; + #ifdef HASH_ARRAY_3 + if (aMatchLen3Exist && aLen3Distance < aDistances[3]) + aDistances[3] = aLen3Distance; + #endif + #endif + + return aMax; + } + } + if(aCurMatch < aMatchMinPos) + break; + } + *aPtrLeft = kEmptyHashValue; + *aPtrRight = kEmptyHashValue; + #ifdef HASH_ARRAY_2 + if (aMatchLen2Exist) + { + if (aMax < 2) + { + aDistances[2] = aLen2Distance; + aMax = 2; + } + else if (aLen2Distance < aDistances[2]) + aDistances[2] = aLen2Distance; + } + #ifdef HASH_ARRAY_3 + if (aMatchLen3Exist) + { + if (aMax < 3) + { + aDistances[3] = aLen3Distance; + aMax = 3; + } + else if (aLen3Distance < aDistances[3]) + aDistances[3] = aLen3Distance; + } + #endif + #endif + return aMax; +} + +void CInTree::DummyLongestMatch() +{ + UINT32 aCurrentLimit; + if (m_Pos + m_MatchMaxLen <= m_StreamPos) + aCurrentLimit = m_MatchMaxLen; + else + { + aCurrentLimit = m_StreamPos - m_Pos; + if(aCurrentLimit < kNumHashBytes) + return; + } + UINT32 aMatchMinPos = (m_Pos > m_HistorySize) ? (m_Pos - m_HistorySize) : 1; + BYTE *aCur = m_Buffer + m_Pos; + + #ifdef HASH_ARRAY_2 + UINT32 aHash2Value; + #ifdef HASH_ARRAY_3 + UINT32 aHash3Value; + UINT32 aHashValue = Hash(aCur, aHash2Value, aHash3Value); + m_Hash3[aHash3Value] = m_Pos; + #else + UINT32 aHashValue = Hash(aCur, aHash2Value); + #endif + m_Hash2[aHash2Value] = m_Pos; + #else + UINT32 aHashValue = Hash(aCur); + #endif + + UINT32 aCurMatch = m_Hash[aHashValue]; + m_Hash[aHashValue] = m_Pos; + if(aCurMatch < aMatchMinPos) + { + m_Son[m_Pos].Left = kEmptyHashValue; + m_Son[m_Pos].Right = kEmptyHashValue; + return; + } + CIndex *aPtrLeft = &m_Son[m_Pos].Right; + CIndex *aPtrRight = &m_Son[m_Pos].Left; + + UINT32 aMax, aMinSameLeft, aMinSameRight, aMinSame; + aMax = aMinSameLeft = aMinSameRight = aMinSame = kNumHashDirectBytes; + for(UINT32 aCount = m_CutValue; aCount > 0; aCount--) + { + BYTE *pby1 = m_Buffer + aCurMatch; + // CIndex aLeft = m_Son[aCurMatch].Left; // it's prefetch + UINT32 aCurrentLen; + for(aCurrentLen = aMinSame; aCurrentLen < aCurrentLimit; aCurrentLen++/*, dwComps++*/) + if (pby1[aCurrentLen] != aCur[aCurrentLen]) + break; + if (aCurrentLen != aCurrentLimit) + { + if (pby1[aCurrentLen] < aCur[aCurrentLen]) + { + *aPtrRight = aCurMatch; + aPtrRight = &m_Son[aCurMatch].Right; + aCurMatch = m_Son[aCurMatch].Right; + if(aCurrentLen > aMinSameLeft) + { + aMinSameLeft = aCurrentLen; + aMinSame = MyMin(aMinSameLeft, aMinSameRight); + } + } + else + { + *aPtrLeft = aCurMatch; + aPtrLeft = &m_Son[aCurMatch].Left; + aCurMatch = m_Son[aCurMatch].Left; + // aCurMatch = aLeft; + if(aCurrentLen > aMinSameRight) + { + aMinSameRight = aCurrentLen; + aMinSame = MyMin(aMinSameLeft, aMinSameRight); + } + } + } + else + { + if(aCurrentLen < m_MatchMaxLen) + { + *aPtrLeft = aCurMatch; + aPtrLeft = &m_Son[aCurMatch].Left; + aCurMatch = m_Son[aCurMatch].Left; + if(aCurrentLen > aMinSameRight) + { + aMinSameRight = aCurrentLen; + aMinSame = MyMin(aMinSameLeft, aMinSameRight); + } + } + else + { + *aPtrLeft = m_Son[aCurMatch].Right; + *aPtrRight = m_Son[aCurMatch].Left; + return; + } + } + if(aCurMatch < aMatchMinPos) + break; + } + *aPtrLeft = kEmptyHashValue; + *aPtrRight = kEmptyHashValue; +} + +void CInTree::AfterMoveBlock() +{ + UINT32 aNumBytesToMove = m_HistorySize * sizeof(CPair); + UINT32 aSpecOffset = ((m_Son + m_Pos) - m_Base) - m_HistorySize; + memmove(m_Base, m_Base + aSpecOffset, aNumBytesToMove); + m_Son -= aSpecOffset; +} + +void CInTree::NormalizeLinks(CIndex *anArray, UINT32 aNumItems, UINT32 aSubValue) +{ + for (UINT32 i = 0; i < aNumItems; i++) + { + UINT32 aValue = anArray[i]; + if (aValue <= aSubValue) + aValue = kEmptyHashValue; + else + aValue -= aSubValue; + anArray[i] = aValue; + } +} + +void CInTree::Normalize() +{ + UINT32 aStartItem = m_Pos - m_HistorySize; + UINT32 aSubValue = aStartItem - 1; + NormalizeLinks((CIndex *)(m_Son + aStartItem), m_HistorySize * 2, aSubValue); + NormalizeLinks(m_Hash, kHashSize, aSubValue); + + #ifdef HASH_ARRAY_2 + NormalizeLinks(m_Hash2, kHash2Size, aSubValue); + #ifdef HASH_ARRAY_3 + NormalizeLinks(m_Hash3, kHash3Size, aSubValue); + #endif + #endif + + ReduceOffsets(aSubValue); +} + +} diff --git a/hostTools/lzma/compress/BitTreeCoder.h b/hostTools/lzma/compress/BitTreeCoder.h new file mode 100644 index 0000000..a77f32f --- /dev/null +++ b/hostTools/lzma/compress/BitTreeCoder.h @@ -0,0 +1,290 @@ +#ifndef __BITTREECODER_H +#define __BITTREECODER_H + +#include "AriBitCoder.h" +#include "RCDefs.h" + +////////////////////////// +// CBitTreeEncoder + +template +class CBitTreeEncoder +{ + CMyBitEncoder m_Models[1 << m_NumBitLevels]; +public: + void Init() + { + for(UINT32 i = 1; i < (1 << m_NumBitLevels); i++) + m_Models[i].Init(); + } + void Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aSymbol) + { + UINT32 aModelIndex = 1; + for (UINT32 aBitIndex = m_NumBitLevels; aBitIndex > 0 ;) + { + aBitIndex--; + UINT32 aBit = (aSymbol >> aBitIndex ) & 1; + m_Models[aModelIndex].Encode(aRangeEncoder, aBit); + aModelIndex = (aModelIndex << 1) | aBit; + } + }; + UINT32 GetPrice(UINT32 aSymbol) const + { + UINT32 aPrice = 0; + UINT32 aModelIndex = 1; + for (UINT32 aBitIndex = m_NumBitLevels; aBitIndex > 0 ;) + { + aBitIndex--; + UINT32 aBit = (aSymbol >> aBitIndex ) & 1; + aPrice += m_Models[aModelIndex].GetPrice(aBit); + aModelIndex = (aModelIndex << 1) + aBit; + } + return aPrice; + } +}; + +////////////////////////// +// CBitTreeDecoder + +template +class CBitTreeDecoder +{ + CMyBitDecoder m_Models[1 << m_NumBitLevels]; +public: + void Init() + { + for(UINT32 i = 1; i < (1 << m_NumBitLevels); i++) + m_Models[i].Init(); + } + UINT32 Decode(CMyRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + RC_INIT_VAR + for(UINT32 aBitIndex = m_NumBitLevels; aBitIndex > 0; aBitIndex--) + { + // aModelIndex = (aModelIndex << 1) + m_Models[aModelIndex].Decode(aRangeDecoder); + RC_GETBIT(aNumMoveBits, m_Models[aModelIndex].m_Probability, aModelIndex) + } + RC_FLUSH_VAR + return aModelIndex - (1 << m_NumBitLevels); + }; +}; + + + +//////////////////////////////// +// CReverseBitTreeEncoder + +template +class CReverseBitTreeEncoder2 +{ + CMyBitEncoder *m_Models; + UINT32 m_NumBitLevels; +public: + CReverseBitTreeEncoder2(): m_Models(0) { } + ~CReverseBitTreeEncoder2() { delete []m_Models; } + bool Create(UINT32 aNumBitLevels) + { + m_NumBitLevels = aNumBitLevels; + m_Models = new CMyBitEncoder[1 << aNumBitLevels]; + return (m_Models != 0); + } + void Init() + { + UINT32 aNumModels = 1 << m_NumBitLevels; + for(UINT32 i = 1; i < aNumModels; i++) + m_Models[i].Init(); + } + void Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aSymbol) + { + UINT32 aModelIndex = 1; + for (UINT32 i = 0; i < m_NumBitLevels; i++) + { + UINT32 aBit = aSymbol & 1; + m_Models[aModelIndex].Encode(aRangeEncoder, aBit); + aModelIndex = (aModelIndex << 1) | aBit; + aSymbol >>= 1; + } + } + UINT32 GetPrice(UINT32 aSymbol) const + { + UINT32 aPrice = 0; + UINT32 aModelIndex = 1; + for (UINT32 i = m_NumBitLevels; i > 0; i--) + { + UINT32 aBit = aSymbol & 1; + aSymbol >>= 1; + aPrice += m_Models[aModelIndex].GetPrice(aBit); + aModelIndex = (aModelIndex << 1) | aBit; + } + return aPrice; + } +}; + +/* +template +class CReverseBitTreeEncoder: public CReverseBitTreeEncoder2 +{ +public: + CReverseBitTreeEncoder() + { Create(aNumBitLevels); } +}; +*/ +//////////////////////////////// +// CReverseBitTreeDecoder + +template +class CReverseBitTreeDecoder2 +{ + CMyBitDecoder *m_Models; + UINT32 m_NumBitLevels; +public: + CReverseBitTreeDecoder2(): m_Models(0) { } + ~CReverseBitTreeDecoder2() { delete []m_Models; } + bool Create(UINT32 aNumBitLevels) + { + m_NumBitLevels = aNumBitLevels; + m_Models = new CMyBitDecoder[1 << aNumBitLevels]; + return (m_Models != 0); + } + void Init() + { + UINT32 aNumModels = 1 << m_NumBitLevels; + for(UINT32 i = 1; i < aNumModels; i++) + m_Models[i].Init(); + } + UINT32 Decode(CMyRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + UINT32 aSymbol = 0; + RC_INIT_VAR + for(UINT32 aBitIndex = 0; aBitIndex < m_NumBitLevels; aBitIndex++) + { + // UINT32 aBit = m_Models[aModelIndex].Decode(aRangeDecoder); + // aModelIndex <<= 1; + // aModelIndex += aBit; + // aSymbol |= (aBit << aBitIndex); + RC_GETBIT2(aNumMoveBits, m_Models[aModelIndex].m_Probability, aModelIndex, ; , aSymbol |= (1 << aBitIndex)) + } + RC_FLUSH_VAR + return aSymbol; + }; +}; +//////////////////////////// +// CReverseBitTreeDecoder2 + +template +class CReverseBitTreeDecoder +{ + CMyBitDecoder m_Models[1 << m_NumBitLevels]; +public: + void Init() + { + for(UINT32 i = 1; i < (1 << m_NumBitLevels); i++) + m_Models[i].Init(); + } + UINT32 Decode(CMyRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + UINT32 aSymbol = 0; + RC_INIT_VAR + for(UINT32 aBitIndex = 0; aBitIndex < m_NumBitLevels; aBitIndex++) + { + // UINT32 aBit = m_Models[aModelIndex].Decode(aRangeDecoder); + // aModelIndex <<= 1; + // aModelIndex += aBit; + // aSymbol |= (aBit << aBitIndex); + RC_GETBIT2(aNumMoveBits, m_Models[aModelIndex].m_Probability, aModelIndex, ; , aSymbol |= (1 << aBitIndex)) + } + RC_FLUSH_VAR + return aSymbol; + } +}; + +/* +////////////////////////// +// CBitTreeEncoder2 + +template +class CBitTreeEncoder2 +{ + NCompression::NArithmetic::CBitEncoder *m_Models; + UINT32 m_NumBitLevels; +public: + bool Create(UINT32 aNumBitLevels) + { + m_NumBitLevels = aNumBitLevels; + m_Models = new NCompression::NArithmetic::CBitEncoder[1 << aNumBitLevels]; + return (m_Models != 0); + } + void Init() + { + UINT32 aNumModels = 1 << m_NumBitLevels; + for(UINT32 i = 1; i < aNumModels; i++) + m_Models[i].Init(); + } + void Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aSymbol) + { + UINT32 aModelIndex = 1; + for (UINT32 aBitIndex = m_NumBitLevels; aBitIndex > 0 ;) + { + aBitIndex--; + UINT32 aBit = (aSymbol >> aBitIndex ) & 1; + m_Models[aModelIndex].Encode(aRangeEncoder, aBit); + aModelIndex = (aModelIndex << 1) | aBit; + } + } + UINT32 GetPrice(UINT32 aSymbol) const + { + UINT32 aPrice = 0; + UINT32 aModelIndex = 1; + for (UINT32 aBitIndex = m_NumBitLevels; aBitIndex > 0 ;) + { + aBitIndex--; + UINT32 aBit = (aSymbol >> aBitIndex ) & 1; + aPrice += m_Models[aModelIndex].GetPrice(aBit); + aModelIndex = (aModelIndex << 1) + aBit; + } + return aPrice; + } +}; + + +////////////////////////// +// CBitTreeDecoder2 + +template +class CBitTreeDecoder2 +{ + NCompression::NArithmetic::CBitDecoder *m_Models; + UINT32 m_NumBitLevels; +public: + bool Create(UINT32 aNumBitLevels) + { + m_NumBitLevels = aNumBitLevels; + m_Models = new NCompression::NArithmetic::CBitDecoder[1 << aNumBitLevels]; + return (m_Models != 0); + } + void Init() + { + UINT32 aNumModels = 1 << m_NumBitLevels; + for(UINT32 i = 1; i < aNumModels; i++) + m_Models[i].Init(); + } + UINT32 Decode(CMyRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + RC_INIT_VAR + for(UINT32 aBitIndex = m_NumBitLevels; aBitIndex > 0; aBitIndex--) + { + // aModelIndex = (aModelIndex << 1) + m_Models[aModelIndex].Decode(aRangeDecoder); + RC_GETBIT(aNumMoveBits, m_Models[aModelIndex].m_Probability, aModelIndex) + } + RC_FLUSH_VAR + return aModelIndex - (1 << m_NumBitLevels); + } +}; +*/ + + +#endif diff --git a/hostTools/lzma/compress/CRC.h b/hostTools/lzma/compress/CRC.h new file mode 100644 index 0000000..cbef4b6 --- /dev/null +++ b/hostTools/lzma/compress/CRC.h @@ -0,0 +1,27 @@ +#ifndef __COMMON_CRC_H +#define __COMMON_CRC_H + +#include "Portable.h" + +class CCRC +{ + UINT32 m_Value; +public: + static UINT32 m_Table[256]; + CCRC(): m_Value(0xFFFFFFFF){}; + void Init() { m_Value = 0xFFFFFFFF; } + void Update(const void *aData, UINT32 aSize); + UINT32 GetDigest() const { return m_Value ^ 0xFFFFFFFF; } + static UINT32 CalculateDigest(const void *aData, UINT32 aSize) + { + CCRC aCRC; + aCRC.Update(aData, aSize); + return aCRC.GetDigest(); + } + static bool VerifyDigest(UINT32 aDigest, const void *aData, UINT32 aSize) + { + return (CalculateDigest(aData, aSize) == aDigest); + } +}; + +#endif diff --git a/hostTools/lzma/compress/Const.h b/hostTools/lzma/compress/Const.h new file mode 100644 index 0000000..7cb9d79 --- /dev/null +++ b/hostTools/lzma/compress/Const.h @@ -0,0 +1,92 @@ +#ifndef __DEFLATE_CONST_H +#define __DEFLATE_CONST_H + +namespace NDeflate { + +const UINT32 kDistTableSize = 30; +const UINT32 kHistorySize = 0x8000; +const UINT32 kNumLenCombinations = 256; + +const UINT32 kNumHuffmanBits = 15; + +const UINT32 kLenTableSize = 29; + +const UINT32 kStaticDistTableSize = 32; +const UINT32 kStaticLenTableSize = 31; + +const UINT32 kReadTableNumber = 0x100; +const UINT32 kMatchNumber = kReadTableNumber + 1; + +const UINT32 kMainTableSize = kMatchNumber + kLenTableSize; //298; +const UINT32 kStaticMainTableSize = kMatchNumber + kStaticLenTableSize; //298; + +const UINT32 kDistTableStart = kMainTableSize; + +const UINT32 kHeapTablesSizesSum = kMainTableSize + kDistTableSize; + +const UINT32 kLevelTableSize = 19; + +const UINT32 kMaxTableSize = kHeapTablesSizesSum; // test it +const UINT32 kStaticMaxTableSize = kStaticMainTableSize + kStaticDistTableSize; + +const UINT32 kTableDirectLevels = 16; +const UINT32 kTableLevelRepNumber = kTableDirectLevels; +const UINT32 kTableLevel0Number = kTableLevelRepNumber + 1; +const UINT32 kTableLevel0Number2 = kTableLevel0Number + 1; + +const UINT32 kLevelMask = 0xF; + +const BYTE kLenStart[kLenTableSize] = {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 255}; +const BYTE kLenDirectBits[kLenTableSize] = {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0}; + + +const UINT32 kDistStart[kDistTableSize] = {0,1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256,384,512,768,1024,1536,2048,3072,4096,6144,8192,12288,16384,24576}; +const BYTE kDistDirectBits[kDistTableSize] = {0,0,0,0,1,1,2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13}; + +const BYTE kLevelDirectBits[kLevelTableSize] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7}; + +const BYTE kCodeLengthAlphabetOrder[kLevelTableSize] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + +const UINT32 kMatchMinLen = 3; +const UINT32 kMatchMaxLen = kNumLenCombinations + kMatchMinLen - 1; // 255 + 2; test it + +const int kFinalBlockFieldSize = 1; + +namespace NFinalBlockField +{ +enum +{ + kNotFinalBlock = 0, + kFinalBlock = 1 +}; +} + +const int kBlockTypeFieldSize = 2; + +namespace NBlockType +{ + enum + { + kStored = 0, + kFixedHuffman = 1, + kDynamicHuffman = 2, + kReserved = 3 + }; +} + +const UINT32 kDeflateNumberOfLengthCodesFieldSize = 5; +const UINT32 kDeflateNumberOfDistanceCodesFieldSize = 5; +const UINT32 kDeflateNumberOfLevelCodesFieldSize = 4; + +const UINT32 kDeflateNumberOfLitLenCodesMin = 257; + +const UINT32 kDeflateNumberOfDistanceCodesMin = 1; +const UINT32 kDeflateNumberOfLevelCodesMin = 4; + +const UINT32 kDeflateLevelCodeFieldSize = 3; + +const UINT32 kDeflateStoredBlockLengthFieldSizeSize = 16; + +} + +#endif diff --git a/hostTools/lzma/compress/IInOutStreams.cpp b/hostTools/lzma/compress/IInOutStreams.cpp new file mode 100644 index 0000000..73bd1e0 --- /dev/null +++ b/hostTools/lzma/compress/IInOutStreams.cpp @@ -0,0 +1,25 @@ +#include "Portable.h" +#include "IInOutStreams.h" + +HRESULT ISequentialInStream::Read(void *aData, UINT32 aSize, UINT32* aProcessedSize) { + if (aSize > size) + aSize = size; + *aProcessedSize = aSize; + memcpy(aData, data, aSize); + size -= aSize; + data += aSize; + return S_OK; +} + +HRESULT ISequentialOutStream::Write(const void *aData, UINT32 aSize, UINT32* aProcessedSize) { + if (aSize > size) { + overflow = true; + aSize = size; + } + *aProcessedSize = aSize; + memcpy(data, aData, aSize); + size -= aSize; + data += aSize; + total += aSize; + return S_OK; +} diff --git a/hostTools/lzma/compress/IInOutStreams.h b/hostTools/lzma/compress/IInOutStreams.h new file mode 100644 index 0000000..c084217 --- /dev/null +++ b/hostTools/lzma/compress/IInOutStreams.h @@ -0,0 +1,31 @@ +#ifndef __IINOUTSTREAMS_H +#define __IINOUTSTREAMS_H + +#include "Portable.h" + +class ISequentialInStream +{ + const char* data; + unsigned size; +public: + ISequentialInStream(const char* Adata, unsigned Asize) : data(Adata), size(Asize) { } + + HRESULT Read(void *aData, UINT32 aSize, UINT32 *aProcessedSize); +}; + +class ISequentialOutStream +{ + char* data; + unsigned size; + bool overflow; + unsigned total; +public: + ISequentialOutStream(char* Adata, unsigned Asize) : data(Adata), size(Asize), overflow(false), total(0) { } + + bool overflow_get() const { return overflow; } + unsigned size_get() const { return total; } + + HRESULT Write(const void *aData, UINT32 aSize, UINT32 *aProcessedSize); +}; + +#endif diff --git a/hostTools/lzma/compress/InByte.cpp b/hostTools/lzma/compress/InByte.cpp new file mode 100644 index 0000000..03f6a62 --- /dev/null +++ b/hostTools/lzma/compress/InByte.cpp @@ -0,0 +1,41 @@ +#include "InByte.h" + +namespace NStream{ + +CInByte::CInByte(UINT32 aBufferSize): + m_BufferSize(aBufferSize), + m_BufferBase(0) +{ + m_BufferBase = new BYTE[m_BufferSize]; +} + +CInByte::~CInByte() +{ + delete []m_BufferBase; +} + +void CInByte::Init(ISequentialInStream *aStream) +{ + m_Stream = aStream; + m_ProcessedSize = 0; + m_Buffer = m_BufferBase; + m_BufferLimit = m_Buffer; + m_StreamWasExhausted = false; +} + +bool CInByte::ReadBlock() +{ + if (m_StreamWasExhausted) + return false; + m_ProcessedSize += (m_Buffer - m_BufferBase); + UINT32 aNumProcessedBytes; + HRESULT aResult = m_Stream->Read(m_BufferBase, m_BufferSize, &aNumProcessedBytes); + if (aResult != S_OK) + throw aResult; + m_Buffer = m_BufferBase; + m_BufferLimit = m_Buffer + aNumProcessedBytes; + m_StreamWasExhausted = (aNumProcessedBytes == 0); + return (!m_StreamWasExhausted); +} + +} diff --git a/hostTools/lzma/compress/InByte.h b/hostTools/lzma/compress/InByte.h new file mode 100644 index 0000000..49bf2f3 --- /dev/null +++ b/hostTools/lzma/compress/InByte.h @@ -0,0 +1,58 @@ +#ifndef __STREAM_INBYTE_H +#define __STREAM_INBYTE_H + +#include "IInOutStreams.h" + +namespace NStream { + +class CInByte +{ + UINT64 m_ProcessedSize; + BYTE *m_BufferBase; + UINT32 m_BufferSize; + BYTE *m_Buffer; + BYTE *m_BufferLimit; + ISequentialInStream* m_Stream; + bool m_StreamWasExhausted; + + bool ReadBlock(); + +public: + CInByte(UINT32 aBufferSize = 0x100000); + ~CInByte(); + + void Init(ISequentialInStream *aStream); + + bool ReadByte(BYTE &aByte) + { + if(m_Buffer >= m_BufferLimit) + if(!ReadBlock()) + return false; + aByte = *m_Buffer++; + return true; + } + BYTE ReadByte() + { + if(m_Buffer >= m_BufferLimit) + if(!ReadBlock()) + return 0x0; + return *m_Buffer++; + } + void ReadBytes(void *aData, UINT32 aSize, UINT32 &aProcessedSize) + { + for(aProcessedSize = 0; aProcessedSize < aSize; aProcessedSize++) + if (!ReadByte(((BYTE *)aData)[aProcessedSize])) + return; + } + bool ReadBytes(void *aData, UINT32 aSize) + { + UINT32 aProcessedSize; + ReadBytes(aData, aSize, aProcessedSize); + return (aProcessedSize == aSize); + } + UINT64 GetProcessedSize() const { return m_ProcessedSize + (m_Buffer - m_BufferBase); } +}; + +} + +#endif diff --git a/hostTools/lzma/compress/LZMA.cpp b/hostTools/lzma/compress/LZMA.cpp new file mode 100644 index 0000000..3774853 --- /dev/null +++ b/hostTools/lzma/compress/LZMA.cpp @@ -0,0 +1,23 @@ +#include "LZMA.h" + +namespace NCompress { +namespace NLZMA { + +UINT32 kDistStart[kDistTableSizeMax]; + +static class CConstInit +{ +public: + CConstInit() + { + UINT32 aStartValue = 0; + int i; + for (i = 0; i < kDistTableSizeMax; i++) + { + kDistStart[i] = aStartValue; + aStartValue += (1 << kDistDirectBits[i]); + } + } +} g_ConstInit; + +}} diff --git a/hostTools/lzma/compress/LZMA.h b/hostTools/lzma/compress/LZMA.h new file mode 100644 index 0000000..1a947f7 --- /dev/null +++ b/hostTools/lzma/compress/LZMA.h @@ -0,0 +1,105 @@ +#include "LenCoder.h" + +#ifndef __LZMA_H +#define __LZMA_H + +namespace NCompress { +namespace NLZMA { + +const UINT32 kNumRepDistances = 4; + +const BYTE kNumStates = 12; + +const BYTE kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +const BYTE kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +const BYTE kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +const BYTE kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +class CState +{ +public: + BYTE m_Index; + void Init() + { m_Index = 0; } + void UpdateChar() + { m_Index = kLiteralNextStates[m_Index]; } + void UpdateMatch() + { m_Index = kMatchNextStates[m_Index]; } + void UpdateRep() + { m_Index = kRepNextStates[m_Index]; } + void UpdateShortRep() + { m_Index = kShortRepNextStates[m_Index]; } +}; + +class CBaseCoder +{ +protected: + CState m_State; + BYTE m_PreviousByte; + bool m_PeviousIsMatch; + UINT32 m_RepDistances[kNumRepDistances]; + void Init() + { + m_State.Init(); + m_PreviousByte = 0; + m_PeviousIsMatch = false; + for(int i = 0 ; i < kNumRepDistances; i++) + m_RepDistances[i] = 0; + } +}; + +const int kNumPosSlotBits = 6; +const int kDicLogSizeMax = 28; +const int kDistTableSizeMax = kDicLogSizeMax * 2; + +extern UINT32 kDistStart[kDistTableSizeMax]; +const BYTE kDistDirectBits[kDistTableSizeMax] = +{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, + 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26 +}; + +const UINT32 kNumLenToPosStates = 4; +inline UINT32 GetLenToPosState(UINT32 aLen) +{ + aLen -= 2; + if (aLen < kNumLenToPosStates) + return aLen; + return kNumLenToPosStates - 1; +} + +const int kMatchMinLen = 2; + +const int kMatchMaxLen = kMatchMinLen + NLength::kNumSymbolsTotal - 1; + +const int kNumAlignBits = 4; +const int kAlignTableSize = 1 << kNumAlignBits; +const UINT32 kAlignMask = (kAlignTableSize - 1); + +const int kStartPosModelIndex = 4; +const int kEndPosModelIndex = 14; +const int kNumPosModels = kEndPosModelIndex - kStartPosModelIndex; + +const int kNumFullDistances = 1 << (kEndPosModelIndex / 2); + + +const int kMainChoiceLiteralIndex = 0; +const int kMainChoiceMatchIndex = 1; + +const int kMatchChoiceDistanceIndex= 0; +const int kMatchChoiceRepetitionIndex = 1; + +const int kNumMoveBitsForMainChoice = 5; +const int kNumMoveBitsForPosCoders = 5; + +const int kNumMoveBitsForAlignCoders = 5; + +const int kNumMoveBitsForPosSlotCoder = 5; + +const int kNumLitPosStatesBitsEncodingMax = 4; +const int kNumLitContextBitsMax = 8; + +}} + +#endif diff --git a/hostTools/lzma/compress/LZMADecoder.cpp b/hostTools/lzma/compress/LZMADecoder.cpp new file mode 100644 index 0000000..1c834a4 --- /dev/null +++ b/hostTools/lzma/compress/LZMADecoder.cpp @@ -0,0 +1,276 @@ +#include "Portable.h" +#include "LZMADecoder.h" + +#define RETURN_E_OUTOFMEMORY_IF_FALSE(x) { if (!(x)) return E_OUTOFMEMORY; } + +namespace NCompress { +namespace NLZMA { + +HRESULT CDecoder::SetDictionarySize(UINT32 aDictionarySize) +{ + if (aDictionarySize > (1 << kDicLogSizeMax)) + return E_INVALIDARG; + + UINT32 aWindowReservSize = MyMax(aDictionarySize, UINT32(1 << 21)); + + if (m_DictionarySize != aDictionarySize) + { + m_OutWindowStream.Create(aDictionarySize, kMatchMaxLen, aWindowReservSize); + m_DictionarySize = aDictionarySize; + } + return S_OK; +} + +HRESULT CDecoder::SetLiteralProperties( + UINT32 aLiteralPosStateBits, UINT32 aLiteralContextBits) +{ + if (aLiteralPosStateBits > 8) + return E_INVALIDARG; + if (aLiteralContextBits > 8) + return E_INVALIDARG; + m_LiteralDecoder.Create(aLiteralPosStateBits, aLiteralContextBits); + return S_OK; +} + +HRESULT CDecoder::SetPosBitsProperties(UINT32 aNumPosStateBits) +{ + if (aNumPosStateBits > NLength::kNumPosStatesBitsMax) + return E_INVALIDARG; + UINT32 aNumPosStates = 1 << aNumPosStateBits; + m_LenDecoder.Create(aNumPosStates); + m_RepMatchLenDecoder.Create(aNumPosStates); + m_PosStateMask = aNumPosStates - 1; + return S_OK; +} + +CDecoder::CDecoder(): + m_DictionarySize((UINT32)-1) +{ + Create(); +} + +HRESULT CDecoder::Create() +{ + for(int i = 0; i < kNumPosModels; i++) + { + RETURN_E_OUTOFMEMORY_IF_FALSE( + m_PosDecoders[i].Create(kDistDirectBits[kStartPosModelIndex + i])); + } + return S_OK; +} + + +HRESULT CDecoder::Init(ISequentialInStream *anInStream, + ISequentialOutStream *anOutStream) +{ + m_RangeDecoder.Init(anInStream); + + m_OutWindowStream.Init(anOutStream); + + int i; + for(i = 0; i < kNumStates; i++) + { + for (UINT32 j = 0; j <= m_PosStateMask; j++) + { + m_MainChoiceDecoders[i][j].Init(); + m_MatchRepShortChoiceDecoders[i][j].Init(); + } + m_MatchChoiceDecoders[i].Init(); + m_MatchRepChoiceDecoders[i].Init(); + m_MatchRep1ChoiceDecoders[i].Init(); + m_MatchRep2ChoiceDecoders[i].Init(); + } + + m_LiteralDecoder.Init(); + + // m_RepMatchLenDecoder.Init(); + + for (i = 0; i < kNumLenToPosStates; i++) + m_PosSlotDecoder[i].Init(); + + for(i = 0; i < kNumPosModels; i++) + m_PosDecoders[i].Init(); + + m_LenDecoder.Init(); + m_RepMatchLenDecoder.Init(); + + m_PosAlignDecoder.Init(); + return S_OK; + +} + +HRESULT CDecoder::CodeReal(ISequentialInStream *anInStream, + ISequentialOutStream *anOutStream, + const UINT64 *anInSize, const UINT64 *anOutSize) +{ + if (anOutSize == NULL) + return E_INVALIDARG; + + Init(anInStream, anOutStream); + + CState aState; + aState.Init(); + bool aPeviousIsMatch = false; + BYTE aPreviousByte = 0; + UINT32 aRepDistances[kNumRepDistances]; + for(int i = 0 ; i < kNumRepDistances; i++) + aRepDistances[i] = 0; + + UINT64 aNowPos64 = 0; + UINT64 aSize = *anOutSize; + while(aNowPos64 < aSize) + { + UINT64 aNext = MyMin(aNowPos64 + (1 << 18), aSize); + while(aNowPos64 < aNext) + { + UINT32 aPosState = UINT32(aNowPos64) & m_PosStateMask; + if (m_MainChoiceDecoders[aState.m_Index][aPosState].Decode(&m_RangeDecoder) == kMainChoiceLiteralIndex) + { + // aCounts[0]++; + aState.UpdateChar(); + if(aPeviousIsMatch) + { + BYTE aMatchByte = m_OutWindowStream.GetOneByte(0 - aRepDistances[0] - 1); + aPreviousByte = m_LiteralDecoder.DecodeWithMatchByte(&m_RangeDecoder, + UINT32(aNowPos64), aPreviousByte, aMatchByte); + aPeviousIsMatch = false; + } + else + aPreviousByte = m_LiteralDecoder.DecodeNormal(&m_RangeDecoder, + UINT32(aNowPos64), aPreviousByte); + m_OutWindowStream.PutOneByte(aPreviousByte); + aNowPos64++; + } + else + { + aPeviousIsMatch = true; + UINT32 aDistance, aLen; + if(m_MatchChoiceDecoders[aState.m_Index].Decode(&m_RangeDecoder) == + kMatchChoiceRepetitionIndex) + { + if(m_MatchRepChoiceDecoders[aState.m_Index].Decode(&m_RangeDecoder) == 0) + { + if(m_MatchRepShortChoiceDecoders[aState.m_Index][aPosState].Decode(&m_RangeDecoder) == 0) + { + aState.UpdateShortRep(); + aPreviousByte = m_OutWindowStream.GetOneByte(0 - aRepDistances[0] - 1); + m_OutWindowStream.PutOneByte(aPreviousByte); + aNowPos64++; + // aCounts[3 + 4]++; + continue; + } + // aCounts[3 + 0]++; + aDistance = aRepDistances[0]; + } + else + { + if(m_MatchRep1ChoiceDecoders[aState.m_Index].Decode(&m_RangeDecoder) == 0) + { + aDistance = aRepDistances[1]; + aRepDistances[1] = aRepDistances[0]; + // aCounts[3 + 1]++; + } + else + { + if (m_MatchRep2ChoiceDecoders[aState.m_Index].Decode(&m_RangeDecoder) == 0) + { + // aCounts[3 + 2]++; + aDistance = aRepDistances[2]; + } + else + { + // aCounts[3 + 3]++; + aDistance = aRepDistances[3]; + aRepDistances[3] = aRepDistances[2]; + } + aRepDistances[2] = aRepDistances[1]; + aRepDistances[1] = aRepDistances[0]; + } + aRepDistances[0] = aDistance; + } + aLen = m_RepMatchLenDecoder.Decode(&m_RangeDecoder, aPosState) + kMatchMinLen; + // aCounts[aLen]++; + aState.UpdateRep(); + } + else + { + aLen = kMatchMinLen + m_LenDecoder.Decode(&m_RangeDecoder, aPosState); + aState.UpdateMatch(); + UINT32 aPosSlot = m_PosSlotDecoder[GetLenToPosState(aLen)].Decode(&m_RangeDecoder); + // aCounts[aPosSlot]++; + if (aPosSlot >= kStartPosModelIndex) + { + aDistance = kDistStart[aPosSlot]; + if (aPosSlot < kEndPosModelIndex) + aDistance += m_PosDecoders[aPosSlot - kStartPosModelIndex].Decode(&m_RangeDecoder); + else + { + aDistance += (m_RangeDecoder.DecodeDirectBits(kDistDirectBits[aPosSlot] - + kNumAlignBits) << kNumAlignBits); + aDistance += m_PosAlignDecoder.Decode(&m_RangeDecoder); + } + } + else + aDistance = aPosSlot; + + + aRepDistances[3] = aRepDistances[2]; + aRepDistances[2] = aRepDistances[1]; + aRepDistances[1] = aRepDistances[0]; + + aRepDistances[0] = aDistance; + // UpdateStat(aLen, aPosSlot); + } + if (aDistance >= aNowPos64) + throw E_INVALIDDATA; + m_OutWindowStream.CopyBackBlock(aDistance, aLen); + aNowPos64 += aLen; + aPreviousByte = m_OutWindowStream.GetOneByte(0 - 1); + } + } + } + return Flush(); +} + +HRESULT CDecoder::Code(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream, const UINT64 *anInSize, const UINT64 *anOutSize) +{ + try { + return CodeReal(anInStream, anOutStream, anInSize, anOutSize); + } catch (HRESULT& e) { + return e; + } catch (...) { + return E_FAIL; + } +} + +HRESULT CDecoder::ReadCoderProperties(ISequentialInStream *anInStream) +{ + UINT32 aNumPosStateBits; + UINT32 aLiteralPosStateBits; + UINT32 aLiteralContextBits; + UINT32 aDictionarySize; + + UINT32 aProcessesedSize; + + BYTE aByte; + RETURN_IF_NOT_S_OK(anInStream->Read(&aByte, sizeof(aByte), &aProcessesedSize)); + if (aProcessesedSize != sizeof(aByte)) + return E_INVALIDARG; + + aLiteralContextBits = aByte % 9; + BYTE aRemainder = aByte / 9; + aLiteralPosStateBits = aRemainder % 5; + aNumPosStateBits = aRemainder / 5; + + RETURN_IF_NOT_S_OK(anInStream->Read(&aDictionarySize, sizeof(aDictionarySize), &aProcessesedSize)); + if (aProcessesedSize != sizeof(aDictionarySize)) + return E_INVALIDARG; + + RETURN_IF_NOT_S_OK(SetDictionarySize(aDictionarySize)); + RETURN_IF_NOT_S_OK(SetLiteralProperties(aLiteralPosStateBits, aLiteralContextBits)); + RETURN_IF_NOT_S_OK(SetPosBitsProperties(aNumPosStateBits)); + + return S_OK; +} + +}} diff --git a/hostTools/lzma/compress/LZMADecoder.h b/hostTools/lzma/compress/LZMADecoder.h new file mode 100644 index 0000000..610a84d --- /dev/null +++ b/hostTools/lzma/compress/LZMADecoder.h @@ -0,0 +1,64 @@ +#ifndef __LZARITHMETIC_DECODER_H +#define __LZARITHMETIC_DECODER_H + +#include "WindowOut.h" +#include "LZMA.h" +#include "LenCoder.h" +#include "LiteralCoder.h" + +namespace NCompress { +namespace NLZMA { + +typedef CMyBitDecoder CMyBitDecoder2; + +class CDecoder +{ + NStream::NWindow::COut m_OutWindowStream; + CMyRangeDecoder m_RangeDecoder; + + CMyBitDecoder2 m_MainChoiceDecoders[kNumStates][NLength::kNumPosStatesMax]; + CMyBitDecoder2 m_MatchChoiceDecoders[kNumStates]; + CMyBitDecoder2 m_MatchRepChoiceDecoders[kNumStates]; + CMyBitDecoder2 m_MatchRep1ChoiceDecoders[kNumStates]; + CMyBitDecoder2 m_MatchRep2ChoiceDecoders[kNumStates]; + CMyBitDecoder2 m_MatchRepShortChoiceDecoders[kNumStates][NLength::kNumPosStatesMax]; + + CBitTreeDecoder m_PosSlotDecoder[kNumLenToPosStates]; + + CReverseBitTreeDecoder2 m_PosDecoders[kNumPosModels]; + CReverseBitTreeDecoder m_PosAlignDecoder; + // CBitTreeDecoder2 m_PosDecoders[kNumPosModels]; + // CBitTreeDecoder m_PosAlignDecoder; + + NLength::CDecoder m_LenDecoder; + NLength::CDecoder m_RepMatchLenDecoder; + + NLiteral::CDecoder m_LiteralDecoder; + + UINT32 m_DictionarySize; + + UINT32 m_PosStateMask; + + HRESULT Create(); + + HRESULT Init(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream); + + HRESULT Flush() { return m_OutWindowStream.Flush(); } + + HRESULT CodeReal(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream, const UINT64 *anInSize, const UINT64 *anOutSize); + +public: + + CDecoder(); + + HRESULT Code(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream, const UINT64 *anInSize, const UINT64 *anOutSize); + HRESULT ReadCoderProperties(ISequentialInStream *anInStream); + + HRESULT SetDictionarySize(UINT32 aDictionarySize); + HRESULT SetLiteralProperties(UINT32 aLiteralPosStateBits, UINT32 aLiteralContextBits); + HRESULT SetPosBitsProperties(UINT32 aNumPosStateBits); +}; + +}} + +#endif diff --git a/hostTools/lzma/compress/LZMAEncoder.cpp b/hostTools/lzma/compress/LZMAEncoder.cpp new file mode 100644 index 0000000..9b30834 --- /dev/null +++ b/hostTools/lzma/compress/LZMAEncoder.cpp @@ -0,0 +1,981 @@ +#include "Portable.h" +#include "LZMAEncoder.h" + +#include "BinTree2Main.h" + +using namespace NCompression; +using namespace NArithmetic; + +#define RETURN_E_OUTOFMEMORY_IF_FALSE(x) { if (!(x)) return E_OUTOFMEMORY; } + +namespace NCompress { +namespace NLZMA { + +BYTE g_FastPos[1024]; + +class CFastPosInit +{ +public: + CFastPosInit() + { + int c = 0; + const int kFastSlots = 20; + c = 0; + for (BYTE aSlotFast = 0; aSlotFast < kFastSlots; aSlotFast++) + { + UINT32 k = (1 << kDistDirectBits[aSlotFast]); + for (UINT32 j = 0; j < k; j++, c++) + g_FastPos[c] = aSlotFast; + } + } +} g_FastPosInit; + +const int kDefaultDictionaryLogSize = 20; +const int kNumFastBytesDefault = 0x20; + +CEncoder::CEncoder(): + m_DictionarySize(1 << kDefaultDictionaryLogSize), + m_DictionarySizePrev(UINT32(-1)), + m_NumFastBytes(kNumFastBytesDefault), + m_NumFastBytesPrev(UINT32(-1)), + m_DistTableSize(kDefaultDictionaryLogSize * 2), + m_PosStateBits(2), + m_PosStateMask(4 - 1), + m_LiteralPosStateBits(0), + m_LiteralContextBits(3) +{ + m_MaxMode = false; + m_FastMode = false; + m_PosAlignEncoder.Create(kNumAlignBits); + for(int i = 0; i < kNumPosModels; i++) + m_PosEncoders[i].Create(kDistDirectBits[kStartPosModelIndex + i]); +} + +HRESULT CEncoder::Create() +{ + if (m_DictionarySize == m_DictionarySizePrev && m_NumFastBytesPrev == m_NumFastBytes) + return S_OK; + RETURN_IF_NOT_S_OK(m_MatchFinder.Create(m_DictionarySize, kNumOpts, m_NumFastBytes, + kMatchMaxLen - m_NumFastBytes)); + m_DictionarySizePrev = m_DictionarySize; + m_NumFastBytesPrev = m_NumFastBytes; + m_LiteralEncoder.Create(m_LiteralPosStateBits, m_LiteralContextBits); + m_LenEncoder.Create(1 << m_PosStateBits); + m_RepMatchLenEncoder.Create(1 << m_PosStateBits); + return S_OK; +} + +HRESULT CEncoder::SetEncoderAlgorithm(UINT32 A) { + UINT32 aMaximize = A; + if (aMaximize > 2) + return E_INVALIDARG; + + m_FastMode = (aMaximize == 0); + m_MaxMode = (aMaximize >= 2); + + return S_OK; +} + +HRESULT CEncoder::SetEncoderNumFastBytes(UINT32 A) { + UINT32 aNumFastBytes = A; + if(aNumFastBytes < 2 || aNumFastBytes > kMatchMaxLen) + return E_INVALIDARG; + + m_NumFastBytes = aNumFastBytes; + + return S_OK; +} + +HRESULT CEncoder::SetDictionarySize(UINT32 aDictionarySize) +{ + if (aDictionarySize > UINT32(1 << kDicLogSizeMax)) + return E_INVALIDARG; + m_DictionarySize = aDictionarySize; + UINT32 aDicLogSize; + for(aDicLogSize = 0; aDicLogSize < kDicLogSizeMax; aDicLogSize++) + if (aDictionarySize <= (UINT32(1) << aDicLogSize)) + break; + m_DistTableSize = aDicLogSize * 2; + return S_OK; +} + +HRESULT CEncoder::SetLiteralProperties(UINT32 aLiteralPosStateBits, UINT32 aLiteralContextBits) +{ + if (aLiteralPosStateBits > kNumLitPosStatesBitsEncodingMax) + return E_INVALIDARG; + if (aLiteralContextBits > kNumLitContextBitsMax) + return E_INVALIDARG; + m_LiteralPosStateBits = aLiteralPosStateBits; + m_LiteralContextBits = aLiteralContextBits; + return S_OK; +} + +HRESULT CEncoder::SetPosBitsProperties(UINT32 aNumPosStateBits) +{ + if (aNumPosStateBits > NLength::kNumPosStatesBitsEncodingMax) + return E_INVALIDARG; + m_PosStateBits = aNumPosStateBits; + m_PosStateMask = (1 << m_PosStateBits) - 1; + return S_OK; +} + + +HRESULT CEncoder::WriteCoderProperties(ISequentialOutStream *anOutStream) +{ + BYTE aByte = (m_PosStateBits * 5 + m_LiteralPosStateBits) * 9 + m_LiteralContextBits; + UINT32 aProcessedSize; + HRESULT aResult = anOutStream->Write(&aByte, sizeof(aByte), &aProcessedSize); + if (aResult != S_OK) + return aResult; + if (aProcessedSize != sizeof(aByte)) + return E_FAIL; + aResult = anOutStream->Write(&m_DictionarySize, sizeof(m_DictionarySize), &aProcessedSize); + if (aResult != S_OK) + return aResult; + if (aProcessedSize != sizeof(m_DictionarySize)) + return E_FAIL; + return S_OK; +} + +HRESULT CEncoder::Init(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream) +{ + CBaseCoder::Init(); + + RETURN_IF_NOT_S_OK(m_MatchFinder.Init(anInStream)); + m_RangeEncoder.Init(anOutStream); + + int i; + for(i = 0; i < kNumStates; i++) + { + for (UINT32 j = 0; j <= m_PosStateMask; j++) + { + m_MainChoiceEncoders[i][j].Init(); + m_MatchRepShortChoiceEncoders[i][j].Init(); + } + m_MatchChoiceEncoders[i].Init(); + m_MatchRepChoiceEncoders[i].Init(); + m_MatchRep1ChoiceEncoders[i].Init(); + m_MatchRep2ChoiceEncoders[i].Init(); + } + + m_LiteralEncoder.Init(); + + // m_RepMatchLenEncoder.Init(); + + for(i = 0; i < kNumLenToPosStates; i++) + m_PosSlotEncoder[i].Init(); + + for(i = 0; i < kNumPosModels; i++) + m_PosEncoders[i].Init(); + + m_LenEncoder.Init(); + m_RepMatchLenEncoder.Init(); + + m_PosAlignEncoder.Init(); + + m_LongestMatchWasFound = false; + m_OptimumEndIndex = 0; + m_OptimumCurrentIndex = 0; + m_AdditionalOffset = 0; + + return S_OK; +} + +void CEncoder::MovePos(UINT32 aNum) +{ + for (;aNum > 0; aNum--) + { + m_MatchFinder.DummyLongestMatch(); + HRESULT aResult = m_MatchFinder.MovePos(); + if (aResult != S_OK) + throw aResult; + m_AdditionalOffset++; + } +} + +UINT32 CEncoder::Backward(UINT32 &aBackRes, UINT32 aCur) +{ + m_OptimumEndIndex = aCur; + UINT32 aPosMem = m_Optimum[aCur].PosPrev; + UINT32 aBackMem = m_Optimum[aCur].BackPrev; + do + { + if (m_Optimum[aCur].Prev1IsChar) + { + m_Optimum[aPosMem].MakeAsChar(); + m_Optimum[aPosMem].PosPrev = aPosMem - 1; + if (m_Optimum[aCur].Prev2) + { + m_Optimum[aPosMem - 1].Prev1IsChar = false; + m_Optimum[aPosMem - 1].PosPrev = m_Optimum[aCur].PosPrev2; + m_Optimum[aPosMem - 1].BackPrev = m_Optimum[aCur].BackPrev2; + } + } + UINT32 aPosPrev = aPosMem; + UINT32 aBackCur = aBackMem; + + aBackMem = m_Optimum[aPosPrev].BackPrev; + aPosMem = m_Optimum[aPosPrev].PosPrev; + + m_Optimum[aPosPrev].BackPrev = aBackCur; + m_Optimum[aPosPrev].PosPrev = aCur; + aCur = aPosPrev; + } + while(aCur > 0); + aBackRes = m_Optimum[0].BackPrev; + m_OptimumCurrentIndex = m_Optimum[0].PosPrev; + return m_OptimumCurrentIndex; +} + +/* +inline UINT32 GetMatchLen(const BYTE *aData, UINT32 aBack, UINT32 aLimit) +{ + aBack++; + for(UINT32 i = 0; i < aLimit && aData[i] == aData[i - aBack]; i++); + return i; +} +*/ + +UINT32 CEncoder::GetOptimum(UINT32 &aBackRes, UINT32 aPosition) +{ + if(m_OptimumEndIndex != m_OptimumCurrentIndex) + { + UINT32 aLen = m_Optimum[m_OptimumCurrentIndex].PosPrev - m_OptimumCurrentIndex; + aBackRes = m_Optimum[m_OptimumCurrentIndex].BackPrev; + m_OptimumCurrentIndex = m_Optimum[m_OptimumCurrentIndex].PosPrev; + return aLen; + } + m_OptimumCurrentIndex = 0; + m_OptimumEndIndex = 0; // test it; + + UINT32 aLenMain; + if (!m_LongestMatchWasFound) + aLenMain = ReadMatchDistances(); + else + { + aLenMain = m_LongestMatchLength; + m_LongestMatchWasFound = false; + } + + + UINT32 aReps[kNumRepDistances]; + UINT32 aRepLens[kNumRepDistances]; + UINT32 RepMaxIndex = 0; + int i; + for(i = 0; i < kNumRepDistances; i++) + { + aReps[i] = m_RepDistances[i]; + aRepLens[i] = m_MatchFinder.GetMatchLen(0 - 1, aReps[i], kMatchMaxLen); + if (i == 0 || aRepLens[i] > aRepLens[RepMaxIndex]) + RepMaxIndex = i; + } + if(aRepLens[RepMaxIndex] > m_NumFastBytes) + { + aBackRes = RepMaxIndex; + MovePos(aRepLens[RepMaxIndex] - 1); + return aRepLens[RepMaxIndex]; + } + + if(aLenMain > m_NumFastBytes) + { + UINT32 aBackMain = (aLenMain < m_NumFastBytes) ? m_MatchDistances[aLenMain] : + m_MatchDistances[m_NumFastBytes]; + aBackRes = aBackMain + kNumRepDistances; + MovePos(aLenMain - 1); + return aLenMain; + } + BYTE aCurrentByte = m_MatchFinder.GetIndexByte(0 - 1); + + m_Optimum[0].State = m_State; + + BYTE aMatchByte; + + aMatchByte = m_MatchFinder.GetIndexByte(0 - m_RepDistances[0] - 1 - 1); + + UINT32 aPosState = (aPosition & m_PosStateMask); + + m_Optimum[1].Price = m_MainChoiceEncoders[m_State.m_Index][aPosState].GetPrice(kMainChoiceLiteralIndex) + + m_LiteralEncoder.GetPrice(aPosition, m_PreviousByte, m_PeviousIsMatch, aMatchByte, aCurrentByte); + m_Optimum[1].MakeAsChar(); + + m_Optimum[1].PosPrev = 0; + + for (i = 0; i < kNumRepDistances; i++) + m_Optimum[0].Backs[i] = aReps[i]; + + UINT32 aMatchPrice = m_MainChoiceEncoders[m_State.m_Index][aPosState].GetPrice(kMainChoiceMatchIndex); + UINT32 aRepMatchPrice = aMatchPrice + + m_MatchChoiceEncoders[m_State.m_Index].GetPrice(kMatchChoiceRepetitionIndex); + + if(aMatchByte == aCurrentByte) + { + UINT32 aShortRepPrice = aRepMatchPrice + GetRepLen1Price(m_State, aPosState); + if(aShortRepPrice < m_Optimum[1].Price) + { + m_Optimum[1].Price = aShortRepPrice; + m_Optimum[1].MakeAsShortRep(); + } + } + if(aLenMain < 2) + { + aBackRes = m_Optimum[1].BackPrev; + return 1; + } + + + UINT32 aNormalMatchPrice = aMatchPrice + + m_MatchChoiceEncoders[m_State.m_Index].GetPrice(kMatchChoiceDistanceIndex); + + if (aLenMain <= aRepLens[RepMaxIndex]) + aLenMain = 0; + + UINT32 aLen; + for(aLen = 2; aLen <= aLenMain; aLen++) + { + m_Optimum[aLen].PosPrev = 0; + m_Optimum[aLen].BackPrev = m_MatchDistances[aLen] + kNumRepDistances; + m_Optimum[aLen].Price = aNormalMatchPrice + + GetPosLenPrice(m_MatchDistances[aLen], aLen, aPosState); + m_Optimum[aLen].Prev1IsChar = false; + } + + if (aLenMain < aRepLens[RepMaxIndex]) + aLenMain = aRepLens[RepMaxIndex]; + + for (; aLen <= aLenMain; aLen++) + m_Optimum[aLen].Price = kIfinityPrice; + + for(i = 0; i < kNumRepDistances; i++) + { + unsigned aRepLen = aRepLens[i]; + for(UINT32 aLenTest = 2; aLenTest <= aRepLen; aLenTest++) + { + UINT32 aCurAndLenPrice = aRepMatchPrice + GetRepPrice(i, aLenTest, m_State, aPosState); + COptimal &anOptimum = m_Optimum[aLenTest]; + if (aCurAndLenPrice < anOptimum.Price) + { + anOptimum.Price = aCurAndLenPrice; + anOptimum.PosPrev = 0; + anOptimum.BackPrev = i; + anOptimum.Prev1IsChar = false; + } + } + } + + UINT32 aCur = 0; + UINT32 aLenEnd = aLenMain; + + while(true) + { + aCur++; + if(aCur == aLenEnd) + return Backward(aBackRes, aCur); + aPosition++; + UINT32 aPosPrev = m_Optimum[aCur].PosPrev; + CState aState; + if (m_Optimum[aCur].Prev1IsChar) + { + aPosPrev--; + if (m_Optimum[aCur].Prev2) + { + aState = m_Optimum[m_Optimum[aCur].PosPrev2].State; + if (m_Optimum[aCur].BackPrev2 < kNumRepDistances) + aState.UpdateRep(); + else + aState.UpdateMatch(); + } + else + aState = m_Optimum[aPosPrev].State; + aState.UpdateChar(); + } + else + aState = m_Optimum[aPosPrev].State; + bool aPrevWasMatch; + if (aPosPrev == aCur - 1) + { + if (m_Optimum[aCur].IsShortRep()) + { + aPrevWasMatch = true; + aState.UpdateShortRep(); + } + else + { + aPrevWasMatch = false; + aState.UpdateChar(); + } + /* + if (m_Optimum[aCur].Prev1IsChar) + for(int i = 0; i < kNumRepDistances; i++) + aReps[i] = m_Optimum[aPosPrev].Backs[i]; + */ + } + else + { + aPrevWasMatch = true; + UINT32 aPos; + if (m_Optimum[aCur].Prev1IsChar && m_Optimum[aCur].Prev2) + { + aPosPrev = m_Optimum[aCur].PosPrev2; + aPos = m_Optimum[aCur].BackPrev2; + aState.UpdateRep(); + } + else + { + aPos = m_Optimum[aCur].BackPrev; + if (aPos < kNumRepDistances) + aState.UpdateRep(); + else + aState.UpdateMatch(); + } + if (aPos < kNumRepDistances) + { + aReps[0] = m_Optimum[aPosPrev].Backs[aPos]; + UINT32 i; + for(i = 1; i <= aPos; i++) + aReps[i] = m_Optimum[aPosPrev].Backs[i - 1]; + for(; i < kNumRepDistances; i++) + aReps[i] = m_Optimum[aPosPrev].Backs[i]; + } + else + { + aReps[0] = (aPos - kNumRepDistances); + for(UINT32 i = 1; i < kNumRepDistances; i++) + aReps[i] = m_Optimum[aPosPrev].Backs[i - 1]; + } + } + m_Optimum[aCur].State = aState; + for(UINT32 i = 0; i < kNumRepDistances; i++) + m_Optimum[aCur].Backs[i] = aReps[i]; + UINT32 aNewLen = ReadMatchDistances(); + if(aNewLen > m_NumFastBytes) + { + m_LongestMatchLength = aNewLen; + m_LongestMatchWasFound = true; + return Backward(aBackRes, aCur); + } + UINT32 aCurPrice = m_Optimum[aCur].Price; + // BYTE aCurrentByte = m_MatchFinder.GetIndexByte(0 - 1); + // BYTE aMatchByte = m_MatchFinder.GetIndexByte(0 - aReps[0] - 1 - 1); + const BYTE *aData = m_MatchFinder.GetPointerToCurrentPos() - 1; + BYTE aCurrentByte = *aData; + BYTE aMatchByte = aData[0 - aReps[0] - 1]; + + UINT32 aPosState = (aPosition & m_PosStateMask); + + UINT32 aCurAnd1Price = aCurPrice + + m_MainChoiceEncoders[aState.m_Index][aPosState].GetPrice(kMainChoiceLiteralIndex) + + m_LiteralEncoder.GetPrice(aPosition, aData[-1], aPrevWasMatch, aMatchByte, aCurrentByte); + + COptimal &aNextOptimum = m_Optimum[aCur + 1]; + + bool aNextIsChar = false; + if (aCurAnd1Price < aNextOptimum.Price) + { + aNextOptimum.Price = aCurAnd1Price; + aNextOptimum.PosPrev = aCur; + aNextOptimum.MakeAsChar(); + aNextIsChar = true; + } + + UINT32 aMatchPrice = aCurPrice + m_MainChoiceEncoders[aState.m_Index][aPosState].GetPrice(kMainChoiceMatchIndex); + UINT32 aRepMatchPrice = aMatchPrice + m_MatchChoiceEncoders[aState.m_Index].GetPrice(kMatchChoiceRepetitionIndex); + + if(aMatchByte == aCurrentByte && + !(aNextOptimum.PosPrev < aCur && aNextOptimum.BackPrev == 0)) + { + UINT32 aShortRepPrice = aRepMatchPrice + GetRepLen1Price(aState, aPosState); + if(aShortRepPrice <= aNextOptimum.Price) + { + aNextOptimum.Price = aShortRepPrice; + aNextOptimum.PosPrev = aCur; + aNextOptimum.MakeAsShortRep(); + // aNextIsChar = false; + } + } + /* + if(aNewLen == 2 && m_MatchDistances[2] >= kDistLimit2) // test it maybe set 2000 ? + continue; + */ + + UINT32 aNumAvailableBytes = m_MatchFinder.GetNumAvailableBytes() + 1; + aNumAvailableBytes = MyMin(kNumOpts - 1 - aCur, aNumAvailableBytes); + + if (aNumAvailableBytes < 2) + continue; + if (aNumAvailableBytes > m_NumFastBytes) + aNumAvailableBytes = m_NumFastBytes; + if (aNumAvailableBytes >= 3 && !aNextIsChar) + { + UINT32 aBackOffset = aReps[0] + 1; + UINT32 aTemp; + for (aTemp = 1; aTemp < aNumAvailableBytes; aTemp++) + if (aData[aTemp] != aData[aTemp - aBackOffset]) + break; + UINT32 aLenTest2 = aTemp - 1; + if (aLenTest2 >= 2) + { + CState aState2 = aState; + aState2.UpdateChar(); + UINT32 aPosStateNext = (aPosition + 1) & m_PosStateMask; + UINT32 aNextRepMatchPrice = aCurAnd1Price + + m_MainChoiceEncoders[aState2.m_Index][aPosStateNext].GetPrice(kMainChoiceMatchIndex) + + m_MatchChoiceEncoders[aState2.m_Index].GetPrice(kMatchChoiceRepetitionIndex); + // for (; aLenTest2 >= 2; aLenTest2--) + { + while(aLenEnd < aCur + 1 + aLenTest2) + m_Optimum[++aLenEnd].Price = kIfinityPrice; + UINT32 aCurAndLenPrice = aNextRepMatchPrice + GetRepPrice( + 0, aLenTest2, aState2, aPosStateNext); + COptimal &anOptimum = m_Optimum[aCur + 1 + aLenTest2]; + if (aCurAndLenPrice < anOptimum.Price) + { + anOptimum.Price = aCurAndLenPrice; + anOptimum.PosPrev = aCur + 1; + anOptimum.BackPrev = 0; + anOptimum.Prev1IsChar = true; + anOptimum.Prev2 = false; + } + } + } + } + for(UINT32 aRepIndex = 0; aRepIndex < kNumRepDistances; aRepIndex++) + { + // UINT32 aRepLen = m_MatchFinder.GetMatchLen(0 - 1, aReps[aRepIndex], aNewLen); // test it; + UINT32 aBackOffset = aReps[aRepIndex] + 1; + UINT32 aLenTest; + for (aLenTest = 0; aLenTest < aNumAvailableBytes; aLenTest++) + if (aData[aLenTest] != aData[aLenTest - aBackOffset]) + break; + for(; aLenTest >= 2; aLenTest--) + { + while(aLenEnd < aCur + aLenTest) + m_Optimum[++aLenEnd].Price = kIfinityPrice; + UINT32 aCurAndLenPrice = aRepMatchPrice + GetRepPrice(aRepIndex, aLenTest, aState, aPosState); + COptimal &anOptimum = m_Optimum[aCur + aLenTest]; + if (aCurAndLenPrice < anOptimum.Price) + { + anOptimum.Price = aCurAndLenPrice; + anOptimum.PosPrev = aCur; + anOptimum.BackPrev = aRepIndex; + anOptimum.Prev1IsChar = false; + } + + /* + if (m_MaxMode) + { + UINT32 aTemp; + for (aTemp = aLenTest + 1; aTemp < aNumAvailableBytes; aTemp++) + if (aData[aTemp] != aData[aTemp - aBackOffset]) + break; + UINT32 aLenTest2 = aTemp - (aLenTest + 1); + if (aLenTest2 >= 2) + { + CState aState2 = aState; + aState2.UpdateRep(); + UINT32 aPosStateNext = (aPosition + aLenTest) & m_PosStateMask; + UINT32 aCurAndLenCharPrice = aCurAndLenPrice + + m_MainChoiceEncoders[aState2.m_Index][aPosStateNext].GetPrice(kMainChoiceLiteralIndex) + + m_LiteralEncoder.GetPrice(aPosition + aLenTest, aData[aLenTest - 1], + true, aData[aLenTest - aBackOffset], aData[aLenTest]); + aState2.UpdateChar(); + aPosStateNext = (aPosition + aLenTest + 1) & m_PosStateMask; + UINT32 aNextMatchPrice = aCurAndLenCharPrice + m_MainChoiceEncoders[aState2.m_Index][aPosStateNext].GetPrice(kMainChoiceMatchIndex); + UINT32 aNextRepMatchPrice = aNextMatchPrice + m_MatchChoiceEncoders[aState2.m_Index].GetPrice(kMatchChoiceRepetitionIndex); + + // for(; aLenTest2 >= 2; aLenTest2--) + { + UINT32 anOffset = aLenTest + 1 + aLenTest2; + while(aLenEnd < aCur + anOffset) + m_Optimum[++aLenEnd].Price = kIfinityPrice; + UINT32 aCurAndLenPrice = aNextRepMatchPrice + GetRepPrice( + 0, aLenTest2, aState2, aPosStateNext); + COptimal &anOptimum = m_Optimum[aCur + anOffset]; + if (aCurAndLenPrice < anOptimum.Price) + { + anOptimum.Price = aCurAndLenPrice; + anOptimum.PosPrev = aCur + aLenTest + 1; + anOptimum.BackPrev = 0; + anOptimum.Prev1IsChar = true; + anOptimum.Prev2 = true; + anOptimum.PosPrev2 = aCur; + anOptimum.BackPrev2 = aRepIndex; + } + } + } + } + */ + } + } + + // for(UINT32 aLenTest = 2; aLenTest <= aNewLen; aLenTest++) + if (aNewLen > aNumAvailableBytes) + aNewLen = aNumAvailableBytes; + if (aNewLen >= 2) + { + if (aNewLen == 2 && m_MatchDistances[2] >= 0x80) + continue; + UINT32 aNormalMatchPrice = aMatchPrice + + m_MatchChoiceEncoders[aState.m_Index].GetPrice(kMatchChoiceDistanceIndex); + while(aLenEnd < aCur + aNewLen) + m_Optimum[++aLenEnd].Price = kIfinityPrice; + + for(UINT32 aLenTest = aNewLen; aLenTest >= 2; aLenTest--) + { + UINT32 aCurBack = m_MatchDistances[aLenTest]; + UINT32 aCurAndLenPrice = aNormalMatchPrice + GetPosLenPrice(aCurBack, aLenTest, aPosState); + COptimal &anOptimum = m_Optimum[aCur + aLenTest]; + if (aCurAndLenPrice < anOptimum.Price) + { + anOptimum.Price = aCurAndLenPrice; + anOptimum.PosPrev = aCur; + anOptimum.BackPrev = aCurBack + kNumRepDistances; + anOptimum.Prev1IsChar = false; + } + + if (m_MaxMode) + { + UINT32 aBackOffset = aCurBack + 1; + UINT32 aTemp; + for (aTemp = aLenTest + 1; aTemp < aNumAvailableBytes; aTemp++) + if (aData[aTemp] != aData[aTemp - aBackOffset]) + break; + UINT32 aLenTest2 = aTemp - (aLenTest + 1); + if (aLenTest2 >= 2) + { + CState aState2 = aState; + aState2.UpdateMatch(); + UINT32 aPosStateNext = (aPosition + aLenTest) & m_PosStateMask; + UINT32 aCurAndLenCharPrice = aCurAndLenPrice + + m_MainChoiceEncoders[aState2.m_Index][aPosStateNext].GetPrice(kMainChoiceLiteralIndex) + + m_LiteralEncoder.GetPrice(aPosition + aLenTest, aData[aLenTest - 1], + true, aData[aLenTest - aBackOffset], aData[aLenTest]); + aState2.UpdateChar(); + aPosStateNext = (aPosition + aLenTest + 1) & m_PosStateMask; + UINT32 aNextMatchPrice = aCurAndLenCharPrice + m_MainChoiceEncoders[aState2.m_Index][aPosStateNext].GetPrice(kMainChoiceMatchIndex); + UINT32 aNextRepMatchPrice = aNextMatchPrice + m_MatchChoiceEncoders[aState2.m_Index].GetPrice(kMatchChoiceRepetitionIndex); + + // for(; aLenTest2 >= 2; aLenTest2--) + { + UINT32 anOffset = aLenTest + 1 + aLenTest2; + while(aLenEnd < aCur + anOffset) + m_Optimum[++aLenEnd].Price = kIfinityPrice; + UINT32 aCurAndLenPrice = aNextRepMatchPrice + GetRepPrice( + 0, aLenTest2, aState2, aPosStateNext); + COptimal &anOptimum = m_Optimum[aCur + anOffset]; + if (aCurAndLenPrice < anOptimum.Price) + { + anOptimum.Price = aCurAndLenPrice; + anOptimum.PosPrev = aCur + aLenTest + 1; + anOptimum.BackPrev = 0; + anOptimum.Prev1IsChar = true; + anOptimum.Prev2 = true; + anOptimum.PosPrev2 = aCur; + anOptimum.BackPrev2 = aCurBack + kNumRepDistances; + } + } + } + } + } + } + } +} + +static bool inline ChangePair(UINT32 aSmall, UINT32 aBig) +{ + const int kDif = 7; + return (aSmall < (UINT32(1) << (32-kDif)) && aBig >= (aSmall << kDif)); +} + + +UINT32 CEncoder::GetOptimumFast(UINT32 &aBackRes, UINT32 aPosition) +{ + UINT32 aLenMain; + if (!m_LongestMatchWasFound) + aLenMain = ReadMatchDistances(); + else + { + aLenMain = m_LongestMatchLength; + m_LongestMatchWasFound = false; + } + UINT32 aRepLens[kNumRepDistances]; + UINT32 RepMaxIndex = 0; + for(int i = 0; i < kNumRepDistances; i++) + { + aRepLens[i] = m_MatchFinder.GetMatchLen(0 - 1, m_RepDistances[i], kMatchMaxLen); + if (i == 0 || aRepLens[i] > aRepLens[RepMaxIndex]) + RepMaxIndex = i; + } + if(aRepLens[RepMaxIndex] >= m_NumFastBytes) + { + aBackRes = RepMaxIndex; + MovePos(aRepLens[RepMaxIndex] - 1); + return aRepLens[RepMaxIndex]; + } + if(aLenMain >= m_NumFastBytes) + { + aBackRes = m_MatchDistances[m_NumFastBytes] + kNumRepDistances; + MovePos(aLenMain - 1); + return aLenMain; + } + while (aLenMain > 2) + { + if (!ChangePair(m_MatchDistances[aLenMain - 1], + m_MatchDistances[aLenMain])) + break; + aLenMain--; + } + if (aLenMain == 2 && m_MatchDistances[2] >= 0x80) + aLenMain = 1; + + UINT32 aBackMain = m_MatchDistances[aLenMain]; + if (aRepLens[RepMaxIndex] >= 2) + { + if (aRepLens[RepMaxIndex] + 1 >= aLenMain || + aRepLens[RepMaxIndex] + 2 >= aLenMain && (aBackMain > (1<<12))) + { + aBackRes = RepMaxIndex; + MovePos(aRepLens[RepMaxIndex] - 1); + return aRepLens[RepMaxIndex]; + } + } + + + if (aLenMain >= 2) + { + m_LongestMatchLength = ReadMatchDistances(); + if (m_LongestMatchLength >= 2 && + ( + (m_LongestMatchLength >= aLenMain && + m_MatchDistances[aLenMain] < aBackMain) || + m_LongestMatchLength == aLenMain + 1 && + !ChangePair(aBackMain, m_MatchDistances[m_LongestMatchLength]) || + m_LongestMatchLength > aLenMain + 1 || + m_LongestMatchLength + 1 >= aLenMain && + ChangePair(m_MatchDistances[aLenMain - 1], aBackMain) + ) + ) + { + m_LongestMatchWasFound = true; + aBackRes = UINT32(-1); + return 1; + } + for(int i = 0; i < kNumRepDistances; i++) + { + UINT32 aRepLen = m_MatchFinder.GetMatchLen(0 - 1, m_RepDistances[i], kMatchMaxLen); + if (aRepLen >= 2 && aRepLen + 1 >= aLenMain) + { + m_LongestMatchWasFound = true; + aBackRes = UINT32(-1); + return 1; + } + } + aBackRes = aBackMain + kNumRepDistances; + MovePos(aLenMain - 2); + return aLenMain; + } + aBackRes = UINT32(-1); + return 1; +} + +HRESULT CEncoder::Flush() +{ + m_RangeEncoder.FlushData(); + return m_RangeEncoder.FlushStream(); +} + +HRESULT CEncoder::CodeReal(ISequentialInStream *anInStream, + ISequentialOutStream *anOutStream, + const UINT64 *anInSize) +{ + RETURN_IF_NOT_S_OK(Create()); + Init(anInStream, anOutStream); + + if (m_MatchFinder.GetNumAvailableBytes() == 0) + return Flush(); + + if (!m_FastMode) + { + FillPosSlotPrices(); + FillDistancesPrices(); + FillAlignPrices(); + } + + m_LenEncoder.SetTableSize(m_NumFastBytes); + m_LenEncoder.UpdateTables(); + m_RepMatchLenEncoder.SetTableSize(m_NumFastBytes); + m_RepMatchLenEncoder.UpdateTables(); + + UINT64 aLastPosSlotFillingPos = 0; + + UINT64 aNowPos64 = 0; + ReadMatchDistances(); + UINT32 aPosState = UINT32(aNowPos64) & m_PosStateMask; + m_MainChoiceEncoders[m_State.m_Index][aPosState].Encode(&m_RangeEncoder, kMainChoiceLiteralIndex); + m_State.UpdateChar(); + BYTE aByte = m_MatchFinder.GetIndexByte(0 - m_AdditionalOffset); + m_LiteralEncoder.Encode(&m_RangeEncoder, UINT32(aNowPos64), m_PreviousByte, false, 0, aByte); + m_PreviousByte = aByte; + m_AdditionalOffset--; + aNowPos64++; + if (m_MatchFinder.GetNumAvailableBytes() == 0) + return Flush(); + while(true) + { + UINT32 aPos; + UINT32 aPosState = UINT32(aNowPos64) & m_PosStateMask; + + UINT32 aLen; + if (m_FastMode) + aLen = GetOptimumFast(aPos, UINT32(aNowPos64)); + else + aLen = GetOptimum(aPos, UINT32(aNowPos64)); + + if(aLen == 1 && aPos == (-1)) + { + m_MainChoiceEncoders[m_State.m_Index][aPosState].Encode(&m_RangeEncoder, kMainChoiceLiteralIndex); + m_State.UpdateChar(); + BYTE aMatchByte; + if(m_PeviousIsMatch) + aMatchByte = m_MatchFinder.GetIndexByte(0 - m_RepDistances[0] - 1 - m_AdditionalOffset); + BYTE aByte = m_MatchFinder.GetIndexByte(0 - m_AdditionalOffset); + m_LiteralEncoder.Encode(&m_RangeEncoder, UINT32(aNowPos64), m_PreviousByte, m_PeviousIsMatch, aMatchByte, aByte); + m_PreviousByte = aByte; + m_PeviousIsMatch = false; + } + else + { + m_PeviousIsMatch = true; + m_MainChoiceEncoders[m_State.m_Index][aPosState].Encode(&m_RangeEncoder, kMainChoiceMatchIndex); + if(aPos < kNumRepDistances) + { + m_MatchChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, kMatchChoiceRepetitionIndex); + if(aPos == 0) + { + m_MatchRepChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, 0); + if(aLen == 1) + m_MatchRepShortChoiceEncoders[m_State.m_Index][aPosState].Encode(&m_RangeEncoder, 0); + else + m_MatchRepShortChoiceEncoders[m_State.m_Index][aPosState].Encode(&m_RangeEncoder, 1); + } + else + { + m_MatchRepChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, 1); + if (aPos == 1) + m_MatchRep1ChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, 0); + else + { + m_MatchRep1ChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, 1); + m_MatchRep2ChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, aPos - 2); + } + } + if (aLen == 1) + m_State.UpdateShortRep(); + else + { + m_RepMatchLenEncoder.Encode(&m_RangeEncoder, aLen - kMatchMinLen, aPosState); + m_State.UpdateRep(); + } + + + UINT32 aDistance = m_RepDistances[aPos]; + if (aPos != 0) + { + for(UINT32 i = aPos; i >= 1; i--) + m_RepDistances[i] = m_RepDistances[i - 1]; + m_RepDistances[0] = aDistance; + } + } + else + { + m_MatchChoiceEncoders[m_State.m_Index].Encode(&m_RangeEncoder, kMatchChoiceDistanceIndex); + m_State.UpdateMatch(); + m_LenEncoder.Encode(&m_RangeEncoder, aLen - kMatchMinLen, aPosState); + aPos -= kNumRepDistances; + UINT32 aPosSlot = GetPosSlot(aPos); + UINT32 aLenToPosState = GetLenToPosState(aLen); + m_PosSlotEncoder[aLenToPosState].Encode(&m_RangeEncoder, aPosSlot); + + UINT32 aFooterBits = kDistDirectBits[aPosSlot]; + UINT32 aPosReduced = aPos - kDistStart[aPosSlot]; + if (aPosSlot >= kStartPosModelIndex) + { + if (aPosSlot < kEndPosModelIndex) + m_PosEncoders[aPosSlot - kStartPosModelIndex].Encode(&m_RangeEncoder, aPosReduced); + else + { + m_RangeEncoder.EncodeDirectBits(aPosReduced >> kNumAlignBits, aFooterBits - kNumAlignBits); + m_PosAlignEncoder.Encode(&m_RangeEncoder, aPosReduced & kAlignMask); + if (!m_FastMode) + if (--m_AlignPriceCount == 0) + FillAlignPrices(); + } + } + UINT32 aDistance = aPos; + for(UINT32 i = kNumRepDistances - 1; i >= 1; i--) + m_RepDistances[i] = m_RepDistances[i - 1]; + m_RepDistances[0] = aDistance; + } + m_PreviousByte = m_MatchFinder.GetIndexByte(aLen - 1 - m_AdditionalOffset); + } + m_AdditionalOffset -= aLen; + aNowPos64 += aLen; + if (!m_FastMode) + if (aNowPos64 - aLastPosSlotFillingPos >= (1 << 9)) + { + FillPosSlotPrices(); + FillDistancesPrices(); + aLastPosSlotFillingPos = aNowPos64; + } + if (m_AdditionalOffset == 0 && m_MatchFinder.GetNumAvailableBytes() == 0) + return Flush(); + } +} + +HRESULT CEncoder::Code(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream, const UINT64 *anInSize) +{ + try { + return CodeReal(anInStream, anOutStream, anInSize); + } catch (HRESULT& e) { + return e; + } catch (...) { + return E_FAIL; + } +} + +void CEncoder::FillPosSlotPrices() +{ + for (int aLenToPosState = 0; aLenToPosState < kNumLenToPosStates; aLenToPosState++) + { + UINT32 aPosSlot; + for (aPosSlot = 0; aPosSlot < kEndPosModelIndex && aPosSlot < m_DistTableSize; aPosSlot++) + m_PosSlotPrices[aLenToPosState][aPosSlot] = m_PosSlotEncoder[aLenToPosState].GetPrice(aPosSlot); + for (; aPosSlot < m_DistTableSize; aPosSlot++) + m_PosSlotPrices[aLenToPosState][aPosSlot] = m_PosSlotEncoder[aLenToPosState].GetPrice(aPosSlot) + + ((kDistDirectBits[aPosSlot] - kNumAlignBits) << kNumBitPriceShiftBits); + } +} + +void CEncoder::FillDistancesPrices() +{ + for (int aLenToPosState = 0; aLenToPosState < kNumLenToPosStates; aLenToPosState++) + { + UINT32 i; + for (i = 0; i < kStartPosModelIndex; i++) + m_DistancesPrices[aLenToPosState][i] = m_PosSlotPrices[aLenToPosState][i]; + for (; i < kNumFullDistances; i++) + { + UINT32 aPosSlot = GetPosSlot(i); + m_DistancesPrices[aLenToPosState][i] = m_PosSlotPrices[aLenToPosState][aPosSlot] + + m_PosEncoders[aPosSlot - kStartPosModelIndex].GetPrice(i - kDistStart[aPosSlot]); + } + } +} + +void CEncoder::FillAlignPrices() +{ + for (int i = 0; i < kAlignTableSize; i++) + m_AlignPrices[i] = m_PosAlignEncoder.GetPrice(i); + m_AlignPriceCount = kAlignTableSize; +} + +}} diff --git a/hostTools/lzma/compress/LZMAEncoder.h b/hostTools/lzma/compress/LZMAEncoder.h new file mode 100644 index 0000000..6368738 --- /dev/null +++ b/hostTools/lzma/compress/LZMAEncoder.h @@ -0,0 +1,228 @@ +#ifndef __LZARITHMETIC_ENCODER_H +#define __LZARITHMETIC_ENCODER_H + +#include "Portable.h" +#include "AriPrice.h" +#include "LZMA.h" +#include "LenCoder.h" +#include "LiteralCoder.h" +#include "AriConst.h" + +// NOTE Here is choosen the MatchFinder +#include "BinTree2.h" +#define MATCH_FINDER NBT2::CMatchFinderBinTree + +namespace NCompress { +namespace NLZMA { + +struct COptimal +{ + CState State; + + bool Prev1IsChar; + bool Prev2; + + UINT32 PosPrev2; + UINT32 BackPrev2; + + UINT32 Price; + UINT32 PosPrev; // posNext; + UINT32 BackPrev; + UINT32 Backs[kNumRepDistances]; + void MakeAsChar() { BackPrev = UINT32(-1); Prev1IsChar = false; } + void MakeAsShortRep() { BackPrev = 0; ; Prev1IsChar = false; } + bool IsShortRep() { return (BackPrev == 0); } +}; + + +extern BYTE g_FastPos[1024]; +inline UINT32 GetPosSlot(UINT32 aPos) +{ + if (aPos < (1 << 10)) + return g_FastPos[aPos]; + if (aPos < (1 << 19)) + return g_FastPos[aPos >> 9] + 18; + return g_FastPos[aPos >> 18] + 36; +} + +inline UINT32 GetPosSlot2(UINT32 aPos) +{ + if (aPos < (1 << 16)) + return g_FastPos[aPos >> 6] + 12; + if (aPos < (1 << 25)) + return g_FastPos[aPos >> 15] + 30; + return g_FastPos[aPos >> 24] + 48; +} + +const int kIfinityPrice = 0xFFFFFFF; + +typedef CMyBitEncoder CMyBitEncoder2; + +const int kNumOpts = 1 << 12; + +class CEncoder : public CBaseCoder +{ + COptimal m_Optimum[kNumOpts]; +public: + MATCH_FINDER m_MatchFinder; + CMyRangeEncoder m_RangeEncoder; +private: + + CMyBitEncoder2 m_MainChoiceEncoders[kNumStates][NLength::kNumPosStatesEncodingMax]; + CMyBitEncoder2 m_MatchChoiceEncoders[kNumStates]; + CMyBitEncoder2 m_MatchRepChoiceEncoders[kNumStates]; + CMyBitEncoder2 m_MatchRep1ChoiceEncoders[kNumStates]; + CMyBitEncoder2 m_MatchRep2ChoiceEncoders[kNumStates]; + CMyBitEncoder2 m_MatchRepShortChoiceEncoders[kNumStates][NLength::kNumPosStatesEncodingMax]; + + CBitTreeEncoder m_PosSlotEncoder[kNumLenToPosStates]; + + CReverseBitTreeEncoder2 m_PosEncoders[kNumPosModels]; + CReverseBitTreeEncoder2 m_PosAlignEncoder; + // CBitTreeEncoder2 m_PosEncoders[kNumPosModels]; + // CBitTreeEncoder2 m_PosAlignEncoder; + + NLength::CPriceTableEncoder m_LenEncoder; + NLength::CPriceTableEncoder m_RepMatchLenEncoder; + + NLiteral::CEncoder m_LiteralEncoder; + + UINT32 m_MatchDistances[kMatchMaxLen + 1]; + + bool m_FastMode; + bool m_MaxMode; + UINT32 m_NumFastBytes; + UINT32 m_LongestMatchLength; + + UINT32 m_AdditionalOffset; + + UINT32 m_OptimumEndIndex; + UINT32 m_OptimumCurrentIndex; + + bool m_LongestMatchWasFound; + + UINT32 m_PosSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; + + UINT32 m_DistancesPrices[kNumLenToPosStates][kNumFullDistances]; + + UINT32 m_AlignPrices[kAlignTableSize]; + UINT32 m_AlignPriceCount; + + UINT32 m_DistTableSize; + + UINT32 m_PosStateBits; + UINT32 m_PosStateMask; + UINT32 m_LiteralPosStateBits; + UINT32 m_LiteralContextBits; + + UINT32 m_DictionarySize; + + + UINT32 m_DictionarySizePrev; + UINT32 m_NumFastBytesPrev; + + + UINT32 ReadMatchDistances() + { + UINT32 aLen = m_MatchFinder.GetLongestMatch(m_MatchDistances); + if (aLen == m_NumFastBytes) + aLen += m_MatchFinder.GetMatchLen(aLen, m_MatchDistances[aLen], + kMatchMaxLen - aLen); + m_AdditionalOffset++; + HRESULT aResult = m_MatchFinder.MovePos(); + if (aResult != S_OK) + throw aResult; + return aLen; + } + + void MovePos(UINT32 aNum); + UINT32 GetRepLen1Price(CState aState, UINT32 aPosState) const + { + return m_MatchRepChoiceEncoders[aState.m_Index].GetPrice(0) + + m_MatchRepShortChoiceEncoders[aState.m_Index][aPosState].GetPrice(0); + } + UINT32 GetRepPrice(UINT32 aRepIndex, UINT32 aLen, CState aState, UINT32 aPosState) const + { + UINT32 aPrice = m_RepMatchLenEncoder.GetPrice(aLen - kMatchMinLen, aPosState); + if(aRepIndex == 0) + { + aPrice += m_MatchRepChoiceEncoders[aState.m_Index].GetPrice(0); + aPrice += m_MatchRepShortChoiceEncoders[aState.m_Index][aPosState].GetPrice(1); + } + else + { + aPrice += m_MatchRepChoiceEncoders[aState.m_Index].GetPrice(1); + if (aRepIndex == 1) + aPrice += m_MatchRep1ChoiceEncoders[aState.m_Index].GetPrice(0); + else + { + aPrice += m_MatchRep1ChoiceEncoders[aState.m_Index].GetPrice(1); + aPrice += m_MatchRep2ChoiceEncoders[aState.m_Index].GetPrice(aRepIndex - 2); + } + } + return aPrice; + } + /* + UINT32 GetPosLen2Price(UINT32 aPos, UINT32 aPosState) const + { + if (aPos >= kNumFullDistances) + return kIfinityPrice; + return m_DistancesPrices[0][aPos] + m_LenEncoder.GetPrice(0, aPosState); + } + UINT32 GetPosLen3Price(UINT32 aPos, UINT32 aLen, UINT32 aPosState) const + { + UINT32 aPrice; + UINT32 aLenToPosState = GetLenToPosState(aLen); + if (aPos < kNumFullDistances) + aPrice = m_DistancesPrices[aLenToPosState][aPos]; + else + aPrice = m_PosSlotPrices[aLenToPosState][GetPosSlot2(aPos)] + + m_AlignPrices[aPos & kAlignMask]; + return aPrice + m_LenEncoder.GetPrice(aLen - kMatchMinLen, aPosState); + } + */ + UINT32 GetPosLenPrice(UINT32 aPos, UINT32 aLen, UINT32 aPosState) const + { + if (aLen == 2 && aPos >= 0x80) + return kIfinityPrice; + UINT32 aPrice; + UINT32 aLenToPosState = GetLenToPosState(aLen); + if (aPos < kNumFullDistances) + aPrice = m_DistancesPrices[aLenToPosState][aPos]; + else + aPrice = m_PosSlotPrices[aLenToPosState][GetPosSlot2(aPos)] + + m_AlignPrices[aPos & kAlignMask]; + return aPrice + m_LenEncoder.GetPrice(aLen - kMatchMinLen, aPosState); + } + + UINT32 Backward(UINT32 &aBackRes, UINT32 aCur); + UINT32 GetOptimum(UINT32 &aBackRes, UINT32 aPosition); + UINT32 GetOptimumFast(UINT32 &aBackRes, UINT32 aPosition); + + void FillPosSlotPrices(); + void FillDistancesPrices(); + void FillAlignPrices(); + + HRESULT Flush(); + + HRESULT Create(); + + HRESULT CodeReal(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream, const UINT64 *anInSize); + + HRESULT Init(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream); + +public: + CEncoder(); + + HRESULT SetEncoderAlgorithm(UINT32 A); + HRESULT SetEncoderNumFastBytes(UINT32 A); + HRESULT SetDictionarySize(UINT32 aDictionarySize); + HRESULT SetLiteralProperties(UINT32 aLiteralPosStateBits, UINT32 aLiteralContextBits); + HRESULT SetPosBitsProperties(UINT32 aNumPosStateBits); + HRESULT Code(ISequentialInStream *anInStream, ISequentialOutStream *anOutStream, const UINT64 *anInSize); + HRESULT WriteCoderProperties(ISequentialOutStream *anOutStream); +}; + +}} + +#endif diff --git a/hostTools/lzma/compress/LenCoder.cpp b/hostTools/lzma/compress/LenCoder.cpp new file mode 100644 index 0000000..35162b2 --- /dev/null +++ b/hostTools/lzma/compress/LenCoder.cpp @@ -0,0 +1,73 @@ +#include "LenCoder.h" + +using namespace NCompression; +using namespace NArithmetic; + +namespace NLength { + +void CEncoder::Init() +{ + m_Choice.Init(); + for (UINT32 aPosState = 0; aPosState < m_NumPosStates; aPosState++) + { + m_LowCoder[aPosState].Init(); + m_MidCoder[aPosState].Init(); + } + m_Choice2.Init(); + m_HighCoder.Init(); +} + +void CEncoder::Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aSymbol, UINT32 aPosState) +{ + if(aSymbol < kNumLowSymbols) + { + m_Choice.Encode(aRangeEncoder, 0); + m_LowCoder[aPosState].Encode(aRangeEncoder, aSymbol); + } + else + { + aSymbol -= kNumLowSymbols; + m_Choice.Encode(aRangeEncoder, 1); + if(aSymbol < kNumMidSymbols) + { + m_Choice2.Encode(aRangeEncoder, 0); + m_MidCoder[aPosState].Encode(aRangeEncoder, aSymbol); + } + else + { + aSymbol -= kNumMidSymbols; + m_Choice2.Encode(aRangeEncoder, 1); + m_HighCoder.Encode(aRangeEncoder, aSymbol); + } + } +} + +UINT32 CEncoder::GetPrice(UINT32 aSymbol, UINT32 aPosState) const +{ + UINT32 aPrice = 0; + if(aSymbol < kNumLowSymbols) + { + aPrice += m_Choice.GetPrice(0); + aPrice += m_LowCoder[aPosState].GetPrice(aSymbol); + } + else + { + aSymbol -= kNumLowSymbols; + aPrice += m_Choice.GetPrice(1); + if(aSymbol < kNumMidSymbols) + { + aPrice += m_Choice2.GetPrice(0); + aPrice += m_MidCoder[aPosState].GetPrice(aSymbol); + } + else + { + aSymbol -= kNumMidSymbols; + aPrice += m_Choice2.GetPrice(1); + aPrice += m_HighCoder.GetPrice(aSymbol); + } + } + return aPrice; +} + +} + diff --git a/hostTools/lzma/compress/LenCoder.h b/hostTools/lzma/compress/LenCoder.h new file mode 100644 index 0000000..ff389b9 --- /dev/null +++ b/hostTools/lzma/compress/LenCoder.h @@ -0,0 +1,122 @@ +#ifndef __LENCODER_H +#define __LENCODER_H + +#include "BitTreeCoder.h" + +namespace NLength { + +const int kNumPosStatesBitsMax = 4; +const int kNumPosStatesMax = (1 << kNumPosStatesBitsMax); + + +const int kNumPosStatesBitsEncodingMax = 4; +const int kNumPosStatesEncodingMax = (1 << kNumPosStatesBitsEncodingMax); + + +const int kNumMoveBits = 5; + +const int kNumLenBits = 3; +const int kNumLowSymbols = 1 << kNumLenBits; +const int kNumMidBits = 3; +const int kNumMidSymbols = 1 << kNumMidBits; + +const int kNumHighBits = 8; + +const int kNumSymbolsTotal = kNumLowSymbols + kNumMidSymbols + (1 << kNumHighBits); + +class CEncoder +{ + CMyBitEncoder m_Choice; + CBitTreeEncoder m_LowCoder[kNumPosStatesEncodingMax]; + CMyBitEncoder m_Choice2; + CBitTreeEncoder m_MidCoder[kNumPosStatesEncodingMax]; + CBitTreeEncoder m_HighCoder; +protected: + UINT32 m_NumPosStates; +public: + void Create(UINT32 aNumPosStates) + { m_NumPosStates = aNumPosStates; } + void Init(); + void Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aSymbol, UINT32 aPosState); + + UINT32 GetPrice(UINT32 aSymbol, UINT32 aPosState) const; +}; + +const int kNumSpecSymbols = kNumLowSymbols + kNumMidSymbols; + +class CPriceTableEncoder: public CEncoder +{ + UINT32 m_Prices[kNumSymbolsTotal][kNumPosStatesEncodingMax]; + UINT32 m_TableSize; + UINT32 m_Counters[kNumPosStatesEncodingMax]; +public: + void SetTableSize(UINT32 aTableSize) + { m_TableSize = aTableSize; } + UINT32 GetPrice(UINT32 aSymbol, UINT32 aPosState) const + { return m_Prices[aSymbol][aPosState]; } + void UpdateTable(UINT32 aPosState) + { + for (UINT32 aLen = 0; aLen < m_TableSize; aLen++) + m_Prices[aLen][aPosState] = CEncoder::GetPrice(aLen , aPosState); + m_Counters[aPosState] = m_TableSize; + } + void UpdateTables() + { + for (UINT32 aPosState = 0; aPosState < m_NumPosStates; aPosState++) + UpdateTable(aPosState); + } + void Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aSymbol, UINT32 aPosState) + { + CEncoder::Encode(aRangeEncoder, aSymbol, aPosState); + if (--m_Counters[aPosState] == 0) + UpdateTable(aPosState); + } +}; + + +class CDecoder +{ + CMyBitDecoder m_Choice; + CBitTreeDecoder m_LowCoder[kNumPosStatesMax]; + CMyBitDecoder m_Choice2; + CBitTreeDecoder m_MidCoder[kNumPosStatesMax]; + CBitTreeDecoder m_HighCoder; + UINT32 m_NumPosStates; +public: + void Create(UINT32 aNumPosStates) + { m_NumPosStates = aNumPosStates; } + void Init() + { + m_Choice.Init(); + for (UINT32 aPosState = 0; aPosState < m_NumPosStates; aPosState++) + { + m_LowCoder[aPosState].Init(); + m_MidCoder[aPosState].Init(); + } + m_Choice2.Init(); + m_HighCoder.Init(); + } + UINT32 Decode(CMyRangeDecoder *aRangeDecoder, UINT32 aPosState) + { + if(m_Choice.Decode(aRangeDecoder) == 0) + return m_LowCoder[aPosState].Decode(aRangeDecoder); + else + { + UINT32 aSymbol = kNumLowSymbols; + if(m_Choice2.Decode(aRangeDecoder) == 0) + aSymbol += m_MidCoder[aPosState].Decode(aRangeDecoder); + else + { + aSymbol += kNumMidSymbols; + aSymbol += m_HighCoder.Decode(aRangeDecoder); + } + return aSymbol; + } + } + +}; + +} + + +#endif diff --git a/hostTools/lzma/compress/LiteralCoder.cpp b/hostTools/lzma/compress/LiteralCoder.cpp new file mode 100644 index 0000000..fdb4fe3 --- /dev/null +++ b/hostTools/lzma/compress/LiteralCoder.cpp @@ -0,0 +1,66 @@ +#include "LiteralCoder.h" + +using namespace NCompression; +using namespace NArithmetic; + +namespace NLiteral { + +void CEncoder2::Init() +{ + for (int i = 0; i < 3; i++) + for (int j = 1; j < (1 << 8); j++) + m_Encoders[i][j].Init(); +} + +void CEncoder2::Encode(CMyRangeEncoder *aRangeEncoder, + bool aMatchMode, BYTE aMatchByte, BYTE aSymbol) +{ + UINT32 aContext = 1; + bool aSame = true; + for (int i = 7; i >= 0; i--) + { + UINT32 aBit = (aSymbol >> i) & 1; + unsigned aState; + if (aMatchMode && aSame) + { + UINT32 aMatchBit = (aMatchByte >> i) & 1; + aState = 1 + aMatchBit; + aSame = (aMatchBit == aBit); + } + else + aState = 0; + m_Encoders[aState][aContext].Encode(aRangeEncoder, aBit); + aContext = (aContext << 1) | aBit; + } +} + +UINT32 CEncoder2::GetPrice(bool aMatchMode, BYTE aMatchByte, BYTE aSymbol) const +{ + UINT32 aPrice = 0; + UINT32 aContext = 1; + int i = 7; + if (aMatchMode) + { + for (; i >= 0; i--) + { + UINT32 aMatchBit = (aMatchByte >> i) & 1; + UINT32 aBit = (aSymbol >> i) & 1; + aPrice += m_Encoders[1 + aMatchBit][aContext].GetPrice(aBit); + aContext = (aContext << 1) | aBit; + if (aMatchBit != aBit) + { + i--; + break; + } + } + } + for (; i >= 0; i--) + { + UINT32 aBit = (aSymbol >> i) & 1; + aPrice += m_Encoders[0][aContext].GetPrice(aBit); + aContext = (aContext << 1) | aBit; + } + return aPrice; +}; + +} diff --git a/hostTools/lzma/compress/LiteralCoder.h b/hostTools/lzma/compress/LiteralCoder.h new file mode 100644 index 0000000..147bf03 --- /dev/null +++ b/hostTools/lzma/compress/LiteralCoder.h @@ -0,0 +1,160 @@ +#ifndef __LITERALCODER_H +#define __LITERALCODER_H + +#include "AriBitCoder.h" +#include "RCDefs.h" + +namespace NLiteral { + +const int kNumMoveBits = 5; + +class CEncoder2 +{ + CMyBitEncoder m_Encoders[3][1 << 8]; +public: + void Init(); + void Encode(CMyRangeEncoder *aRangeEncoder, bool aMatchMode, BYTE aMatchByte, BYTE aSymbol); + UINT32 GetPrice(bool aMatchMode, BYTE aMatchByte, BYTE aSymbol) const; +}; + +class CDecoder2 +{ + CMyBitDecoder m_Decoders[3][1 << 8]; +public: + void Init() + { + for (int i = 0; i < 3; i++) + for (int j = 1; j < (1 << 8); j++) + m_Decoders[i][j].Init(); + } + + BYTE DecodeNormal(CMyRangeDecoder *aRangeDecoder) + { + UINT32 aSymbol = 1; + RC_INIT_VAR + do + { + // aSymbol = (aSymbol << 1) | m_Decoders[0][aSymbol].Decode(aRangeDecoder); + RC_GETBIT(kNumMoveBits, m_Decoders[0][aSymbol].m_Probability, aSymbol) + } + while (aSymbol < 0x100); + RC_FLUSH_VAR + return aSymbol; + } + + BYTE DecodeWithMatchByte(CMyRangeDecoder *aRangeDecoder, BYTE aMatchByte) + { + UINT32 aSymbol = 1; + RC_INIT_VAR + do + { + UINT32 aMatchBit = (aMatchByte >> 7) & 1; + aMatchByte <<= 1; + // UINT32 aBit = m_Decoders[1 + aMatchBit][aSymbol].Decode(aRangeDecoder); + // aSymbol = (aSymbol << 1) | aBit; + UINT32 aBit; + RC_GETBIT2(kNumMoveBits, m_Decoders[1 + aMatchBit][aSymbol].m_Probability, aSymbol, + aBit = 0, aBit = 1) + if (aMatchBit != aBit) + { + while (aSymbol < 0x100) + { + // aSymbol = (aSymbol << 1) | m_Decoders[0][aSymbol].Decode(aRangeDecoder); + RC_GETBIT(kNumMoveBits, m_Decoders[0][aSymbol].m_Probability, aSymbol) + } + break; + } + } + while (aSymbol < 0x100); + RC_FLUSH_VAR + return aSymbol; + } +}; + +/* +const UINT32 kNumPrevByteBits = 1; +const UINT32 kNumPrevByteStates = (1 << kNumPrevByteBits); + +inline UINT32 GetLiteralState(BYTE aPrevByte) + { return (aPrevByte >> (8 - kNumPrevByteBits)); } +*/ + +class CEncoder +{ + CEncoder2 *m_Coders; + UINT32 m_NumPrevBits; + UINT32 m_NumPosBits; + UINT32 m_PosMask; +public: + CEncoder(): m_Coders(0) {} + ~CEncoder() { Free(); } + void Free() + { + delete []m_Coders; + m_Coders = 0; + } + void Create(UINT32 aNumPosBits, UINT32 aNumPrevBits) + { + Free(); + m_NumPosBits = aNumPosBits; + m_PosMask = (1 << aNumPosBits) - 1; + m_NumPrevBits = aNumPrevBits; + UINT32 aNumStates = 1 << (m_NumPrevBits + m_NumPosBits); + m_Coders = new CEncoder2[aNumStates]; + } + void Init() + { + UINT32 aNumStates = 1 << (m_NumPrevBits + m_NumPosBits); + for (UINT32 i = 0; i < aNumStates; i++) + m_Coders[i].Init(); + } + UINT32 GetState(UINT32 aPos, BYTE aPrevByte) const + { return ((aPos & m_PosMask) << m_NumPrevBits) + (aPrevByte >> (8 - m_NumPrevBits)); } + void Encode(CMyRangeEncoder *aRangeEncoder, UINT32 aPos, BYTE aPrevByte, + bool aMatchMode, BYTE aMatchByte, BYTE aSymbol) + { m_Coders[GetState(aPos, aPrevByte)].Encode(aRangeEncoder, aMatchMode, + aMatchByte, aSymbol); } + UINT32 GetPrice(UINT32 aPos, BYTE aPrevByte, bool aMatchMode, BYTE aMatchByte, BYTE aSymbol) const + { return m_Coders[GetState(aPos, aPrevByte)].GetPrice(aMatchMode, aMatchByte, aSymbol); } +}; + +class CDecoder +{ + CDecoder2 *m_Coders; + UINT32 m_NumPrevBits; + UINT32 m_NumPosBits; + UINT32 m_PosMask; +public: + CDecoder(): m_Coders(0) {} + ~CDecoder() { Free(); } + void Free() + { + delete []m_Coders; + m_Coders = 0; + } + void Create(UINT32 aNumPosBits, UINT32 aNumPrevBits) + { + Free(); + m_NumPosBits = aNumPosBits; + m_PosMask = (1 << aNumPosBits) - 1; + m_NumPrevBits = aNumPrevBits; + UINT32 aNumStates = 1 << (m_NumPrevBits + m_NumPosBits); + m_Coders = new CDecoder2[aNumStates]; + } + void Init() + { + UINT32 aNumStates = 1 << (m_NumPrevBits + m_NumPosBits); + for (UINT32 i = 0; i < aNumStates; i++) + m_Coders[i].Init(); + } + UINT32 GetState(UINT32 aPos, BYTE aPrevByte) const + { return ((aPos & m_PosMask) << m_NumPrevBits) + (aPrevByte >> (8 - m_NumPrevBits)); } + BYTE DecodeNormal(CMyRangeDecoder *aRangeDecoder, UINT32 aPos, BYTE aPrevByte) + { return m_Coders[GetState(aPos, aPrevByte)].DecodeNormal(aRangeDecoder); } + BYTE DecodeWithMatchByte(CMyRangeDecoder *aRangeDecoder, UINT32 aPos, BYTE aPrevByte, BYTE aMatchByte) + { return m_Coders[GetState(aPos, aPrevByte)].DecodeWithMatchByte(aRangeDecoder, aMatchByte); } +}; + +} + +#endif diff --git a/hostTools/lzma/compress/OutByte.cpp b/hostTools/lzma/compress/OutByte.cpp new file mode 100644 index 0000000..a658798 --- /dev/null +++ b/hostTools/lzma/compress/OutByte.cpp @@ -0,0 +1,45 @@ +#include "OutByte.h" + +namespace NStream { + +COutByte::COutByte(UINT32 aBufferSize): + m_BufferSize(aBufferSize) +{ + m_Buffer = new BYTE[m_BufferSize]; +} + +COutByte::~COutByte() +{ + delete []m_Buffer; +} + +void COutByte::Init(ISequentialOutStream *aStream) +{ + m_Stream = aStream; + m_ProcessedSize = 0; + m_Pos = 0; +} + +HRESULT COutByte::Flush() +{ + if (m_Pos == 0) + return S_OK; + UINT32 aProcessedSize; + HRESULT aResult = m_Stream->Write(m_Buffer, m_Pos, &aProcessedSize); + if (aResult != S_OK) + return aResult; + if (m_Pos != aProcessedSize) + return E_FAIL; + m_ProcessedSize += aProcessedSize; + m_Pos = 0; + return S_OK; +} + +void COutByte::WriteBlock() +{ + HRESULT aResult = Flush(); + if (aResult != S_OK) + throw aResult; +} + +} diff --git a/hostTools/lzma/compress/OutByte.h b/hostTools/lzma/compress/OutByte.h new file mode 100644 index 0000000..dc9ff0a --- /dev/null +++ b/hostTools/lzma/compress/OutByte.h @@ -0,0 +1,42 @@ +#ifndef __STREAM_OUTBYTE_H +#define __STREAM_OUTBYTE_H + +#include "Portable.h" +#include "IInOutStreams.h" + +namespace NStream { + +class COutByte +{ + BYTE *m_Buffer; + UINT32 m_Pos; + UINT32 m_BufferSize; + ISequentialOutStream* m_Stream; + UINT64 m_ProcessedSize; + + void WriteBlock(); +public: + COutByte(UINT32 aBufferSize = (1 << 20)); + ~COutByte(); + + void Init(ISequentialOutStream *aStream); + HRESULT Flush(); + + void WriteByte(BYTE aByte) + { + m_Buffer[m_Pos++] = aByte; + if(m_Pos >= m_BufferSize) + WriteBlock(); + } + void WriteBytes(const void *aBytes, UINT32 aSize) + { + for (UINT32 i = 0; i < aSize; i++) + WriteByte(((const BYTE *)aBytes)[i]); + } + + UINT64 GetProcessedSize() const { return m_ProcessedSize + m_Pos; } +}; + +} + +#endif diff --git a/hostTools/lzma/compress/Portable.h b/hostTools/lzma/compress/Portable.h new file mode 100644 index 0000000..1985b1b --- /dev/null +++ b/hostTools/lzma/compress/Portable.h @@ -0,0 +1,48 @@ +#ifndef __PORTABLE_H +#define __PORTABLE_H + +#include + +typedef signed char INT8; +typedef unsigned char UINT8; +typedef short INT16; +typedef unsigned short UINT16; +typedef long INT32; +typedef unsigned long UINT32; +#if GNU +typedef long long INT64; +typedef unsigned long long UINT64; +#else +typedef __int64 INT64; +typedef unsigned __int64 UINT64; +#endif + +typedef UINT8 BYTE; +typedef UINT16 WORD; +typedef UINT32 DWORD; + +typedef unsigned UINT_PTR; + +typedef int BOOL; +#define FALSE 0 +#define TRUE 1 + +#define HRESULT int +#define S_OK 0 +#define E_INVALIDARG -1 +#define E_OUTOFMEMORY -2 +#define E_FAIL -3 +#define E_INTERNAL_ERROR -4 +#define E_INVALIDDATA -5 + +template inline T MyMin(T a, T b) { + return a < b ? a : b; +} + +template inline T MyMax(T a, T b) { + return a > b ? a : b; +} + +#define RETURN_IF_NOT_S_OK(x) { HRESULT __aResult_ = (x); if(__aResult_ != S_OK) return __aResult_; } + +#endif diff --git a/hostTools/lzma/compress/RCDefs.h b/hostTools/lzma/compress/RCDefs.h new file mode 100644 index 0000000..326f5f3 --- /dev/null +++ b/hostTools/lzma/compress/RCDefs.h @@ -0,0 +1,42 @@ +#ifndef __RCDEFS_H +#define __RCDEFS_H + +#include "AriBitCoder.h" +#include "AriConst.h" + +#define RC_INIT_VAR \ + UINT32 aRange = aRangeDecoder->m_Range; \ + UINT32 aCode = aRangeDecoder->m_Code; + +#define RC_FLUSH_VAR \ + aRangeDecoder->m_Range = aRange; \ + aRangeDecoder->m_Code = aCode; + +#define RC_NORMALIZE \ + if (aRange < NCompression::NArithmetic::kTopValue) \ + { \ + aCode = (aCode << 8) | aRangeDecoder->m_Stream.ReadByte(); \ + aRange <<= 8; } + +#define RC_GETBIT2(aNumMoveBits, aProb, aModelIndex, Action0, Action1) \ + {UINT32 aNewBound = (aRange >> NCompression::NArithmetic::kNumBitModelTotalBits) * aProb; \ + if (aCode < aNewBound) \ + { \ + Action0; \ + aRange = aNewBound; \ + aProb += (NCompression::NArithmetic::kBitModelTotal - aProb) >> aNumMoveBits; \ + aModelIndex <<= 1; \ + } \ + else \ + { \ + Action1; \ + aRange -= aNewBound; \ + aCode -= aNewBound; \ + aProb -= (aProb) >> aNumMoveBits; \ + aModelIndex = (aModelIndex << 1) + 1; \ + }} \ + RC_NORMALIZE + +#define RC_GETBIT(aNumMoveBits, aProb, aModelIndex) RC_GETBIT2(aNumMoveBits, aProb, aModelIndex, ; , ;) + +#endif diff --git a/hostTools/lzma/compress/RangeCoder.h b/hostTools/lzma/compress/RangeCoder.h new file mode 100644 index 0000000..a9e30f5 --- /dev/null +++ b/hostTools/lzma/compress/RangeCoder.h @@ -0,0 +1,232 @@ +#ifndef __COMPRESSION_RANGECODER_H +#define __COMPRESSION_RANGECODER_H + +#include "InByte.h" +#include "OutByte.h" + +namespace NCompression { +namespace NArithmetic { + +const UINT32 kNumTopBits = 24; +const UINT32 kTopValue = (1 << kNumTopBits); + +class CRangeEncoder +{ + NStream::COutByte m_Stream; + UINT64 m_Low; + UINT32 m_Range; + UINT32 m_FFNum; + BYTE m_Cache; + +public: + void Init(ISequentialOutStream *aStream) + { + m_Stream.Init(aStream); + m_Low = 0; + m_Range = UINT32(-1); + m_FFNum = 0; + m_Cache = 0; + } + + void FlushData() + { + // m_Low += 1; + for(int i = 0; i < 5; i++) + ShiftLow(); + } + + HRESULT FlushStream() + { return m_Stream.Flush(); } + + void Encode(UINT32 aStart, UINT32 aSize, UINT32 aTotal) + { + m_Low += aStart * (m_Range /= aTotal); + m_Range *= aSize; + while (m_Range < kTopValue) + { + m_Range <<= 8; + ShiftLow(); + } + } + + /* + void EncodeDirectBitsDiv(UINT32 aValue, UINT32 aNumTotalBits) + { + m_Low += aValue * (m_Range >>= aNumTotalBits); + Normalize(); + } + + void EncodeDirectBitsDiv2(UINT32 aValue, UINT32 aNumTotalBits) + { + if (aNumTotalBits <= kNumBottomBits) + EncodeDirectBitsDiv(aValue, aNumTotalBits); + else + { + EncodeDirectBitsDiv(aValue >> kNumBottomBits, (aNumTotalBits - kNumBottomBits)); + EncodeDirectBitsDiv(aValue & ((1 << kBottomValueBits) - 1), kNumBottomBits); + } + } + */ + void ShiftLow() + { + if (m_Low < (UINT32)0xFF000000 || UINT32(m_Low >> 32) == 1) + { + m_Stream.WriteByte(m_Cache + BYTE(m_Low >> 32)); + for (;m_FFNum != 0; m_FFNum--) + m_Stream.WriteByte(0xFF + BYTE(m_Low >> 32)); + m_Cache = BYTE(UINT32(m_Low) >> 24); + } + else + m_FFNum++; + m_Low = UINT32(m_Low) << 8; + } + + void EncodeDirectBits(UINT32 aValue, UINT32 aNumTotalBits) + { + for (int i = aNumTotalBits - 1; i >= 0; i--) + { + m_Range >>= 1; + if (((aValue >> i) & 1) == 1) + m_Low += m_Range; + if (m_Range < kTopValue) + { + m_Range <<= 8; + ShiftLow(); + } + } + } + + void EncodeBit(UINT32 aSize0, UINT32 aNumTotalBits, UINT32 aSymbol) + { + UINT32 aNewBound = (m_Range >> aNumTotalBits) * aSize0; + if (aSymbol == 0) + m_Range = aNewBound; + else + { + m_Low += aNewBound; + m_Range -= aNewBound; + } + while (m_Range < kTopValue) + { + m_Range <<= 8; + ShiftLow(); + } + } + + UINT64 GetProcessedSize() { return m_Stream.GetProcessedSize() + m_FFNum; } +}; + +class CRangeDecoder +{ +public: + NStream::CInByte m_Stream; + UINT32 m_Range; + UINT32 m_Code; + UINT32 m_Word; + void Normalize() + { + while (m_Range < kTopValue) + { + m_Code = (m_Code << 8) | m_Stream.ReadByte(); + m_Range <<= 8; + } + } + + void Init(ISequentialInStream *aStream) + { + m_Stream.Init(aStream); + m_Code = 0; + m_Range = UINT32(-1); + for(int i = 0; i < 5; i++) + m_Code = (m_Code << 8) | m_Stream.ReadByte(); + } + + UINT32 GetThreshold(UINT32 aTotal) + { + return (m_Code) / ( m_Range /= aTotal); + } + + void Decode(UINT32 aStart, UINT32 aSize, UINT32 aTotal) + { + m_Code -= aStart * m_Range; + m_Range *= aSize; + Normalize(); + } + + /* + UINT32 DecodeDirectBitsDiv(UINT32 aNumTotalBits) + { + m_Range >>= aNumTotalBits; + UINT32 aThreshold = m_Code / m_Range; + m_Code -= aThreshold * m_Range; + + Normalize(); + return aThreshold; + } + + UINT32 DecodeDirectBitsDiv2(UINT32 aNumTotalBits) + { + if (aNumTotalBits <= kNumBottomBits) + return DecodeDirectBitsDiv(aNumTotalBits); + UINT32 aResult = DecodeDirectBitsDiv(aNumTotalBits - kNumBottomBits) << kNumBottomBits; + return (aResult | DecodeDirectBitsDiv(kNumBottomBits)); + } + */ + + UINT32 DecodeDirectBits(UINT32 aNumTotalBits) + { + UINT32 aRange = m_Range; + UINT32 aCode = m_Code; + UINT32 aResult = 0; + for (UINT32 i = aNumTotalBits; i > 0; i--) + { + aRange >>= 1; + /* + aResult <<= 1; + if (aCode >= aRange) + { + aCode -= aRange; + aResult |= 1; + } + */ + UINT32 t = (aCode - aRange) >> 31; + aCode -= aRange & (t - 1); + // aRange = aRangeTmp + ((aRange & 1) & (1 - t)); + aResult = (aResult << 1) | (1 - t); + + if (aRange < kTopValue) + { + aCode = (aCode << 8) | m_Stream.ReadByte(); + aRange <<= 8; + } + } + m_Range = aRange; + m_Code = aCode; + return aResult; + } + + UINT32 DecodeBit(UINT32 aSize0, UINT32 aNumTotalBits) + { + UINT32 aNewBound = (m_Range >> aNumTotalBits) * aSize0; + UINT32 aSymbol; + if (m_Code < aNewBound) + { + aSymbol = 0; + m_Range = aNewBound; + } + else + { + aSymbol = 1; + m_Code -= aNewBound; + m_Range -= aNewBound; + } + Normalize(); + return aSymbol; + } + + UINT64 GetProcessedSize() {return m_Stream.GetProcessedSize(); } +}; + +}} + +#endif diff --git a/hostTools/lzma/compress/StdAfx.cpp b/hostTools/lzma/compress/StdAfx.cpp new file mode 100644 index 0000000..3ea4c90 --- /dev/null +++ b/hostTools/lzma/compress/StdAfx.cpp @@ -0,0 +1,8 @@ +// stdafx.cpp : source file that includes just the standard includes +// ProgramStore.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#include "StdAfx.h" + +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file diff --git a/hostTools/lzma/compress/StdAfx.h b/hostTools/lzma/compress/StdAfx.h new file mode 100644 index 0000000..8c419c9 --- /dev/null +++ b/hostTools/lzma/compress/StdAfx.h @@ -0,0 +1,22 @@ +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#if !defined(AFX_STDAFX_H__FF26AFB2_1227_11D4_A62F_00C04F69DA2B__INCLUDED_) +#define AFX_STDAFX_H__FF26AFB2_1227_11D4_A62F_00C04F69DA2B__INCLUDED_ + +#if _MSC_VER > 1000 +#pragma once +#endif // _MSC_VER > 1000 + +#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers + +#include + +// TODO: reference additional headers your program requires here + +//{{AFX_INSERT_LOCATION}} +// Microsoft Visual C++ will insert additional declarations immediately before the previous line. + +#endif // !defined(AFX_STDAFX_H__FF26AFB2_1227_11D4_A62F_00C04F69DA2B__INCLUDED_) diff --git a/hostTools/lzma/compress/WindowIn.cpp b/hostTools/lzma/compress/WindowIn.cpp new file mode 100644 index 0000000..20b0859 --- /dev/null +++ b/hostTools/lzma/compress/WindowIn.cpp @@ -0,0 +1,97 @@ +#include "Portable.h" +#include "WindowIn.h" + +namespace NStream { +namespace NWindow { + +CIn::CIn(): + m_BufferBase(0) +{} + +void CIn::Free() +{ + delete []m_BufferBase; + m_BufferBase = 0; +} + +void CIn::Create(UINT32 aKeepSizeBefore, UINT32 aKeepSizeAfter, UINT32 aKeepSizeReserv) +{ + m_KeepSizeBefore = aKeepSizeBefore; + m_KeepSizeAfter = aKeepSizeAfter; + m_KeepSizeReserv = aKeepSizeReserv; + m_BlockSize = aKeepSizeBefore + aKeepSizeAfter + aKeepSizeReserv; + Free(); + m_BufferBase = new BYTE[m_BlockSize]; + m_PointerToLastSafePosition = m_BufferBase + m_BlockSize - aKeepSizeAfter; +} + +CIn::~CIn() +{ + Free(); +} + +HRESULT CIn::Init(ISequentialInStream *aStream) +{ + m_Stream = aStream; + m_Buffer = m_BufferBase; + m_Pos = 0; + m_StreamPos = 0; + m_StreamEndWasReached = false; + return ReadBlock(); +} + +/////////////////////////////////////////// +// ReadBlock + +// In State: +// (m_Buffer + m_StreamPos) <= (m_BufferBase + m_BlockSize) +// Out State: +// m_PosLimit <= m_BlockSize - m_KeepSizeAfter; +// if(m_StreamEndWasReached == false): +// m_StreamPos >= m_Pos + m_KeepSizeAfter +// m_PosLimit = m_StreamPos - m_KeepSizeAfter; +// else +// + +HRESULT CIn::ReadBlock() +{ + if(m_StreamEndWasReached) + return S_OK; + while(true) + { + UINT32 aSize = (m_BufferBase + m_BlockSize) - (m_Buffer + m_StreamPos); + if(aSize == 0) + return S_OK; + UINT32 aNumReadBytes; + RETURN_IF_NOT_S_OK(m_Stream->Read(m_Buffer + m_StreamPos, + aSize, &aNumReadBytes)); + if(aNumReadBytes == 0) + { + m_PosLimit = m_StreamPos; + const BYTE *aPointerToPostion = m_Buffer + m_PosLimit; + if(aPointerToPostion > m_PointerToLastSafePosition) + m_PosLimit = m_PointerToLastSafePosition - m_Buffer; + m_StreamEndWasReached = true; + return S_OK; + } + m_StreamPos += aNumReadBytes; + if(m_StreamPos >= m_Pos + m_KeepSizeAfter) + { + m_PosLimit = m_StreamPos - m_KeepSizeAfter; + return S_OK; + } + } +} + +void CIn::MoveBlock() +{ + BeforeMoveBlock(); + UINT32 anOffset = (m_Buffer + m_Pos - m_KeepSizeBefore) - m_BufferBase; + UINT32 aNumBytes = (m_Buffer + m_StreamPos) - (m_BufferBase + anOffset); + memmove(m_BufferBase, m_BufferBase + anOffset, aNumBytes); + m_Buffer -= anOffset; + AfterMoveBlock(); +} + + +}} diff --git a/hostTools/lzma/compress/WindowIn.h b/hostTools/lzma/compress/WindowIn.h new file mode 100644 index 0000000..039e507 --- /dev/null +++ b/hostTools/lzma/compress/WindowIn.h @@ -0,0 +1,91 @@ +#ifndef __STREAM_WINDOWIN_H +#define __STREAM_WINDOWIN_H + +#include "IInOutStreams.h" + +namespace NStream { +namespace NWindow { + +class CIn +{ + BYTE *m_BufferBase; // pointer to buffer with data + ISequentialInStream* m_Stream; + UINT32 m_PosLimit; // offset (from m_Buffer) of first byte when new block reading must be done + bool m_StreamEndWasReached; // if (true) then m_StreamPos shows real end of stream + + const BYTE *m_PointerToLastSafePosition; + +protected: + BYTE *m_Buffer; // Pointer to virtual Buffer begin + UINT32 m_BlockSize; // Size of Allocated memory block + UINT32 m_Pos; // offset (from m_Buffer) of curent byte + UINT32 m_KeepSizeBefore; // how many BYTEs must be kept in buffer before m_Pos + UINT32 m_KeepSizeAfter; // how many BYTEs must be kept buffer after m_Pos + UINT32 m_KeepSizeReserv; // how many BYTEs must be kept as reserv + UINT32 m_StreamPos; // offset (from m_Buffer) of first not read byte from Stream + + virtual void BeforeMoveBlock() {}; + virtual void AfterMoveBlock() {}; + void MoveBlock(); + virtual HRESULT ReadBlock(); + void Free(); +public: + CIn(); + void Create(UINT32 aKeepSizeBefore, UINT32 aKeepSizeAfter, + UINT32 aKeepSizeReserv = (1<<17)); + virtual ~CIn(); + + HRESULT Init(ISequentialInStream *aStream); + + BYTE *GetBuffer() const { return m_Buffer; } + + const BYTE *GetPointerToCurrentPos() const { return m_Buffer + m_Pos; } + + HRESULT MovePos() + { + m_Pos++; + if (m_Pos > m_PosLimit) + { + const BYTE *aPointerToPostion = m_Buffer + m_Pos; + if(aPointerToPostion > m_PointerToLastSafePosition) + MoveBlock(); + return ReadBlock(); + } + else + return S_OK; + } + // BYTE GetCurrentByte()const; + BYTE GetIndexByte(UINT32 anIndex)const + { return m_Buffer[m_Pos + anIndex]; } + + // UINT32 GetCurPos()const { return m_Pos;}; + // BYTE *GetBufferBeg()const { return m_Buffer;}; + + // aIndex + aLimit have not to exceed m_KeepSizeAfter; + UINT32 GetMatchLen(UINT32 aIndex, UINT32 aBack, UINT32 aLimit) const + { + if(m_StreamEndWasReached) + if ((m_Pos + aIndex) + aLimit > m_StreamPos) + aLimit = m_StreamPos - (m_Pos + aIndex); + aBack++; + BYTE *pby = m_Buffer + m_Pos + aIndex; + UINT32 i; + for(i = 0; i < aLimit && pby[i] == pby[i - aBack]; i++); + return i; + } + + UINT32 GetNumAvailableBytes() const { return m_StreamPos - m_Pos; } + + void ReduceOffsets(UINT32 aSubValue) + { + m_Buffer += aSubValue; + m_PosLimit -= aSubValue; + m_Pos -= aSubValue; + m_StreamPos -= aSubValue; + } + +}; + +}} + +#endif diff --git a/hostTools/lzma/compress/WindowOut.cpp b/hostTools/lzma/compress/WindowOut.cpp new file mode 100644 index 0000000..aad4af6 --- /dev/null +++ b/hostTools/lzma/compress/WindowOut.cpp @@ -0,0 +1,71 @@ +#include "WindowOut.h" + +namespace NStream { +namespace NWindow { + +void COut::Create(UINT32 aKeepSizeBefore, UINT32 aKeepSizeAfter, UINT32 aKeepSizeReserv) +{ + m_Pos = 0; + m_PosLimit = aKeepSizeReserv + aKeepSizeBefore; + m_KeepSizeBefore = aKeepSizeBefore; + m_KeepSizeAfter = aKeepSizeAfter; + m_KeepSizeReserv = aKeepSizeReserv; + m_StreamPos = 0; + m_MoveFrom = m_KeepSizeReserv; + m_WindowSize = aKeepSizeBefore; + UINT32 aBlockSize = m_KeepSizeBefore + m_KeepSizeAfter + m_KeepSizeReserv; + delete []m_Buffer; + m_Buffer = new BYTE[aBlockSize]; +} + +COut::~COut() +{ + delete []m_Buffer; +} + +void COut::SetWindowSize(UINT32 aWindowSize) +{ + m_WindowSize = aWindowSize; + m_MoveFrom = m_KeepSizeReserv + m_KeepSizeBefore - aWindowSize; +} + +void COut::Init(ISequentialOutStream *aStream, bool aSolid) +{ + m_Stream = aStream; + + if(aSolid) + m_StreamPos = m_Pos; + else + { + m_Pos = 0; + m_PosLimit = m_KeepSizeReserv + m_KeepSizeBefore; + m_StreamPos = 0; + } +} + +HRESULT COut::Flush() +{ + UINT32 aSize = m_Pos - m_StreamPos; + if(aSize == 0) + return S_OK; + UINT32 aProcessedSize; + HRESULT aResult = m_Stream->Write(m_Buffer + m_StreamPos, aSize, &aProcessedSize); + if (aResult != S_OK) + return aResult; + if (aSize != aProcessedSize) + return E_FAIL; + m_StreamPos = m_Pos; + return S_OK; +} + +void COut::MoveBlockBackward() +{ + HRESULT aResult = Flush(); + if (aResult != S_OK) + throw aResult; + memmove(m_Buffer, m_Buffer + m_MoveFrom, m_WindowSize + m_KeepSizeAfter); + m_Pos -= m_MoveFrom; + m_StreamPos -= m_MoveFrom; +} + +}} diff --git a/hostTools/lzma/compress/WindowOut.h b/hostTools/lzma/compress/WindowOut.h new file mode 100644 index 0000000..8adad98 --- /dev/null +++ b/hostTools/lzma/compress/WindowOut.h @@ -0,0 +1,71 @@ +#ifndef __STREAM_WINDOWOUT_H +#define __STREAM_WINDOWOUT_H + +#include "IInOutStreams.h" + +namespace NStream { +namespace NWindow { + +// m_KeepSizeBefore: how mach BYTEs must be in buffer before m_Pos; +// m_KeepSizeAfter: how mach BYTEs must be in buffer after m_Pos; +// m_KeepSizeReserv: how mach BYTEs must be in buffer for Moving Reserv; +// must be >= aKeepSizeAfter; // test it + +class COut +{ + BYTE *m_Buffer; + UINT32 m_Pos; + UINT32 m_PosLimit; + UINT32 m_KeepSizeBefore; + UINT32 m_KeepSizeAfter; + UINT32 m_KeepSizeReserv; + UINT32 m_StreamPos; + + UINT32 m_WindowSize; + UINT32 m_MoveFrom; + + ISequentialOutStream *m_Stream; + + virtual void MoveBlockBackward(); +public: + COut(): m_Buffer(0), m_Stream(0) {} + virtual ~COut(); + void Create(UINT32 aKeepSizeBefore, + UINT32 aKeepSizeAfter, UINT32 aKeepSizeReserv = (1<<17)); + void SetWindowSize(UINT32 aWindowSize); + + void Init(ISequentialOutStream *aStream, bool aSolid = false); + HRESULT Flush(); + + UINT32 GetCurPos() const { return m_Pos; } + const BYTE *GetPointerToCurrentPos() const { return m_Buffer + m_Pos;}; + + void CopyBackBlock(UINT32 aDistance, UINT32 aLen) + { + if (m_Pos >= m_PosLimit) + MoveBlockBackward(); + BYTE *p = m_Buffer + m_Pos; + aDistance++; + for(UINT32 i = 0; i < aLen; i++) + p[i] = p[i - aDistance]; + m_Pos += aLen; + } + + void PutOneByte(BYTE aByte) + { + if (m_Pos >= m_PosLimit) + MoveBlockBackward(); + m_Buffer[m_Pos++] = aByte; + } + + BYTE GetOneByte(UINT32 anIndex) const + { + return m_Buffer[m_Pos + anIndex]; + } + + BYTE *GetBuffer() const { return m_Buffer; } +}; + +}} + +#endif diff --git a/hostTools/lzma/compress/lzDecomp.cpp b/hostTools/lzma/compress/lzDecomp.cpp new file mode 100644 index 0000000..f45cb4b --- /dev/null +++ b/hostTools/lzma/compress/lzDecomp.cpp @@ -0,0 +1,45 @@ +#include "stdio.h" +#include "LZMADecoder.h" + +//static LzmaDecoder cc; +ISequentialInStream in_stream; +ISequentialOutStream out_stream; +int decompress_lzma_7z( unsigned char* in_data, + unsigned in_size, + unsigned char* out_data, + unsigned out_size) { +// LzmaDecoder cc; + int RC; + UINT64 in_size_l = in_size; + UINT64 out_size_l = out_size; + + + InStreamInit(in_data, in_size); + + OutStreamInit((char *)out_data, out_size); + + LzmaDecoderConstructor(&cc); + + if ((RC = LzmaDecoderReadCoderProperties(&cc)) != S_OK) + { + return RC; + } + + if (LzmaDecoderCode(&cc, &in_size_l, &out_size_l) != S_OK) + { + return -2; + } + + if (out_stream.size != out_size) + { + return -3; + } + + if ( out_stream.overflow ) + { + return -4; + } + +printf( "\nDecompressed size: %d\n", out_stream.total ); + return 0; +} diff --git a/hostTools/lzma/decompress/7z.h b/hostTools/lzma/decompress/7z.h new file mode 100644 index 0000000..ca8ea7f --- /dev/null +++ b/hostTools/lzma/decompress/7z.h @@ -0,0 +1,16 @@ +#ifndef __7Z_H +#define __7Z_H + +#if defined __cplusplus +extern "C" +{ +#endif + +int decompress_lzma_7z(unsigned char* in_data, unsigned in_size, unsigned char* out_data, unsigned out_size); + +#if defined __cplusplus +} +#endif + +#endif + diff --git a/hostTools/lzma/decompress/7zlzma.c b/hostTools/lzma/decompress/7zlzma.c new file mode 100644 index 0000000..f45d6c2 --- /dev/null +++ b/hostTools/lzma/decompress/7zlzma.c @@ -0,0 +1,57 @@ +#include "7z.h" + +#ifdef _HOST_TOOL +#include "stdio.h" +#endif + +#include "LZMADecoder.h" + + +static LzmaDecoder cc; +ISequentialInStream in_stream; +ISequentialOutStream out_stream; +int decompress_lzma_7z( unsigned char* in_data, + unsigned in_size, + unsigned char* out_data, + unsigned out_size) { +// LzmaDecoder cc; + int RC; + UINT64 in_size_l = in_size; + UINT64 out_size_l = out_size; + + + InStreamInit(in_data, in_size); + + OutStreamInit((char *)out_data, out_size); + + LzmaDecoderConstructor(&cc); + + if ((RC = LzmaDecoderReadCoderProperties(&cc)) != S_OK) + { + return RC; + } + + if (LzmaDecoderCode(&cc, &in_size_l, &out_size_l) != S_OK) + { + return -2; + } + + if (out_stream.size != out_size) + { + return -3; + } + + if ( out_stream.overflow ) + { + return -4; + } + + return 0; +} + +//BRCM modification +#ifdef __KERNEL__ +EXPORT_SYMBOL(decompress_lzma_7z); +#endif + + diff --git a/hostTools/lzma/decompress/AriBitCoder.h b/hostTools/lzma/decompress/AriBitCoder.h new file mode 100644 index 0000000..cedc189 --- /dev/null +++ b/hostTools/lzma/decompress/AriBitCoder.h @@ -0,0 +1,51 @@ +#ifndef __COMPRESSION_BITCODER_H +#define __COMPRESSION_BITCODER_H + +#include "RangeCoder.h" + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) + +#define kNumMoveReducingBits 2 + + +typedef UINT32 CBitDecoder; + +INLINE void BitDecoderInit(CBitDecoder *bitDecoder) + { + *bitDecoder = kBitModelTotal / 2; + } + +#if 0 +UINT32 BitDecode(ISequentialInStream *in_stream, CBitDecoder *bitDecoder, CRangeDecoder *aRangeDecoder); +#else +INLINE UINT32 BitDecode(ISequentialInStream *in_stream, CBitDecoder *bitDecoder, CRangeDecoder *aRangeDecoder) + { + UINT32 aNewBound = (aRangeDecoder->m_Range >> kNumBitModelTotalBits) * (*bitDecoder); + if (aRangeDecoder->m_Code < aNewBound) + { + aRangeDecoder->m_Range = aNewBound; + *bitDecoder += (kBitModelTotal - *bitDecoder) >> kNumMoveBits; + if (aRangeDecoder->m_Range < kTopValue) + { + aRangeDecoder->m_Code = (aRangeDecoder->m_Code << 8) | InStreamReadByte(in_stream); + aRangeDecoder->m_Range <<= 8; + } + return 0; + } + else + { + aRangeDecoder->m_Range -= aNewBound; + aRangeDecoder->m_Code -= aNewBound; + *bitDecoder -= (*bitDecoder) >> kNumMoveBits; + if (aRangeDecoder->m_Range < kTopValue) + { + aRangeDecoder->m_Code = (aRangeDecoder->m_Code << 8) | InStreamReadByte(in_stream); + aRangeDecoder->m_Range <<= 8; + } + return 1; + } + } +#endif + +#endif diff --git a/hostTools/lzma/decompress/BitTreeCoder.h b/hostTools/lzma/decompress/BitTreeCoder.h new file mode 100644 index 0000000..8c663e6 --- /dev/null +++ b/hostTools/lzma/decompress/BitTreeCoder.h @@ -0,0 +1,160 @@ +#ifndef __BITTREECODER_H +#define __BITTREECODER_H + +#include "AriBitCoder.h" +#include "RCDefs.h" + +//BRCM modification start +#ifdef _HOST_TOOL +#include "stdio.h" +#include "stdlib.h" +#include "malloc.h" +#endif + +#ifdef _CFE_ +#include "lib_malloc.h" +#include "lib_printf.h" +#define malloc(x) KMALLOC(x, 0) +#endif + +#ifdef __KERNEL__ +#include +#include +#include +#define printf printk +//#define malloc(x) kmalloc(x,GFP_KERNEL) +#define malloc(x) vmalloc(x) +#define free(x) vfree(x) +#endif +//BRCM modification end + +////////////////////////// +// CBitTreeDecoder + +typedef struct CBitTreeDecoder +{ + UINT32 m_NumBitLevels; + CBitDecoder *m_Models; +} CBitTreeDecoder; + +// ~CBitTreeDecoder() { free(m_Models); } +INLINE void BitTreeDecoderInit(CBitTreeDecoder *bitTreeDecoder, UINT32 aNumBitLevels) + { + int i; + bitTreeDecoder->m_NumBitLevels = aNumBitLevels; + bitTreeDecoder->m_Models = (CBitDecoder *)malloc( sizeof(CBitDecoder) * (1 << bitTreeDecoder->m_NumBitLevels)); + //BRCM modification + //printf("malloc in BitTreeDecoderInit=%d\n",sizeof(CBitDecoder) * (1 << bitTreeDecoder->m_NumBitLevels)); + if (!bitTreeDecoder->m_Models) { + printf("Error in allocating memory for bitTreeDecoder!\n"); + return; + } + for(i = 1; i < (1 << aNumBitLevels); i++) + BitDecoderInit(&bitTreeDecoder->m_Models[i]); + } +INLINE UINT32 BitTreeDecode(ISequentialInStream *in_stream, CBitTreeDecoder *bitTreeDecoder, CRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + UINT32 aRange = aRangeDecoder->m_Range; + UINT32 aCode = aRangeDecoder->m_Code; + UINT32 aBitIndex; + for(aBitIndex = bitTreeDecoder->m_NumBitLevels; aBitIndex > 0; aBitIndex--) + { + RC_GETBIT(kNumMoveBits, bitTreeDecoder->m_Models[aModelIndex], aModelIndex) + } + aRangeDecoder->m_Range = aRange; + aRangeDecoder->m_Code = aCode; + return aModelIndex - (1 << bitTreeDecoder->m_NumBitLevels); + } + + +//////////////////////////////// +// CReverseBitTreeDecoder2 + +typedef struct CReverseBitTreeDecoder2 +{ + UINT32 m_NumBitLevels; + CBitDecoder *m_Models; +} CReverseBitTreeDecoder2; + +// CReverseBitTreeDecoder2(): m_Models(0) { } +// ~CReverseBitTreeDecoder2() { free(m_Models); } +INLINE BOOL ReverseBitTreeDecoder2Create(CReverseBitTreeDecoder2 *reverseBitTreeDecoder2, UINT32 aNumBitLevels) + { + reverseBitTreeDecoder2->m_NumBitLevels = aNumBitLevels; + reverseBitTreeDecoder2->m_Models = (CBitDecoder *)malloc( sizeof(CBitDecoder) * (1 << reverseBitTreeDecoder2->m_NumBitLevels)); + //printf("malloc in ReverseBitTreeDecoder2Create=%d\n",sizeof(CBitDecoder) * (1 << reverseBitTreeDecoder2->m_NumBitLevels)); + if (!reverseBitTreeDecoder2->m_Models) { + printf("Error in allocating memory for reverseBitTreeDecoder2!\n"); + return 0; + } + return (reverseBitTreeDecoder2->m_Models != 0); + } +INLINE void ReverseBitTreeDecoder2Init(CReverseBitTreeDecoder2 *reverseBitTreeDecoder2) + { + UINT32 aNumModels = 1 << reverseBitTreeDecoder2->m_NumBitLevels; + UINT32 i; + for(i = 1; i < aNumModels; i++) + BitDecoderInit(&reverseBitTreeDecoder2->m_Models[i]); + } +INLINE UINT32 ReverseBitTreeDecoder2Decode(ISequentialInStream *in_stream, CReverseBitTreeDecoder2 *reverseBitTreeDecoder2, CRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + UINT32 aSymbol = 0; + UINT32 aRange = aRangeDecoder->m_Range; + UINT32 aCode = aRangeDecoder->m_Code; + UINT32 aBitIndex; + for(aBitIndex = 0; aBitIndex < reverseBitTreeDecoder2->m_NumBitLevels; aBitIndex++) + { + RC_GETBIT2(kNumMoveBits, reverseBitTreeDecoder2->m_Models[aModelIndex], aModelIndex, ; , aSymbol |= (1 << aBitIndex)) + } + aRangeDecoder->m_Range = aRange; + aRangeDecoder->m_Code = aCode; + return aSymbol; + } + + +//////////////////////////// +// CReverseBitTreeDecoder + +typedef struct CReverseBitTreeDecoder +{ + UINT32 m_NumBitLevels; + CBitDecoder *m_Models; +} CReverseBitTreeDecoder; + +// CReverseBitTreeDecoder(): m_Models(0) { } +// ~CReverseBitTreeDecoder() { free(m_Models); } +INLINE void ReverseBitTreeDecoderInit(CReverseBitTreeDecoder *reverseBitTreeDecoder, UINT32 aNumBitLevels) + { + int i; + reverseBitTreeDecoder->m_NumBitLevels = aNumBitLevels; + reverseBitTreeDecoder->m_Models = (CBitDecoder *)malloc( sizeof(CBitDecoder) * (1 << reverseBitTreeDecoder->m_NumBitLevels)); + //printf("malloc in ReverseBitTreeDecoderInit=%d\n",sizeof(CBitDecoder) * (1 << reverseBitTreeDecoder->m_NumBitLevels)); + if (!reverseBitTreeDecoder->m_Models) { + printf("Error in allocating memory for reverseBitTreeDecoder!\n"); + return; + } + for(i = 1; i < (1 << reverseBitTreeDecoder->m_NumBitLevels); i++) + BitDecoderInit(&reverseBitTreeDecoder->m_Models[i]); + } + +INLINE UINT32 ReverseBitTreeDecoderDecode(ISequentialInStream *in_stream, CReverseBitTreeDecoder *reverseBitTreeDecoder, CRangeDecoder *aRangeDecoder) + { + UINT32 aModelIndex = 1; + UINT32 aSymbol = 0; + UINT32 aRange = aRangeDecoder->m_Range; + UINT32 aCode = aRangeDecoder->m_Code; + UINT32 aBitIndex; + for(aBitIndex = 0; aBitIndex < reverseBitTreeDecoder->m_NumBitLevels; aBitIndex++) + { + RC_GETBIT2(kNumMoveBits, reverseBitTreeDecoder->m_Models[aModelIndex], aModelIndex, ; , aSymbol |= (1 << aBitIndex)) + } + aRangeDecoder->m_Range = aRange; + aRangeDecoder->m_Code = aCode; + return aSymbol; + } + + + +#endif diff --git a/hostTools/lzma/decompress/IInOutStreams.c b/hostTools/lzma/decompress/IInOutStreams.c new file mode 100644 index 0000000..789c4ae --- /dev/null +++ b/hostTools/lzma/decompress/IInOutStreams.c @@ -0,0 +1,38 @@ +#include "IInOutStreams.h" +// BRCM modification +static void *lib_memcpy(void *dest,const void *src,size_t cnt); +static void *lib_memcpy(void *dest,const void *src,size_t cnt) +{ + unsigned char *d; + const unsigned char *s; + + d = (unsigned char *) dest; + s = (const unsigned char *) src; + + while (cnt) { + *d++ = *s++; + cnt--; + } + + return dest; +} + +HRESULT InStreamRead(void *aData, UINT32 aSize, UINT32* aProcessedSize) { + if (aSize > in_stream.remainingBytes) + aSize = in_stream.remainingBytes; + *aProcessedSize = aSize; + lib_memcpy(aData, in_stream.data, aSize); // brcm modification + in_stream.remainingBytes -= aSize; + in_stream.data += aSize; + return S_OK; + } + +#if 0 +BYTE InStreamReadByte() + { + if (in_stream.remainingBytes == 0) + return 0x0; + in_stream.remainingBytes--; + return (BYTE) *in_stream.data++; + } +#endif diff --git a/hostTools/lzma/decompress/IInOutStreams.h b/hostTools/lzma/decompress/IInOutStreams.h new file mode 100644 index 0000000..69abf39 --- /dev/null +++ b/hostTools/lzma/decompress/IInOutStreams.h @@ -0,0 +1,62 @@ +#ifndef __IINOUTSTREAMS_H +#define __IINOUTSTREAMS_H + +#include "Portable.h" + +typedef struct ISequentialInStream +{ + unsigned char* data; + unsigned remainingBytes; +} ISequentialInStream; + +extern ISequentialInStream in_stream; + +INLINE void InStreamInit(unsigned char * Adata, unsigned Asize) + { + in_stream.data = Adata; + in_stream.remainingBytes = Asize; + } + +HRESULT InStreamRead(void *aData, UINT32 aSize, UINT32* aProcessedSize); + +#if 0 +BYTE InStreamReadByte(); +#else +INLINE BYTE InStreamReadByte(ISequentialInStream *in_stream) + { + if (in_stream->remainingBytes == 0) + return 0x0; + in_stream->remainingBytes--; + return (BYTE) *in_stream->data++; + } +#endif + + + +typedef struct ISequentialOutStream +{ + char* data; + unsigned size; + BOOL overflow; + unsigned total; +} ISequentialOutStream; + +extern ISequentialOutStream out_stream; + +#define OutStreamInit(Adata, Asize) \ +{ \ + out_stream.data = Adata; \ + out_stream.size = Asize; \ + out_stream.overflow = FALSE; \ + out_stream.total = 0; \ +} + +#define OutStreamSizeSet(newsize) \ + { \ + out_stream.total = newsize; \ + if (out_stream.total > out_stream.size) \ + out_stream.overflow = TRUE; \ + } + + +#endif diff --git a/hostTools/lzma/decompress/LZMA.h b/hostTools/lzma/decompress/LZMA.h new file mode 100644 index 0000000..368328c --- /dev/null +++ b/hostTools/lzma/decompress/LZMA.h @@ -0,0 +1,83 @@ +#include "LenCoder.h" + +#ifndef __LZMA_H +#define __LZMA_H + + +#define kNumRepDistances 4 + +#define kNumStates 12 + +static const BYTE kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const BYTE kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const BYTE kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const BYTE kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +typedef BYTE CState; + +INLINE void CStateInit(CState *m_Index) + { *m_Index = 0; } +INLINE void CStateUpdateChar(CState *m_Index) + { *m_Index = kLiteralNextStates[*m_Index]; } +INLINE void CStateUpdateMatch(CState *m_Index) + { *m_Index = kMatchNextStates[*m_Index]; } +INLINE void CStateUpdateRep(CState *m_Index) + { *m_Index = kRepNextStates[*m_Index]; } +INLINE void CStateUpdateShortRep(CState *m_Index) + { *m_Index = kShortRepNextStates[*m_Index]; } + + +#define kNumPosSlotBits 6 +#define kDicLogSizeMax 28 +#define kDistTableSizeMax 56 + +//extern UINT32 kDistStart[kDistTableSizeMax]; +static const BYTE kDistDirectBits[kDistTableSizeMax] = +{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, + 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26 +}; + +#define kNumLenToPosStates 4 +INLINE UINT32 GetLenToPosState(UINT32 aLen) +{ + aLen -= 2; + if (aLen < kNumLenToPosStates) + return aLen; + return kNumLenToPosStates - 1; +} + +#define kMatchMinLen 2 + +#define kMatchMaxLen (kMatchMinLen + kNumSymbolsTotal - 1) + +#define kNumAlignBits 4 +#define kAlignTableSize 16 +#define kAlignMask 15 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumPosModels 10 + +#define kNumFullDistances (1 << (kEndPosModelIndex / 2)) + + +#define kMainChoiceLiteralIndex 0 +#define kMainChoiceMatchIndex 1 + +#define kMatchChoiceDistanceIndex0 +#define kMatchChoiceRepetitionIndex 1 + +#define kNumMoveBitsForMainChoice 5 +#define kNumMoveBitsForPosCoders 5 + +#define kNumMoveBitsForAlignCoders 5 + +#define kNumMoveBitsForPosSlotCoder 5 + +#define kNumLitPosStatesBitsEncodingMax 4 +#define kNumLitContextBitsMax 8 + + +#endif diff --git a/hostTools/lzma/decompress/LZMADecoder.c b/hostTools/lzma/decompress/LZMADecoder.c new file mode 100644 index 0000000..fe6d86d --- /dev/null +++ b/hostTools/lzma/decompress/LZMADecoder.c @@ -0,0 +1,398 @@ +#include "Portable.h" +#ifdef _HOST_TOOL +#include "stdio.h" +#endif +#include "LZMADecoder.h" + + +//#define RETURN_E_OUTOFMEMORY_IF_FALSE(x) { if (!(x)) return E_OUTOFMEMORY; } + + +static UINT32 kDistStart[kDistTableSizeMax]; +struct WindowOut out_window; + +/* + * BRCM modification: free all the allocated buffer by malloc + * + */ +static void LzmaDecoderFreeBuffer(LzmaDecoder *lzmaDecoder) +{ + int i,aPosState; + + //printf("free lzmaDecoder->m_LiteralDecoder\n"); + free((&lzmaDecoder->m_LiteralDecoder)->m_Coders); + + for (i = 0; i < kNumLenToPosStates; i++) { + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free((&lzmaDecoder->m_PosSlotDecoder[i])->m_Models); + } + // from LenDecoderInit(&lzmaDecoder->m_LenDecoder; + for (aPosState = 0; aPosState < (&lzmaDecoder->m_LenDecoder)->m_NumPosStates; aPosState++) { + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free( (&(&lzmaDecoder->m_LenDecoder)->m_LowCoder[aPosState])->m_Models ); + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free( (&(&lzmaDecoder->m_LenDecoder)->m_MidCoder[aPosState])->m_Models ); + } + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free( (&(&lzmaDecoder->m_LenDecoder)->m_HighCoder)->m_Models ); + + + // from LenDecoderInit(&lzmaDecoder->m_RepMatchLenDecoder); + for (aPosState = 0; aPosState < (&lzmaDecoder->m_RepMatchLenDecoder)->m_NumPosStates; aPosState++) { + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free( (&(&lzmaDecoder->m_RepMatchLenDecoder)->m_LowCoder[aPosState])->m_Models ); + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free( (&(&lzmaDecoder->m_RepMatchLenDecoder)->m_MidCoder[aPosState])->m_Models ); + } + //printf("free lzmaDecoder->m_PosSlotDecoder\n"); + free( (&(&lzmaDecoder->m_RepMatchLenDecoder)->m_HighCoder)->m_Models ); + + + //printf("free lzmaDecoder->m_PosAlignDecoder\n"); + free((&lzmaDecoder->m_PosAlignDecoder)->m_Models); + + for(i = 0; i < kNumPosModels; i++) { + //printf("free lzmaDecoder->m_PosDecoders\n"); + free((&lzmaDecoder->m_PosDecoders[i])->m_Models); + } + +} + +HRESULT LzmaDecoderSetDictionarySize( + LzmaDecoder *lzmaDecoder, + UINT32 aDictionarySize) +{ + if (aDictionarySize > (1 << kDicLogSizeMax)) + return E_INVALIDARG; + +// UINT32 aWindowReservSize = MyMax(aDictionarySize, UINT32(1 << 21)); + + if (lzmaDecoder->m_DictionarySize != aDictionarySize) + { + lzmaDecoder->m_DictionarySize = aDictionarySize; + } + return S_OK; +} + +HRESULT LzmaDecoderSetLiteralProperties( + LzmaDecoder *lzmaDecoder, + UINT32 aLiteralPosStateBits, + UINT32 aLiteralContextBits) +{ + if (aLiteralPosStateBits > 8) + return E_INVALIDARG; + if (aLiteralContextBits > 8) + return E_INVALIDARG; + LitDecoderCreate(&lzmaDecoder->m_LiteralDecoder, aLiteralPosStateBits, aLiteralContextBits); + return S_OK; +} + +HRESULT LzmaDecoderSetPosBitsProperties( + LzmaDecoder *lzmaDecoder, + UINT32 aNumPosStateBits) +{ + UINT32 aNumPosStates; + if (aNumPosStateBits > (UINT32) kNumPosStatesBitsMax) + return E_INVALIDARG; + aNumPosStates = 1 << aNumPosStateBits; + LenDecoderCreate(&lzmaDecoder->m_LenDecoder, aNumPosStates); + LenDecoderCreate(&lzmaDecoder->m_RepMatchLenDecoder, aNumPosStates); + lzmaDecoder->m_PosStateMask = aNumPosStates - 1; + return S_OK; +} + + +void LzmaDecoderConstructor(LzmaDecoder *lzmaDecoder) +{ + lzmaDecoder->m_DictionarySize = ((UINT32)-1); + LzmaDecoderCreate(lzmaDecoder); +} + +HRESULT LzmaDecoderCreate(LzmaDecoder *lzmaDecoder) +{ + int i; + for(i = 0; i < kNumPosModels; i++) + { + if (!(ReverseBitTreeDecoder2Create(&lzmaDecoder->m_PosDecoders[i],kDistDirectBits[kStartPosModelIndex + i]))) + return E_OUTOFMEMORY;; + } + return S_OK; +} + + +HRESULT LzmaDecoderInit(LzmaDecoder *lzmaDecoder) +{ + int i; + UINT32 j; + + RangeDecoderInit(&in_stream, &lzmaDecoder->m_RangeDecoder); + + OutWindowInit(); + + for(i = 0; i < kNumStates; i++) + { + for (j = 0; j <= lzmaDecoder->m_PosStateMask; j++) + { + BitDecoderInit(&lzmaDecoder->m_MainChoiceDecoders[i][j]); + BitDecoderInit(&lzmaDecoder->m_MatchRepShortChoiceDecoders[i][j]); + } + BitDecoderInit(&lzmaDecoder->m_MatchChoiceDecoders[i]); + BitDecoderInit(&lzmaDecoder->m_MatchRepChoiceDecoders[i]); + BitDecoderInit(&lzmaDecoder->m_MatchRep1ChoiceDecoders[i]); + BitDecoderInit(&lzmaDecoder->m_MatchRep2ChoiceDecoders[i]); + } + + LitDecoderInit(&lzmaDecoder->m_LiteralDecoder); + + for (i = 0; i < (int) kNumLenToPosStates; i++) + BitTreeDecoderInit(&lzmaDecoder->m_PosSlotDecoder[i],kNumPosSlotBits); + + for(i = 0; i < kNumPosModels; i++) + ReverseBitTreeDecoder2Init(&lzmaDecoder->m_PosDecoders[i]); + + LenDecoderInit(&lzmaDecoder->m_LenDecoder); + LenDecoderInit(&lzmaDecoder->m_RepMatchLenDecoder); + + ReverseBitTreeDecoderInit(&lzmaDecoder->m_PosAlignDecoder, kNumAlignBits); + return S_OK; + +} + +HRESULT LzmaDecoderCodeReal( + LzmaDecoder *lzmaDecoder, + UINT64 *anInSize, + UINT64 *anOutSize) +{ + BOOL aPeviousIsMatch = FALSE; + BYTE aPreviousByte = 0; + UINT32 aRepDistances[kNumRepDistances]; + int i; + UINT64 aNowPos64 = 0; + UINT64 aSize = *anOutSize; + ISequentialInStream my_in_stream; +// WindowOut out_window; + CState aState; + + CStateInit(&aState); + + if (anOutSize == NULL) + { + printf("CodeReal: invalid argument %x\n", (UINT32) anOutSize ); + return E_INVALIDARG; + } + + + LzmaDecoderInit(lzmaDecoder); + + my_in_stream.data = in_stream.data; + my_in_stream.remainingBytes = in_stream.remainingBytes; + + for(i = 0 ; i < (int) kNumRepDistances; i++) + aRepDistances[i] = 0; + + //while(aNowPos64 < aSize) + while(my_in_stream.remainingBytes > 0) + { + UINT64 aNext = MyMin(aNowPos64 + (1 << 18), aSize); + while(aNowPos64 < aNext) + { + UINT32 aPosState = (UINT32)(aNowPos64) & lzmaDecoder->m_PosStateMask; + if (BitDecode(&my_in_stream, + &lzmaDecoder->m_MainChoiceDecoders[aState][aPosState], + &lzmaDecoder->m_RangeDecoder) == (UINT32) kMainChoiceLiteralIndex) + { + CStateUpdateChar(&aState); + if(aPeviousIsMatch) + { + BYTE aMatchByte = OutWindowGetOneByte(0 - aRepDistances[0] - 1); + aPreviousByte = LitDecodeWithMatchByte(&my_in_stream, + &lzmaDecoder->m_LiteralDecoder, + &lzmaDecoder->m_RangeDecoder, + (UINT32)(aNowPos64), + aPreviousByte, + aMatchByte); + aPeviousIsMatch = FALSE; + } + else + aPreviousByte = LitDecodeNormal(&my_in_stream, + &lzmaDecoder->m_LiteralDecoder, + &lzmaDecoder->m_RangeDecoder, + (UINT32)(aNowPos64), + aPreviousByte); + OutWindowPutOneByte(aPreviousByte); + aNowPos64++; + } + else + { + UINT32 aDistance, aLen; + aPeviousIsMatch = TRUE; + if(BitDecode(&my_in_stream, + &lzmaDecoder->m_MatchChoiceDecoders[aState], + &lzmaDecoder->m_RangeDecoder) == (UINT32) kMatchChoiceRepetitionIndex) + { + if(BitDecode(&my_in_stream, + &lzmaDecoder->m_MatchRepChoiceDecoders[aState], + &lzmaDecoder->m_RangeDecoder) == 0) + { + if(BitDecode(&my_in_stream, + &lzmaDecoder->m_MatchRepShortChoiceDecoders[aState][aPosState], + &lzmaDecoder->m_RangeDecoder) == 0) + { + CStateUpdateShortRep(&aState); + aPreviousByte = OutWindowGetOneByte(0 - aRepDistances[0] - 1); + OutWindowPutOneByte(aPreviousByte); + aNowPos64++; + continue; + } + aDistance = aRepDistances[0]; + } + else + { + if(BitDecode(&my_in_stream, + &lzmaDecoder->m_MatchRep1ChoiceDecoders[aState], + &lzmaDecoder->m_RangeDecoder) == 0) + { + aDistance = aRepDistances[1]; + aRepDistances[1] = aRepDistances[0]; + } + else + { + if (BitDecode(&my_in_stream, + &lzmaDecoder->m_MatchRep2ChoiceDecoders[aState], + &lzmaDecoder->m_RangeDecoder) == 0) + { + aDistance = aRepDistances[2]; + } + else + { + aDistance = aRepDistances[3]; + aRepDistances[3] = aRepDistances[2]; + } + aRepDistances[2] = aRepDistances[1]; + aRepDistances[1] = aRepDistances[0]; + } + aRepDistances[0] = aDistance; + } + aLen = LenDecode(&my_in_stream, + &lzmaDecoder->m_RepMatchLenDecoder, + &lzmaDecoder->m_RangeDecoder, + aPosState) + kMatchMinLen; + CStateUpdateRep(&aState); + } + else + { + UINT32 aPosSlot; + aLen = kMatchMinLen + LenDecode(&my_in_stream, + &lzmaDecoder->m_LenDecoder, + &lzmaDecoder->m_RangeDecoder, + aPosState); + CStateUpdateMatch(&aState); + aPosSlot = BitTreeDecode(&my_in_stream, + &lzmaDecoder->m_PosSlotDecoder[GetLenToPosState(aLen)], + &lzmaDecoder->m_RangeDecoder); + if (aPosSlot >= (UINT32) kStartPosModelIndex) + { + aDistance = kDistStart[aPosSlot]; + if (aPosSlot < (UINT32) kEndPosModelIndex) + aDistance += ReverseBitTreeDecoder2Decode(&my_in_stream, + &lzmaDecoder->m_PosDecoders[aPosSlot - kStartPosModelIndex], + &lzmaDecoder->m_RangeDecoder); + else + { + aDistance += (RangeDecodeDirectBits(&my_in_stream, + &lzmaDecoder->m_RangeDecoder, + kDistDirectBits[aPosSlot] - kNumAlignBits) << kNumAlignBits); + aDistance += ReverseBitTreeDecoderDecode(&my_in_stream, + &lzmaDecoder->m_PosAlignDecoder, + &lzmaDecoder->m_RangeDecoder); + } + } + else + aDistance = aPosSlot; + + + aRepDistances[3] = aRepDistances[2]; + aRepDistances[2] = aRepDistances[1]; + aRepDistances[1] = aRepDistances[0]; + + aRepDistances[0] = aDistance; + } + if (aDistance >= aNowPos64) + { + printf("CodeReal: invalid data\n" ); + return E_INVALIDDATA; + } + OutWindowCopyBackBlock(aDistance, aLen); + aNowPos64 += aLen; + aPreviousByte = OutWindowGetOneByte(0 - 1); + } + } + } + + //BRCM modification + LzmaDecoderFreeBuffer(lzmaDecoder); + + OutWindowFlush(); + return S_OK; +} + +HRESULT LzmaDecoderCode( + LzmaDecoder *lzmaDecoder, + UINT64 *anInSize, + UINT64 *anOutSize) +{ + + UINT32 aStartValue = 0; + int i; + + for (i = 0; i < kDistTableSizeMax; i++) + { + kDistStart[i] = aStartValue; + aStartValue += (1 << kDistDirectBits[i]); + } + return LzmaDecoderCodeReal( + lzmaDecoder, + anInSize, + anOutSize); +} + +HRESULT LzmaDecoderReadCoderProperties(LzmaDecoder *lzmaDecoder) +{ + UINT32 aNumPosStateBits; + UINT32 aLiteralPosStateBits; + UINT32 aLiteralContextBits; + UINT32 aDictionarySize; + BYTE aRemainder; + UINT32 aProcessesedSize; + + BYTE aByte; + RETURN_IF_NOT_S_OK(InStreamRead(&aByte, + sizeof(aByte), + &aProcessesedSize)); + + if (aProcessesedSize != sizeof(aByte)) + return E_INVALIDARG; + + aLiteralContextBits = aByte % 9; + aRemainder = aByte / 9; + aLiteralPosStateBits = aRemainder % 5; + aNumPosStateBits = aRemainder / 5; + + RETURN_IF_NOT_S_OK(InStreamRead(&aDictionarySize, + sizeof(aDictionarySize), + &aProcessesedSize)); + + if (aProcessesedSize != sizeof(aDictionarySize)) + return E_INVALIDARG; + + RETURN_IF_NOT_S_OK( LzmaDecoderSetDictionarySize(lzmaDecoder, + aDictionarySize) ); + RETURN_IF_NOT_S_OK( LzmaDecoderSetLiteralProperties(lzmaDecoder, + aLiteralPosStateBits, + aLiteralContextBits) ); + RETURN_IF_NOT_S_OK( LzmaDecoderSetPosBitsProperties(lzmaDecoder, + aNumPosStateBits) ); + + return S_OK; +} + diff --git a/hostTools/lzma/decompress/LZMADecoder.h b/hostTools/lzma/decompress/LZMADecoder.h new file mode 100644 index 0000000..76fa536 --- /dev/null +++ b/hostTools/lzma/decompress/LZMADecoder.h @@ -0,0 +1,60 @@ +#ifndef __LZARITHMETIC_DECODER_H +#define __LZARITHMETIC_DECODER_H + +#include "WindowOut.h" +#include "LZMA.h" +#include "LenCoder.h" +#include "LiteralCoder.h" + + +typedef struct LzmaDecoder +{ + CRangeDecoder m_RangeDecoder; + + CBitDecoder m_MainChoiceDecoders[kNumStates][kNumPosStatesMax]; + CBitDecoder m_MatchChoiceDecoders[kNumStates]; + CBitDecoder m_MatchRepChoiceDecoders[kNumStates]; + CBitDecoder m_MatchRep1ChoiceDecoders[kNumStates]; + CBitDecoder m_MatchRep2ChoiceDecoders[kNumStates]; + CBitDecoder m_MatchRepShortChoiceDecoders[kNumStates][kNumPosStatesMax]; + + CBitTreeDecoder m_PosSlotDecoder[kNumLenToPosStates]; + + CReverseBitTreeDecoder2 m_PosDecoders[kNumPosModels]; + CReverseBitTreeDecoder m_PosAlignDecoder; + + LenDecoder m_LenDecoder; + LenDecoder m_RepMatchLenDecoder; + + LitDecoder m_LiteralDecoder; + + UINT32 m_DictionarySize; + + UINT32 m_PosStateMask; +} LzmaDecoder; + + HRESULT LzmaDecoderCreate(LzmaDecoder *lzmaDecoder); + + HRESULT LzmaDecoderInit(LzmaDecoder *lzmaDecoder); + +//static inline HRESULT LzmaDecoderFlush() { return OutWindowFlush(); } + + HRESULT LzmaDecoderCodeReal( + LzmaDecoder *lzmaDecoder, +// ISequentialInStream *in_stream, + UINT64 *anInSize, +// WindowOut *out_window, + UINT64 *anOutSize); + + + void LzmaDecoderConstructor( LzmaDecoder *lzmaDecoder ); + + HRESULT LzmaDecoderCode( LzmaDecoder *lzmaDecoder, UINT64 *anInSize, UINT64 *anOutSize); + HRESULT LzmaDecoderReadCoderProperties(LzmaDecoder *lzmaDecoder ); + + HRESULT LzmaDecoderSetDictionarySize(LzmaDecoder *lzmaDecoder, UINT32 aDictionarySize); + HRESULT LzmaDecoderSetLiteralProperties(LzmaDecoder *lzmaDecoder, UINT32 aLiteralPosStateBits, UINT32 aLiteralContextBits); + HRESULT LzmaDecoderSetPosBitsProperties(LzmaDecoder *lzmaDecoder, UINT32 aNumPosStateBits); + + +#endif diff --git a/hostTools/lzma/decompress/LenCoder.h b/hostTools/lzma/decompress/LenCoder.h new file mode 100644 index 0000000..40552b8 --- /dev/null +++ b/hostTools/lzma/decompress/LenCoder.h @@ -0,0 +1,75 @@ +#ifndef __LENCODER_H +#define __LENCODER_H + +#include "BitTreeCoder.h" + + +#define kNumPosStatesBitsMax 4 +#define kNumPosStatesMax 16 + + +#define kNumPosStatesBitsEncodingMax 4 +#define kNumPosStatesEncodingMax 16 + + +//#define kNumMoveBits 5 + +#define kNumLenBits 3 +#define kNumLowSymbols (1 << kNumLenBits) + +#define kNumMidBits 3 +#define kNumMidSymbols (1 << kNumMidBits) + +#define kNumHighBits 8 + +#define kNumSymbolsTotal (kNumLowSymbols + kNumMidSymbols + (1 << kNumHighBits)) + +typedef struct LenDecoder +{ + CBitDecoder m_Choice; + CBitDecoder m_Choice2; + CBitTreeDecoder m_LowCoder[kNumPosStatesMax]; + CBitTreeDecoder m_MidCoder[kNumPosStatesMax]; + CBitTreeDecoder m_HighCoder; + UINT32 m_NumPosStates; +} LenDecoder; + +INLINE void LenDecoderCreate(LenDecoder *lenCoder, UINT32 aNumPosStates) + { + lenCoder->m_NumPosStates = aNumPosStates; + } + +INLINE void LenDecoderInit(LenDecoder *lenCoder) + { + UINT32 aPosState; + BitDecoderInit(&lenCoder->m_Choice); + for (aPosState = 0; aPosState < lenCoder->m_NumPosStates; aPosState++) + { + BitTreeDecoderInit(&lenCoder->m_LowCoder[aPosState],kNumLenBits); + BitTreeDecoderInit(&lenCoder->m_MidCoder[aPosState],kNumMidBits); + } + BitTreeDecoderInit(&lenCoder->m_HighCoder,kNumHighBits); + BitDecoderInit(&lenCoder->m_Choice2); + } + +INLINE UINT32 LenDecode(ISequentialInStream *in_stream, LenDecoder *lenCoder, CRangeDecoder *aRangeDecoder, UINT32 aPosState) + { + if(BitDecode(in_stream, &lenCoder->m_Choice, aRangeDecoder) == 0) + return BitTreeDecode(in_stream, &lenCoder->m_LowCoder[aPosState],aRangeDecoder); + else + { + UINT32 aSymbol = kNumLowSymbols; + if(BitDecode(in_stream, &lenCoder->m_Choice2, aRangeDecoder) == 0) + aSymbol += BitTreeDecode(in_stream, &lenCoder->m_MidCoder[aPosState],aRangeDecoder); + else + { + aSymbol += kNumMidSymbols; + aSymbol += BitTreeDecode(in_stream, &lenCoder->m_HighCoder,aRangeDecoder); + } + return aSymbol; + } + } + + + +#endif diff --git a/hostTools/lzma/decompress/LiteralCoder.h b/hostTools/lzma/decompress/LiteralCoder.h new file mode 100644 index 0000000..2b1670d --- /dev/null +++ b/hostTools/lzma/decompress/LiteralCoder.h @@ -0,0 +1,146 @@ +#ifndef __LITERALCODER_H +#define __LITERALCODER_H + +#include "AriBitCoder.h" +#include "RCDefs.h" + +//BRCM modification start +#ifdef _HOST_TOOL +#include "stdio.h" +#include "malloc.h" +#endif + +#ifdef _CFE_ +#include "lib_malloc.h" +#include "lib_printf.h" +#define malloc(x) KMALLOC(x, 0) +#define free(x) KFREE(x) +#endif + +#ifdef __KERNEL__ +#include +#include +#define printf printk +//#define malloc(x) kmalloc(x,GFP_KERNEL) +#define malloc(x) vmalloc(x) +#define free(x) vfree(x) +#endif +//BRCM modification end + +//#define kNumMoveBits 5 + +typedef struct LitDecoder2 +{ + CBitDecoder m_Decoders[3][1 << 8]; +} LitDecoder2; + + +INLINE void LitDecoder2Init(LitDecoder2 *litDecoder2) + { + int i, j; + for (i = 0; i < 3; i++) + for (j = 1; j < (1 << 8); j++) + BitDecoderInit(&litDecoder2->m_Decoders[i][j]); + } + +INLINE BYTE LitDecoder2DecodeNormal(ISequentialInStream *in_stream, LitDecoder2 *litDecoder2, CRangeDecoder *aRangeDecoder) + { + UINT32 aSymbol = 1; + UINT32 aRange = aRangeDecoder->m_Range; + UINT32 aCode = aRangeDecoder->m_Code; + do + { + RC_GETBIT(kNumMoveBits, litDecoder2->m_Decoders[0][aSymbol], aSymbol) + } + while (aSymbol < 0x100); + aRangeDecoder->m_Range = aRange; + aRangeDecoder->m_Code = aCode; + return aSymbol; + } + +INLINE BYTE LitDecoder2DecodeWithMatchByte(ISequentialInStream *in_stream, LitDecoder2 *litDecoder2, CRangeDecoder *aRangeDecoder, BYTE aMatchByte) + { + UINT32 aSymbol = 1; + UINT32 aRange = aRangeDecoder->m_Range; + UINT32 aCode = aRangeDecoder->m_Code; + do + { + UINT32 aBit; + UINT32 aMatchBit = (aMatchByte >> 7) & 1; + aMatchByte <<= 1; + RC_GETBIT2(kNumMoveBits, litDecoder2->m_Decoders[1 + aMatchBit][aSymbol], aSymbol, + aBit = 0, aBit = 1) + if (aMatchBit != aBit) + { + while (aSymbol < 0x100) + { + RC_GETBIT(kNumMoveBits, litDecoder2->m_Decoders[0][aSymbol], aSymbol) + } + break; + } + } + while (aSymbol < 0x100); + aRangeDecoder->m_Range = aRange; + aRangeDecoder->m_Code = aCode; + return aSymbol; + } + + +typedef struct LitDecoder +{ + LitDecoder2 *m_Coders; + UINT32 m_NumPrevBits; + UINT32 m_NumPosBits; + UINT32 m_PosMask; +} LitDecoder; + + +// LitDecoder(): m_Coders(0) {} +// ~LitDecoder() { Free(); } + +/* +INLINE void LitDecoderFree(LitDecoder *litDecoder) + { + free( (char *) litDecoder->m_Coders ); + litDecoder->m_Coders = 0; + } +*/ + +INLINE void LitDecoderCreate(LitDecoder *litDecoder, UINT32 aNumPosBits, UINT32 aNumPrevBits) + { +// LitDecoderFree(litDecoder); + UINT32 aNumStates; + litDecoder->m_NumPosBits = aNumPosBits; + litDecoder->m_PosMask = (1 << aNumPosBits) - 1; + litDecoder->m_NumPrevBits = aNumPrevBits; + aNumStates = 1 << (aNumPrevBits + aNumPosBits); + litDecoder->m_Coders = (LitDecoder2*) malloc( sizeof( LitDecoder2 ) * aNumStates ); + //printf("malloc in LitDecoderCreate=%d\n",sizeof( LitDecoder2 ) * aNumStates); + if (litDecoder->m_Coders == 0) + printf( "Error allocating memory for LitDecoder m_Coders!\n" ); + } + +INLINE void LitDecoderInit(LitDecoder *litDecoder) + { + UINT32 i; + UINT32 aNumStates = 1 << (litDecoder->m_NumPrevBits + litDecoder->m_NumPosBits); + for (i = 0; i < aNumStates; i++) + LitDecoder2Init(&litDecoder->m_Coders[i]); + } + +INLINE UINT32 LitDecoderGetState(LitDecoder *litDecoder, UINT32 aPos, BYTE aPrevByte) + { + return ((aPos & litDecoder->m_PosMask) << litDecoder->m_NumPrevBits) + (aPrevByte >> (8 - litDecoder->m_NumPrevBits)); + } + +INLINE BYTE LitDecodeNormal(ISequentialInStream *in_stream, LitDecoder *litDecoder, CRangeDecoder *aRangeDecoder, UINT32 aPos, BYTE aPrevByte) + { + return LitDecoder2DecodeNormal(in_stream, &litDecoder->m_Coders[LitDecoderGetState(litDecoder, aPos, aPrevByte)], aRangeDecoder); + } + +INLINE BYTE LitDecodeWithMatchByte(ISequentialInStream *in_stream, LitDecoder *litDecoder, CRangeDecoder *aRangeDecoder, UINT32 aPos, BYTE aPrevByte, BYTE aMatchByte) + { + return LitDecoder2DecodeWithMatchByte(in_stream, &litDecoder->m_Coders[LitDecoderGetState(litDecoder, aPos, aPrevByte)], aRangeDecoder, aMatchByte); + } + +#endif diff --git a/hostTools/lzma/decompress/Makefile b/hostTools/lzma/decompress/Makefile new file mode 100644 index 0000000..e21ff1d --- /dev/null +++ b/hostTools/lzma/decompress/Makefile @@ -0,0 +1,6 @@ + +BSPOBJS += \ + 7zlzma.o \ + LZMADecoder.o \ + IInOutStreams.o \ + diff --git a/hostTools/lzma/decompress/Portable.h b/hostTools/lzma/decompress/Portable.h new file mode 100644 index 0000000..698f30b --- /dev/null +++ b/hostTools/lzma/decompress/Portable.h @@ -0,0 +1,59 @@ +#ifndef __PORTABLE_H +#define __PORTABLE_H + +//BRCM modification +#ifdef _HOST_TOOL +#include +#endif + +#ifdef _CFE_ +#include +#endif + +#ifdef __KERNEL__ +#include +#endif + +//bcm +//#ifdef __GNUC__ +//#include +//#define INLINE static inline +//#else +typedef char INT8; +typedef unsigned char UINT8; +typedef short INT16; +typedef unsigned short UINT16; +typedef int INT32; +typedef unsigned int UINT32; +typedef int BOOL; +#define INLINE static inline +//#define INLINE static __inline__ +//#endif +typedef long long INT64; // %%%% Changed from "long long" +typedef unsigned long long UINT64; // %%%% Changed from "long long" + +typedef UINT8 BYTE; +typedef UINT16 WORD; +typedef UINT32 DWORD; + +typedef unsigned UINT_PTR; +#define FALSE 0 +#define TRUE 1 + +#define HRESULT int +#define S_OK 0 +#define E_INVALIDARG -1 +#define E_OUTOFMEMORY -2 +#define E_FAIL -3 +#define E_INTERNAL_ERROR -4 +#define E_INVALIDDATA -5 + +#define MyMin( a, b ) ( a < b ? a : b ) + +#define MyMax( a, b ) ( a > b ? a : b ) + +#define kNumMoveBits 5 + +#define RETURN_IF_NOT_S_OK(x) { HRESULT __aResult_ = (x); if(__aResult_ != S_OK) return __aResult_; } + +#endif diff --git a/hostTools/lzma/decompress/RCDefs.h b/hostTools/lzma/decompress/RCDefs.h new file mode 100644 index 0000000..f260ab4 --- /dev/null +++ b/hostTools/lzma/decompress/RCDefs.h @@ -0,0 +1,43 @@ +#ifndef __RCDEFS_H +#define __RCDEFS_H + +#include "AriBitCoder.h" + +/* +#define RC_INIT_VAR \ + UINT32 aRange = aRangeDecoder->m_Range; \ + UINT32 aCode = aRangeDecoder->m_Code; + +#define RC_FLUSH_VAR \ + aRangeDecoder->m_Range = aRange; \ + aRangeDecoder->m_Code = aCode; +*/ + + +#if 1 +#define RC_GETBIT2(aNumMoveBits, aProb, aModelIndex, Action0, Action1) \ + {UINT32 aNewBound = (aRange >> kNumBitModelTotalBits) * aProb; \ + if (aCode < aNewBound) \ + { \ + Action0; \ + aRange = aNewBound; \ + aProb += (kBitModelTotal - aProb) >> aNumMoveBits; \ + aModelIndex <<= 1; \ + } \ + else \ + { \ + Action1; \ + aRange -= aNewBound; \ + aCode -= aNewBound; \ + aProb -= (aProb) >> aNumMoveBits; \ + aModelIndex = (aModelIndex << 1) + 1; \ + }} \ + if (aRange < kTopValue) \ + { \ + aCode = (aCode << 8) | InStreamReadByte(in_stream); \ + aRange <<= 8; } + +#define RC_GETBIT(aNumMoveBits, aProb, aModelIndex) RC_GETBIT2(aNumMoveBits, aProb, aModelIndex, ; , ;) +#endif + +#endif diff --git a/hostTools/lzma/decompress/RangeCoder.h b/hostTools/lzma/decompress/RangeCoder.h new file mode 100644 index 0000000..ae6f974 --- /dev/null +++ b/hostTools/lzma/decompress/RangeCoder.h @@ -0,0 +1,56 @@ +#ifndef __COMPRESSION_RANGECODER_H +#define __COMPRESSION_RANGECODER_H + +#include "IInOutStreams.h" + +#define kNumTopBits 24 +#define kTopValue (1 << kNumTopBits) + +typedef struct CRangeDecoder +{ + UINT32 m_Range; + UINT32 m_Code; +} CRangeDecoder; + + + +INLINE void RangeDecoderInit( + ISequentialInStream *in_stream, + CRangeDecoder *rangeDecoder) + { + int i; + rangeDecoder->m_Code = 0; + rangeDecoder->m_Range = (UINT32)(-1); + for(i = 0; i < 5; i++) + rangeDecoder->m_Code = (rangeDecoder->m_Code << 8) | InStreamReadByte(in_stream); + } + +INLINE UINT32 RangeDecodeDirectBits( + ISequentialInStream *in_stream, + CRangeDecoder *rangeDecoder, + UINT32 aNumTotalBits) + { + UINT32 aRange = rangeDecoder->m_Range; + UINT32 aCode = rangeDecoder->m_Code; + UINT32 aResult = 0; + UINT32 i; + for (i = aNumTotalBits; i > 0; i--) + { + UINT32 t; + aRange >>= 1; + t = (aCode - aRange) >> 31; + aCode -= aRange & (t - 1); + aResult = (aResult << 1) | (1 - t); + + if (aRange < kTopValue) + { + aCode = (aCode << 8) | InStreamReadByte(in_stream); + aRange <<= 8; + } + } + rangeDecoder->m_Range = aRange; + rangeDecoder->m_Code = aCode; + return aResult; + } + +#endif diff --git a/hostTools/lzma/decompress/WindowOut.h b/hostTools/lzma/decompress/WindowOut.h new file mode 100644 index 0000000..d774cac --- /dev/null +++ b/hostTools/lzma/decompress/WindowOut.h @@ -0,0 +1,47 @@ +#ifndef __STREAM_WINDOWOUT_H +#define __STREAM_WINDOWOUT_H + +#include "IInOutStreams.h" + +typedef struct WindowOut +{ + BYTE *Buffer; + UINT32 Pos; +} WindowOut; + +extern WindowOut out_window; + +#define OutWindowInit() \ + { \ + out_window.Buffer = (BYTE *) out_stream.data; \ + out_window.Pos = 0; \ + } + +#define OutWindowFlush() \ + { \ + OutStreamSizeSet( out_window.Pos ); \ + } + +// BRCM modification +INLINE void OutWindowCopyBackBlock(UINT32 aDistance, UINT32 aLen) + { + BYTE *p = out_window.Buffer + out_window.Pos; + UINT32 i; + aDistance++; + for(i = 0; i < aLen; i++) + p[i] = p[i - aDistance]; + out_window.Pos += aLen; + } + + +#define OutWindowPutOneByte(aByte) \ + { \ + out_window.Buffer[out_window.Pos++] = aByte; \ + } + +#define OutWindowGetOneByte(anIndex) \ + (out_window.Buffer[out_window.Pos + anIndex]) + + + +#endif diff --git a/hostTools/lzma/decompress/vxTypesOld.h b/hostTools/lzma/decompress/vxTypesOld.h new file mode 100644 index 0000000..7ee57bf --- /dev/null +++ b/hostTools/lzma/decompress/vxTypesOld.h @@ -0,0 +1,289 @@ +/* vxTypesOld.h - old VxWorks type definition header */ + +/* Copyright 1984-1997 Wind River Systems, Inc. */ + +/* +modification history +-------------------- +02c,15aug97,cym added simnt support. +02d,26mar97,cdp added Thumb (ARM7TDMI_T) support. +02c,28nov96,cdp added ARM support. +02b,28sep95,ms removed "static __inline__" (SPR #4500) +02b,12jul95,ism added simsolaris support +02a,19mar95,dvs removed tron references. +01z,01sep94,ism fixed comment as per SPR# 1512. +01y,02dec93,pme added Am29K family support. +01x,12jun93,rrr vxsim. +02a,26may94,yao added PPC support. +01w,09jun93,hdn added support for I80X86 +01v,12feb93,srh added C++ versions of FUNCPTR, et al. +01u,13nov92,dnw added definition of VOID (SPR #1781) +01t,02oct92,srh replaced conditional around volatile, const, and signed so + they won't be elided when __STDC__ is defined. + added __cplusplus to __STDC__ condition. +01s,22sep92,rrr added support for c++ +01r,08sep92,smb made some additions for the MIPS. +01q,07sep92,smb added __STDC__ and modes to maintain compatibility with 5.0 +01p,07jul92,rrr moved STACK_GROW and ENDIAN to vxArch.h +01o,03jul92,smb changed name from vxTypes.h. +01n,26may92,rrr the tree shuffle +01m,25nov91,llk included sys/types.h. +01l,04oct91,rrr passed through the ansification filter + -fixed #else and #endif + -removed TINY and UTINY + -changed VOID to void + -changed ASMLANGUAGE to _ASMLANGUAGE + -changed copyright notice +01k,01oct91,jpb fixed MIPS conditional for undefined CPU_FAMILY. +01j,20sep91,wmd conditionalized out defines for const, unsigned and volatile + for the MIPS architecture. +01i,02aug91,ajm added support for MIPS_R3k. +01h,15may91,gae added define for "signed" when not available for pre-ANSI. +01g,29apr91,hdn added defines and macros for TRON architecture. +01f,28apr91,del added defines of __volatile__ and __const__ if !_STDC_ + && _GNUC__ +01f,24mar91,del added INSTR * define for I960. +01e,28jan91,kdl added DBLFUNCPTR and FLTFUNCPTR. +01d,25oct90,dnw changed void to void except when linting. +01c,05oct90,shl added copyright notice. + made #endif ANSI style. +01b,10aug90,dnw added VOIDFUNCPTR +01a,29may90,del written. +*/ + +/* +DESCRIPTION +This header file contains a mixture of stuff. +1) the old style typedefs (ie. POSIX now says they must end with _t). + These will be phased out gradually. +2) a mechanism for getting rid of const warning which are produced by the + GNU C compiler. Hopefully, this will be removed in the future. +3) macros that are so longer needed for vxWorks source code but maybe needed + by some customer applications and are therefore provided for backward + compatability. +4) system III typedefs (used by netinet) which do not fit in anywhere else. + +*/ + +#ifndef __INCvxTypesOldh +#define __INCvxTypesOldh + +#ifdef __cplusplus +extern "C" { +#endif + +#include "sys/types.h" + +/* vxWorks types */ + +typedef char INT8; +typedef short INT16; +typedef int INT32; + +typedef unsigned char UINT8; +typedef unsigned short UINT16; +typedef unsigned int UINT32; + +typedef unsigned char UCHAR; +typedef unsigned short USHORT; +typedef unsigned int UINT; +typedef unsigned long ULONG; + +typedef int BOOL; +typedef int STATUS; +typedef int ARGINT; + +typedef void VOID; + +#ifdef __cplusplus +typedef int (*FUNCPTR) (...); /* ptr to function returning int */ +typedef void (*VOIDFUNCPTR) (...); /* ptr to function returning void */ +typedef double (*DBLFUNCPTR) (...); /* ptr to function returning double*/ +typedef float (*FLTFUNCPTR) (...); /* ptr to function returning float */ +#else +typedef int (*FUNCPTR) (); /* ptr to function returning int */ +typedef void (*VOIDFUNCPTR) (); /* ptr to function returning void */ +typedef double (*DBLFUNCPTR) (); /* ptr to function returning double*/ +typedef float (*FLTFUNCPTR) (); /* ptr to function returning float */ +#endif /* _cplusplus */ + + +/* This structure and the following definitions are needed to get rid + of const warning produced by the GNU C compiler. + */ + +#if defined(__STDC__) || defined(__cplusplus) +typedef union + { + long pm_int; + void *pm_v; + const void *pm_cv; + char *pm_c; + unsigned char *pm_uc; + + signed char *pm_sc; + const char *pm_cc; + const unsigned char *pm_cuc; + const signed char *pm_csc; + short *pm_s; + ushort_t *pm_us; + const short *pm_cs; + const ushort_t *pm_cus; + int *pm_i; + uint_t *pm_ui; + const int *pm_ci; + const uint_t *pm_cui; + long *pm_l; + ulong_t *pm_ul; + const long *pm_cl; + const ulong_t *pm_cul; + + int8_t *pm_i8; + uint8_t *pm_ui8; + const int8_t *pm_ci8; + const uint8_t *pm_cui8; + int16_t *pm_i16; + uint16_t *pm_ui16; + const int16_t *pm_ci16; + const uint16_t *pm_cui16; + int32_t *pm_i32; + uint32_t *pm_ui32; + const int32_t *pm_ci32; + const uint32_t *pm_cui32; +#if _ARCH_MOVE_SIZE > 4 + int64_t *pm_i64; + const int64_t *pm_ci64; +#if _ARCH_MOVE_SIZE > 8 + int128_t *pm_i128; + const int128_t *pm_ci128; +#endif +#endif + } pointer_mix_t; + +#define CHAR_FROM_CONST(x) (char *)(x) +#define VOID_FROM_CONST(x) (void *)(x) + +#endif /* __STDC__ */ + +#define STACK_DIR _ARCH_STACK_DIR +#define ALIGN_MEMORY _ARCH_ALIGN_MEMORY +#define ALIGN_STACK _ARCH_ALIGN_STACK +#define ALIGN_REGS _ARCH_ALIGN_REGS + +#define NBBY 8 /* number of bits in a byte */ + +/* modes - must match O_RDONLY/O_WRONLY/O_RDWR in ioLib.h! */ + +#define READ 0 +#define WRITE 1 +#define UPDATE 2 + +/* Select uses bit masks of file descriptors in longs. + * These macros manipulate such bit fields (the filesystem macros use chars). + * FD_SETSIZE may be defined by the user, but the default here + * should be >= maxFiles parameter in iosInit call found in usrConfig.c. + * If this define is changed, recompile the source, or else select() will + * not work. + */ + +#ifndef FD_SETSIZE +#define FD_SETSIZE 256 +#endif /* FD_SETSIZE */ + +typedef long fd_mask; +#define NFDBITS (sizeof(fd_mask) * NBBY) /* bits per mask */ +#ifndef howmany +#define howmany(x, y) ((unsigned int)(((x)+((y)-1)))/(unsigned int)(y)) +#endif /* howmany */ + +typedef struct fd_set + { + fd_mask fds_bits[howmany(FD_SETSIZE, NFDBITS)]; + } fd_set; + +#define FD_SET(n, p) ((p)->fds_bits[(n)/NFDBITS] |= (1 << ((n) % NFDBITS))) +#define FD_CLR(n, p) ((p)->fds_bits[(n)/NFDBITS] &= ~(1 << ((n) % NFDBITS))) +#define FD_ISSET(n, p) ((p)->fds_bits[(n)/NFDBITS] & (1 << ((n) % NFDBITS))) +#define FD_ZERO(p) bzero((char *)(p), sizeof(*(p))) + + +/* system III typedefs (used by netinet) */ + +typedef unsigned char u_char; +typedef unsigned short u_short; +typedef unsigned int u_int; +typedef unsigned long u_long; +typedef unsigned short ushort; + + +/* historical definitions - now obsolete */ + +typedef char TBOOL; /* obsolete */ + + +/* architecture dependent typedefs */ + +#ifdef CPU_FAMILY + +#if CPU_FAMILY==MC680X0 +typedef unsigned short INSTR; /* word-aligned instructions */ +#endif /* CPU_FAMILY==MC680X0 */ + +#if CPU_FAMILY==SPARC || CPU_FAMILY==MIPS || CPU_FAMILY==SIMSPARCSUNOS || CPU_FAMILY==SIMHPPA || CPU_FAMILY==SIMSPARCSOLARIS +typedef unsigned long INSTR; /* 32 bit word-aligned instructions */ +#endif /* CPU_FAMILY==SPARC || CPU_FAMILY==MIPS || CPU_FAMILY==SIMSPARCSUNOS || CPU_FAMILY==SIMHPPA || CPU_FAMILY==SIMSPARCSOLARIS */ + +#if CPU_FAMILY==I960 +typedef unsigned long INSTR; /* 32 bit word-aligned instructions */ +#endif /* CPU_FAMILY==I960 */ + +#if CPU_FAMILY==I80X86 || CPU_FAMILY==SIMNT +typedef unsigned char INSTR; /* char instructions */ +#endif /* CPU_FAMILY==I80X86 || CPU_FAMILY==SIMNT */ + +#if CPU_FAMILY==AM29XXX +typedef unsigned long INSTR; /* 32 bit word-aligned instructions */ +#endif /* CPU_FAMILY==AM29XXX */ + +#if (CPU_FAMILY==PPC) +typedef unsigned long INSTR; /* 32 bit word-aligned instructions */ +#endif /* (CPU_FAMILY==PPC) */ + +#if CPU_FAMILY==ARM +#if CPU==ARM7TDMI_T +typedef unsigned short INSTR; /* 16 bit instructions */ +#else +typedef unsigned long INSTR; /* 32 bit word-aligned instructions */ +#endif +#endif /* CPU_FAMILY==ARM */ + +#endif /* CPU_FAMILY */ + +/* ANSI type qualifiers */ + +#if !defined(__STDC__) && !defined(__cplusplus) + +#ifdef __GNUC__ +#define volatile __volatile__ +#define const __const__ +#define signed __signed__ +#else +#if !(defined(CPU_FAMILY) && CPU_FAMILY==MIPS) +#define volatile +#define const +#define signed +#endif /* !(defined(CPU_FAMILY) && CPU_FAMILY==MIPS) */ +#endif /* __GNUC__ */ + +#endif /* !defined(__STDC__) && !defined(__cplusplus) */ + +#if CPU_FAMILY==MIPS +#define CHAR_FROM_CONST(x) (char *)(x) +#define VOID_FROM_CONST(x) (void *)(x) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __INCvxTypesOldh */ diff --git a/hostTools/mkfs.jffs2 b/hostTools/mkfs.jffs2 new file mode 100644 index 0000000..f15cd12 Binary files /dev/null and b/hostTools/mkfs.jffs2 differ diff --git a/hostTools/nvramcrc.c b/hostTools/nvramcrc.c new file mode 100644 index 0000000..a737879 --- /dev/null +++ b/hostTools/nvramcrc.c @@ -0,0 +1,216 @@ +/* File Name : nvramcrc.c + * fix CRC of NVRAM in CFE + * gcc -m32 -o nvramcrc nvramcrc.c + * usage: nvramcrc inputfile outputfile + */ + + +#include +#include +#include +#include +#include + +typedef unsigned char byte; +typedef unsigned int UINT32; + +#define BUF_SIZE 100 * 1024 + +/*****************************************************************************/ +/* NVRAM Offset and definition */ +/*****************************************************************************/ +#define SDRAM_TYPE_ADDRESS_OFFSET 16 +#define THREAD_NUM_ADDRESS_OFFSET 20 +#define NVRAM_DATA_OFFSET 0x0580 + +#define NVRAM_PSI_DEFAULT 64 // default psi in K byes +#define ONEK 1024 +#define FLASH_LENGTH_BOOT_ROM (128*ONEK) +#define FLASH_RESERVED_AT_END (64*ONEK) /*reserved for PSI, scratch pad*/ + +#define NVRAM_LENGTH ONEK // 1k nvram +#define NVRAM_VERSION_NUMBER 2 +#define NVRAM_VERSION_NUMBER_ADDRESS 0 + +#define NVRAM_BOOTLINE_LEN 256 +#define NVRAM_BOARD_ID_STRING_LEN 16 +#define NVRAM_MAC_ADDRESS_LEN 6 +#define NVRAM_MAC_COUNT_MAX 32 +#define NVRAM_MAC_COUNT_DEFAULT 0 + +struct +{ + unsigned long ulVersion; + char szBootline[NVRAM_BOOTLINE_LEN]; + char szBoardId[NVRAM_BOARD_ID_STRING_LEN]; + unsigned long ulReserved1[2]; + unsigned long ulNumMacAddrs; + unsigned char ucaBaseMacAddr[NVRAM_MAC_ADDRESS_LEN]; + char chReserved[2]; + unsigned long ulCheckSum; +} NVRAM_DATA, *PNVRAM_DATA; + + + + +#define CRC32_INIT_VALUE 0xffffffff /* Initial CRC32 checksum value */ +#define CRC_LEN 4 + +static unsigned long Crc32_table[256] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D +}; + +byte buffer[BUF_SIZE]; + +/*************************************************************************** +// Function Name: getCrc32 +// Description : caculate the CRC 32 of the given data. +// Parameters : pdata - array of data. +// size - number of input data bytes. +// crc - either CRC32_INIT_VALUE or previous return value. +// Returns : crc. +****************************************************************************/ +UINT32 getCrc32(byte *pdata, UINT32 size, UINT32 crc) +{ + while (size-- > 4) + crc = (crc >> 8) ^ Crc32_table[(crc ^ *pdata++) & 0xff]; + crc = (crc >> 8) ^ Crc32_table[(crc ^ 0x00) & 0xff]; + crc = (crc >> 8) ^ Crc32_table[(crc ^ 0x00) & 0xff]; + crc = (crc >> 8) ^ Crc32_table[(crc ^ 0x00) & 0xff]; + crc = (crc >> 8) ^ Crc32_table[(crc ^ 0x00) & 0xff]; +printf("CRC = %x\n", crc); + return crc; +} + + +int main(int argc, char **argv) +{ + FILE *stream_in, *stream_out; + int count, numWritten = 0; + UINT32 imageCrc; + + if (argc != 3) + { + printf("Usage:\n"); + printf("nvramcrc input-cfe-file output-cfe-file-crc-fixed\n"); + exit(1); + } + if( (stream_in = fopen(argv[1], "rb")) == NULL) + { + printf("failed on open input file: %s\n", argv[1]); + exit(1); + } + if( (stream_out = fopen(argv[2], "wb+")) == NULL) + { + printf("failed on open output file: %s\n", argv[2]); + exit(1); + } + + count = 0; + imageCrc = CRC32_INIT_VALUE; + printf( "NVRAM_DATA length should be 300, calculated size = %d\n", sizeof(NVRAM_DATA)); + + fseek (stream_in, NVRAM_DATA_OFFSET, SEEK_SET); + count = fread( buffer, sizeof( char ), sizeof(NVRAM_DATA), stream_in ); + + if( ferror( stream_in ) ) + { + perror( "Read error" ); + exit(1); + } + + imageCrc = getCrc32(buffer, count, imageCrc); + + fseek (stream_in, 0L, SEEK_SET); + count = fread( buffer, sizeof( char ), BUF_SIZE, stream_in ); + numWritten = fwrite(buffer, sizeof(char), count, stream_out); + + if (ferror(stream_out)) + { + perror("nvramcrc: Write image file error"); + exit(1); + } + + // *** assume it is always in network byte order (big endian) + imageCrc = htonl(imageCrc); + + + // write the crc + fseek (stream_out, NVRAM_DATA_OFFSET+sizeof(NVRAM_DATA)-sizeof(NVRAM_DATA.ulCheckSum), SEEK_SET); + numWritten = fwrite((byte*)&imageCrc, sizeof(char), CRC_LEN, stream_out); + if (ferror(stream_out)) + { + perror("nvramcrc: Write image file error"); + exit(1); + } + + fclose(stream_in); + fclose(stream_out); + + printf( "crc = %x\n", imageCrc); + + exit(0); +} diff --git a/hostTools/nvramembed.c b/hostTools/nvramembed.c new file mode 100644 index 0000000..704d7e1 --- /dev/null +++ b/hostTools/nvramembed.c @@ -0,0 +1,416 @@ +/*************************************************************************** + * File Name : nvramembed.c + * + * Description: This program embeds the NVRAM into the CFE. + * + * Build: + +gcc -m32 -D _BCM96348_ -I ../shared/opensource/include/bcm963xx/ \ +-o nvramembed nvramembed.c ../shared/opensource/boardparms/bcm963xx/boardparms.c + + * + * Example : nvramembed -b blue5g9 -n 4 -m FA:BA:DA:18:12:01 + * -i cfe6348.bin -o cfe6348-nvram.bin + * where: + * -b = board id + * -n = number of mac address + * -m = base mac address + * -i = input file name + * -o = output file name + * + * + ***************************************************************************/ + +/* Includes. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef unsigned char byte; +typedef unsigned int UINT32; +#define BCMTAG_EXE_USE +#include "bcmTag.h" +#include "boardparms.h" +#define MAX_MAC_STR_LEN 19 // mac address string 18+1 in regular format - 02:18:10:11:aa:bb + +//#include "board.h" +#define SDRAM_TYPE_ADDRESS_OFFSET 16 +#define THREAD_NUM_ADDRESS_OFFSET 20 +#define NVRAM_DATA_OFFSET 0x0580 +#define NVRAM_VERSION_NUMBER 2 +#define NVRAM_BOOTLINE_LEN 256 +#define NVRAM_BOARD_ID_STRING_LEN 16 +#define NVRAM_MAC_ADDRESS_LEN 6 + +typedef struct +{ + unsigned long ulVersion; + char szBootline[NVRAM_BOOTLINE_LEN]; + char szBoardId[NVRAM_BOARD_ID_STRING_LEN]; + unsigned long ulReserved1[2]; + unsigned long ulNumMacAddrs; + unsigned char ucaBaseMacAddr[NVRAM_MAC_ADDRESS_LEN]; + char chReserved[2]; + unsigned long ulCheckSum; +} NVRAM_DATA, *PNVRAM_DATA; + + + +static struct option longopts[] = +{ + { "boardid", 1, 0, 'b'}, + { "numbermac", 1, 0, 'n'}, + { "macaddr", 1, 0, 'm'}, + { "inputfile", 1, 0, 'i'}, + { "outputfile", 1, 0, 'o'}, + { "help", 0, 0, 'h'}, + { "version", 0, 0, 'v'}, + { 0, 0, 0, 0 } +}; + +static char version[] = "Broadcom Creator version 0.3"; + +static char usage[] = +"Usage:\t%s [-bhvnmio] [--help] [--version]\n" +"\texample:\n" +"\t-b blue5g9 -n 4 -m FA:BA:DA:18:12:01 -i cfe6348.bin -o cfe6348-nvram.bin\n" +"\twhere:\n" +"\t[-b] board id name\n" +"\t[-n] number of mac address\n" +"\t[-m] base mac address\n" +"\t[-i] input image name\n" +"\t[-o] ouput image name\n"; + +static char *progname; + +/*************************************************************************** +// Function Name: getCrc32 +// Description : caculate the CRC 32 of the given data. +// Parameters : pdata - array of data. +// size - number of input data bytes. +// crc - either CRC32_INIT_VALUE or previous return value. +// Returns : crc. +****************************************************************************/ +UINT32 getCrc32(byte *pdata, UINT32 size, UINT32 crc) +{ + while (size-- > 0) + crc = (crc >> 8) ^ Crc32_table[(crc ^ *pdata++) & 0xff]; + + printf("\tNVRAM CRC = 0x%x\n\n", crc); + return crc; +} + +/*************************************************************************** +// Function Name: parsexdigit +// Description : parse hex digit +// Parameters : input char +// Returns : hex int +****************************************************************************/ +static int parsexdigit(char str) +{ + int digit; + + if ((str >= '0') && (str <= '9')) + digit = str - '0'; + else if ((str >= 'a') && (str <= 'f')) + digit = str - 'a' + 10; + else if ((str >= 'A') && (str <= 'F')) + digit = str - 'A' + 10; + else + return -1; + + return digit; +} + +/*************************************************************************** +// Function Name: parsehwaddr +// Description : convert in = "255.255.255.0" to fffffff00 +// Parameters : str = "255.255.255.0" ; +// Return : if not -1, hwaddr = fffffff00 +****************************************************************************/ +int parsehwaddr(unsigned char *str, unsigned char *hwaddr) +{ + int digit1,digit2; + int idx = 6; + + if (strlen(str) != MAX_MAC_STR_LEN-2) + return -1; + if (*(str+2) != ':' || *(str+5) != ':' || *(str+8) != ':' || *(str+11) != ':' || *(str+14) != ':') + return -1; + + while (*str && (idx > 0)) { + digit1 = parsexdigit(*str); + if (digit1 < 0) + return -1; + str++; + if (!*str) + return -1; + + if (*str == ':') { + digit2 = digit1; + digit1 = 0; + } + else { + digit2 = parsexdigit(*str); + if (digit2 < 0) + return -1; + str++; + } + + *hwaddr++ = (digit1 << 4) | digit2; + idx--; + + if (*str == ':') + str++; + } + return 0; +} + +/*************************************************************************** + * Function Name: fillInOutputBuffer + * Description : Fills the output buffer with the flash image contents. + * Returns : len of output buffer - OK, -1 - failed. + ***************************************************************************/ + +int fillInOutputBuffer(int cfesize, int numOfMac, unsigned char *ucaBaseMac, + int sdramType, char *szBoardId, unsigned long ulOutputSize, char *pszIn, + char *pszOut) +{ + int nLen; + unsigned long ulCfeLen = 0; + NVRAM_DATA nvramData; + UINT32 crc = CRC32_INIT_VALUE; + unsigned long ulCmtThread = 0; + + ulCfeLen = cfesize; + + nLen = ulCfeLen; + + if( nLen > (int) ulOutputSize ) + { + printf( "\nERROR: The image size is greater than the output buffer " + "allocated by this program.\n" ); + return( -1 ); + } + + memcpy( pszOut, pszIn, ulCfeLen ); + + // add board thread number in the CFE space + BpGetCMTThread(&ulCmtThread); + memcpy(pszOut + THREAD_NUM_ADDRESS_OFFSET, (char*)&ulCmtThread, sizeof(int)); + + // add memory size type in the CFE space + sdramType = htonl(sdramType); + memcpy(pszOut + SDRAM_TYPE_ADDRESS_OFFSET, (char*)&sdramType, sizeof(int)); + + // create nvram data. No default bootline. CFE will create one on the first boot + memset((char *)&nvramData, 0, sizeof(NVRAM_DATA)); + nvramData.ulVersion = htonl(NVRAM_VERSION_NUMBER); + strncpy(nvramData.szBoardId, szBoardId, NVRAM_BOARD_ID_STRING_LEN); + nvramData.ulNumMacAddrs = htonl(numOfMac); + memcpy(nvramData.ucaBaseMacAddr, ucaBaseMac, NVRAM_MAC_ADDRESS_LEN); + nvramData.ulCheckSum = 0; + crc = getCrc32((char *)&nvramData, (UINT32) sizeof(NVRAM_DATA), crc); + nvramData.ulCheckSum = htonl(crc); + + memcpy(pszOut + NVRAM_DATA_OFFSET, (char *)&nvramData, sizeof(NVRAM_DATA)); + + return nLen; + +} /*fillInOutputBuffer */ + +/*************************************************************************** + * Function Name: createImage + * Description : Creates the flash image file. + * Returns : None. + ***************************************************************************/ +void createImage(int numOfMac, unsigned char *ucaBaseMac, + int sdramType, char *szBoardId, char *inputFile, + char *outputFile) +{ + FILE *hInput = NULL; + struct stat StatBuf; + char *pszIn = NULL, *pszOut = NULL; + unsigned long ulOutputSize = 4 * 1024 * 1024; + int cfesize; + + if (stat(inputFile, &StatBuf ) == 0 && (hInput = fopen(inputFile, "rb" )) == NULL) + { + printf( "nvramembed: Error opening input file %s.\n\n", inputFile); + return; + } + + cfesize = StatBuf.st_size; + printf("\tCFE size is %d\n", cfesize); + + /* Allocate buffers to read the entire input file and to write the + * entire flash. + */ + pszIn = (char *) malloc(StatBuf.st_size); + pszOut = (char *) malloc(ulOutputSize); + if (!pszIn || !pszOut) + { + printf( "Memory allocation error, in=0x%8.8lx, out=0x%8.8lx.\n\n", + (unsigned long) pszIn, (unsigned long) pszOut ); + fclose( hInput ); + return; + } + + /* Read the input file into memory. */ + if (fread( pszIn, sizeof(char), StatBuf.st_size, hInput ) == StatBuf.st_size) + { + FILE *hOutput; + int nLen; + + /* Fill in the output buffer with the flash image. */ + memset( (unsigned char *) pszOut, 0xff, ulOutputSize ); + if ((nLen = fillInOutputBuffer(cfesize, numOfMac, ucaBaseMac, + sdramType, szBoardId, ulOutputSize, pszIn, pszOut)) > 0) + { + /* Open output file. */ + if ((hOutput = fopen(outputFile, "w+" )) != NULL) + { + /* Write the output buffer to the output file. */ + if (fwrite(pszOut, sizeof(char), nLen, hOutput) == nLen) + printf( "\t%s CFE with embedded NVRAM\n\tsuccessfully created.\n\n", outputFile); + else + printf( "nvramembed: Error writing to output file.\n\n" ); + fclose( hOutput ); + } + else + printf ("nvramembed: Error opening output file %s.\n\n", outputFile); + } + } + + if( pszIn ) + free( pszIn ); + + if( pszOut ) + free( pszOut ); + + fclose( hInput ); + +} /* createImage */ + + + +/************************************************************* + * Function Name: main + * Description : Program entry point that parses command line parameters + * and calls a function to create the image. + * Returns : 0 + ***************************************************************************/ +int main (int argc, char **argv) +{ + int optc; + int option_index = 0; + int h = 0, v = 0, lose = 0; + char inputFile[256], outputFile[256], macString[256]; + char szBoardId[BP_BOARD_ID_LEN] = ""; + int numOfMac = 0, sdramSize = 0, enetExtPhy = 0; + unsigned long sdramType = 0; + unsigned char ucaBaseMacAddr[NVRAM_MAC_ADDRESS_LEN]; + + /* Parse command line options. */ + progname = argv[0]; + inputFile[0] = outputFile[0] = macString[0] = '\0'; + + while ((optc = getopt_long (argc, argv, "hvp:n:m:b:i:o:", longopts, &option_index)) != EOF) + { + switch (optc) + { + case 'v': + v = 1; + break; + case 'h': + h = 1; + break; + case 'n': + numOfMac = strtoul(optarg, NULL, 10); + break; + case 'm': + strcpy(macString, optarg); + break; + case 'i': + strcpy(inputFile, optarg); + break; + case 'o': + strcpy(outputFile, optarg); + break; + case 'b': + strcpy(szBoardId, optarg); + break; + default: + lose = 1; + break; + } + } + if (v) + { + /* Print version number. */ + fprintf (stderr, "%s\n", version); + if (!h) + exit(0); + } + + if (h) + { + /* Print help info and exit. */ + fprintf(stderr, "%s\n", version); + fprintf(stderr, usage, progname); + exit(0); + } + + if (lose || optind < argc || argc < 2 || + inputFile[0] == '\0' || outputFile[0] == '\0' || + (parsehwaddr(macString, ucaBaseMacAddr) == -1)) + { + fprintf (stderr, usage, progname); + exit(1); + } + + if (szBoardId[0] != '\0') + { + BpSetBoardId( szBoardId ); + if( BpGetSdramSize( (unsigned long *) &sdramType ) == BP_SUCCESS ) + { + switch (sdramType) + { + case BP_MEMORY_8MB_1_CHIP: + sdramSize = 8; + break; + case BP_MEMORY_16MB_1_CHIP: + case BP_MEMORY_16MB_2_CHIP: + sdramSize = 16; + break; + case BP_MEMORY_32MB_1_CHIP: + case BP_MEMORY_32MB_2_CHIP: + sdramSize = 32; + break; + case BP_MEMORY_64MB_2_CHIP: + sdramSize = 64; + break; + } + } + } + + printf("nvramembed: Embed the NVRAM with the following inputs:\n" ); + printf("\tBoard id : %s\n", szBoardId); + printf("\tNumber of Mac Addresses : %d\n", numOfMac); + printf("\tBase Mac Address: : %s\n", macString); + printf("\tMemory size : %dMB\n", sdramSize); + printf("\tInput File Name : %s\n", inputFile); + printf("\tOutput File Name : %s\n\n", outputFile); + + createImage(numOfMac, ucaBaseMacAddr, sdramType, szBoardId, + inputFile, outputFile); + + return(0); + +} /* main */ + diff --git a/hostTools/scripts/Menuconfig b/hostTools/scripts/Menuconfig new file mode 100644 index 0000000..ff49bc5 --- /dev/null +++ b/hostTools/scripts/Menuconfig @@ -0,0 +1,1601 @@ +#! /bin/sh +# +# This script is used to configure the linux kernel. +# +# It was inspired by a desire to not have to hit 9 million times +# or startup the X server just to change a single kernel parameter. +# +# This script attempts to parse the configuration files, which are +# scattered throughout the kernel source tree, and creates a temporary +# set of mini scripts which are in turn used to create nested menus and +# radiolists. +# +# It uses a very modified/mutilated version of the "dialog" utility +# written by Savio Lam (lam836@cs.cuhk.hk). Savio is not responsible +# for this script or the version of dialog used by this script. +# Please do not contact him with questions. The official version of +# dialog is available at sunsite.unc.edu or a sunsite mirror. +# +# Portions of this script were borrowed from the original Configure +# script. +# +# William Roadcap was the original author of Menuconfig. +# Michael Elizabeth Chastain (mec@shout.net) is the current maintainer. +# +# 070497 Bernhard Kaindl (bkaindl@netway.at) - get default values for +# new bool, tristate and dep_tristate parameters from the defconfig file. +# new configuration parameters are marked with '(NEW)' as in make config. +# +# 180697 Bernhard Kaindl (bkaindl@netway.at) - added the needed support +# for string options. They are handled like the int and hex options. +# +# 081297 Pavel Machek (pavel@atrey.karlin.mff.cuni.cz) - better error +# handling +# +# 131197 Michael Chastain (mec@shout.net) - output all lines for a +# choice list, not just the selected one. This makes the output +# the same as Configure output, which is important for smart config +# dependencies. +# +# 101297 Michael Chastain (mec@shout.net) - remove sound driver cruft. +# +# 221297 Michael Chastain (mec@shout.net) - make define_bool actually +# define its arguments so that later tests on them work right. +# +# 160198 Michael Chastain (mec@shout.net) - fix bug with 'c' command +# (complement existing value) when used on virgin uninitialized variables. +# +# 090398 Axel Boldt (boldt@math.ucsb.edu) - allow for empty lines in help +# texts. +# +# 12 Dec 1998, Michael Elizabeth Chastain (mec@shout.net) +# Remove a /tmp security hole in get_def (also makes it faster). +# Give uninitialized variables canonical values rather than null value. +# Change a lot of places to call set_x_info uniformly. +# Take out message about preparing version (old sound driver cruft). +# +# 13 Dec 1998, Riley H Williams +# When an error occurs, actually display the error message as well as +# our comments thereon. +# +# 31 Dec 1998, Michael Elizabeth Chastain (mec@shout.net) +# Fix mod_bool to honor $CONFIG_MODULES. +# Fix dep_tristate to call define_bool when dependency is "n". +# +# 02 January 1999, Michael Elizabeth Chastain (mec@shout.net) +# Blow away lxdialog.scrltmp on entry to activate_menu. This protects +# against people who use commands like ' ' to select menus. +# +# 24 January 1999, Michael Elizabeth Chastain, +# - Improve the exit message (Jeff Ronne). +# +# 06 July 1999, Andrzej M. Krzysztofowicz, +# - Support for multiple conditions in dep_tristate(). +# - Implemented new functions: define_tristate(), define_int(), define_hex(), +# define_string(), dep_bool(). +# + + +# +# Change this to TRUE if you prefer all kernel options listed +# in a single menu rather than the standard menu hierarchy. +# +single_menu_mode= + +# +# Make sure we're really running bash. +# +[ -z "$BASH" ] && { echo "Menuconfig requires bash" 1>&2; exit 1; } + +# +# Cache function definitions, turn off posix compliance +# +set -h +o posix + + + +# Given a configuration variable, set the global variable $x to its value, +# and the global variable $info to the string " (NEW)" if this is a new +# variable. +# +# This function looks for: (1) the current value, or (2) the default value +# from the arch-dependent defconfig file, or (3) a default passed by the caller. + +function set_x_info () { + eval x=\$$1 + if [ -z "$x" ]; then + eval `sed -n -e 's/# \(.*\) is not set.*/\1=n/' -e "/^$1=/p" arch/$ARCH/defconfig` + eval x=\${$1:-"$2"} + eval $1=$x + eval INFO_$1="' (NEW)'" + fi + eval info="\$INFO_$1" +} + +# +# Load the functions used by the config.in files. +# +# I do this because these functions must be redefined depending +# on whether they are being called for interactive use or for +# saving a configuration to a file. +# +# Thank the heavens bash supports nesting function definitions. +# +load_functions () { + +# +# Additional comments +# +function comment () { + comment_ctr=$[ comment_ctr + 1 ] + echo -ne "': $comment_ctr' '--- $1' " >>MCmenu +} + +# +# Define a boolean to a specific value. +# +function define_bool () { + eval $1=$2 +} + +function define_tristate () { + eval $1=$2 +} + +function define_tristate2 () { + eval $1=$2 +} +function define_quad () { + eval $1=$2 +} + +function define_hex () { + eval $1=$2 +} + +function define_int () { + eval $1=$2 +} + +function define_string () { + eval $1="$2" +} + +# +# Create a boolean (Yes/No) function for our current menu +# which calls our local bool function. +# +function bool () { + set_x_info "$2" "n" + + case $x in + y|m) flag="*" ;; + n) flag=" " ;; + esac + + echo -ne "'$2' '[$flag] $1$info' " >>MCmenu + + echo -e "function $2 () { l_bool '$2' \"\$1\" ;}\n" >>MCradiolists +} + +# +# Create a tristate (Yes/No/Module) radiolist function +# which calls our local tristate function. +# +# Collapses to a boolean (Yes/No) if module support is disabled. +# +function tristate () { + if [ "$CONFIG_MODULES" != "y" ] + then + bool "$1" "$2" + else + set_x_info "$2" "n" + + case $x in + y) flag="*" ;; + m) flag="M" ;; + *) flag=" " ;; + esac + + echo -ne "'$2' '<$flag> $1$info' " >>MCmenu + + echo -e " + function $2 () { l_tristate '$2' \"\$1\" ;}" >>MCradiolists + fi +} + +function tristate2 () { + + set_x_info "$2" "n" + + case $x in + static) flag="static" ;; + dynamic) flag="dynamic" ;; + *) flag=" " ;; + esac + + echo -ne "'$2' '<$flag> $1$info' " >>MCmenu + + echo -e " + function $2 () { l_tristate2 '$2' \"\$1\" ;}" >>MCradiolists + +} + +function quad () { + + set_x_info "$2" "n" + + case $x in + static) flag="static" ;; + dynamic) flag="dynamic" ;; + debug) flag="debug" ;; + *) flag=" " ;; + esac + + echo -ne "'$2' '<$flag> $1$info' " >>MCmenu + + echo -e " + function $2 () { l_quad '$2' \"\$1\" ;}" >>MCradiolists + +} + +# +# Create a tristate radiolist function which is dependent on +# another kernel configuration option. +# +# Quote from the original configure script: +# +# If the option we depend upon is a module, +# then the only allowable options are M or N. If Y, then +# this is a normal tristate. This is used in cases where modules +# are nested, and one module requires the presence of something +# else in the kernel. +# +function dep_tristate () { + ques="$1" + var="$2" + dep=y + shift 2 + while [ $# -gt 0 ]; do + if [ "$1" = y ]; then + shift + elif [ "$1" = m ]; then + dep=m + shift + else + dep=n + shift $# + fi + done + if [ "$dep" = y ]; then + tristate "$ques" "$var" + elif [ "$dep" = m ]; then + mod_bool "$ques" "$var" + else + define_tristate "$var" n + fi +} + +# +# Same as above, but now only Y and N are allowed as dependency +# (i.e. third and next arguments). +# +function dep_bool () { + ques="$1" + var="$2" + dep=y + shift 2 + while [ $# -gt 0 ]; do + if [ "$1" = y ]; then + shift + else + dep=n + shift $# + fi + done + if [ "$dep" = y ]; then + bool "$ques" "$var" + else + define_bool "$var" n + fi +} + +function dep_mbool () { + ques="$1" + var="$2" + dep=y + shift 2 + while [ $# -gt 0 ]; do + if [ "$1" = y -o "$1" = m ]; then + shift + else + dep=n + shift $# + fi + done + if [ "$dep" = y ]; then + bool "$ques" "$var" + else + define_bool "$var" n + fi +} + +# +# Add a menu item which will call our local int function. +# +function int () { + set_x_info "$2" "$3" + + echo -ne "'$2' '($x) $1$info' " >>MCmenu + + echo -e "function $2 () { l_int '$1' '$2' '$3' '$x' ;}" >>MCradiolists +} + +# +# Add a menu item which will call our local hex function. +# +function hex () { + set_x_info "$2" "$3" + x=${x##*[x,X]} + + echo -ne "'$2' '($x) $1$info' " >>MCmenu + + echo -e "function $2 () { l_hex '$1' '$2' '$3' '$x' ;}" >>MCradiolists +} + +# +# Add a menu item which will call our local string function. +# +function string () { + set_x_info "$2" "$3" + + echo -ne "'$2' ' $1: \"$x\"$info' " >>MCmenu + + echo -e "function $2 () { l_string '$1' '$2' '$3' '$x' ;}" >>MCradiolists +} + +# +# Add a menu item which will call our local One-of-Many choice list. +# +function choice () { + # + # Need to remember params cause they're gonna get reset. + # + title=$1 + choices=$2 + default=$3 + current= + + # + # Find out if one of the choices is already set. + # If it's not then make it the default. + # + set -- $choices + firstchoice=$2 + + while [ -n "$2" ] + do + if eval [ "_\$$2" = "_y" ] + then + current=$1 + break + fi + shift ; shift + done + + : ${current:=$default} + + echo -ne "'$firstchoice' '($current) $title' " >>MCmenu + + echo -e " + function $firstchoice () \ + { l_choice '$title' \"$choices\" \"$current\" ;}" >>MCradiolists +} + +} # END load_functions() + + + + + +# +# Extract available help for an option from Configure.help +# and send it to standard output. +# +# Most of this function was borrowed from the original kernel +# Configure script. +# +function extract_help () { + if [ -f Documentation/Configure.help ] + then + #first escape regexp special characters in the argument: + var=$(echo "$1"|sed 's/[][\/.^$*]/\\&/g') + #now pick out the right help text: + text=$(sed -n "/^$var[ ]*\$/,\${ + /^$var[ ]*\$/c\\ +${var}:\\ + + /^#/b + /^[^ ]/q + s/^ // + /]*\\)>/s//\\1/g + p + }" Documentation/Configure.help) + + if [ -z "$text" ] + then + echo "There is no help available for this kernel option." + return 1 + else + echo "$text" + fi + else + echo "There is no help available for this kernel option." + return 1 + fi +} + +# +# Activate a help dialog. +# +function help () { + if extract_help $1 >help.out + then + $DIALOG --backtitle "$backtitle" --title "$2"\ + --textbox help.out $ROWS $COLS + else + $DIALOG --backtitle "$backtitle" \ + --textbox help.out $ROWS $COLS + fi + rm -f help.out +} + +# +# Show the README file. +# +function show_readme () { + $DIALOG --backtitle "$backtitle" \ + --textbox scripts/README.Menuconfig $ROWS $COLS +} + +# +# Begin building the dialog menu command and Initialize the +# Radiolist function file. +# +function menu_name () { + echo -ne "$DIALOG --title '$1'\ + --backtitle '$backtitle' \ + --menu '$menu_instructions' \ + $ROWS $COLS $((ROWS-10)) \ + '$default' " >MCmenu + >MCradiolists +} + +# +# Add a submenu option to the menu currently under construction. +# +function submenu () { + echo -ne "'activate_menu $2' '$1 --->' " >>MCmenu +} + +# +# Handle a boolean (Yes/No) option. +# +function l_bool () { + if [ -n "$2" ] + then + case "$2" in + y|m) eval $1=y ;; + c) eval x=\$$1 + case $x in + y) eval $1=n ;; + n) eval $1=y ;; + *) eval $1=y ;; + esac ;; + *) eval $1=n ;; + esac + else + echo -ne "\007" + fi +} + +# +# Same as bool() except options are (Module/No) +# +function mod_bool () { + if [ "$CONFIG_MODULES" != "y" ]; then + define_bool "$2" "n" + else + set_x_info "$2" "n" + + case $x in + y|m) flag='M' ;; + *) flag=' ' ;; + esac + + echo -ne "'$2' '<$flag> $1$info' " >>MCmenu + + echo -e "function $2 () { l_mod_bool '$2' \"\$1\" ;}" >>MCradiolists + fi +} + +# +# Same as l_bool() except options are (Module/No) +# +function l_mod_bool() { + if [ -n "$2" ] + then + case "$2" in + y) echo -en "\007" + ${DIALOG} --backtitle "$backtitle" \ + --infobox "\ +This feature depends on another which has been configured as a module. \ +As a result, this feature will be built as a module." 4 70 + sleep 5 + eval $1=m ;; + m) eval $1=m ;; + c) eval x=\$$1 + case $x in + m) eval $1=n ;; + n) eval $1=m ;; + *) eval $1=m ;; + esac ;; + *) eval $1=n ;; + esac + else + echo -ne "\007" + fi +} + +# +# Handle a tristate (Yes/No/Module) option. +# +function l_tristate () { + if [ -n "$2" ] + then + eval x=\$$1 + + case "$2" in + y) eval $1=y ;; + m) eval $1=m ;; + c) eval x=\$$1 + case $x in + y) eval $1=n ;; + n) eval $1=m ;; + m) eval $1=y ;; + *) eval $1=y ;; + esac ;; + *) eval $1=n ;; + esac + else + echo -ne "\007" + fi +} + +function l_tristate2 () { + if [ -n "$2" ] + then + eval x=\$$1 + + case "$2" in + static) eval $1=static ;; + dynamic) eval $1=dynamic ;; + c) eval x=\$$1 + case $x in + static) eval $1=n ;; + n) eval $1=dynamic ;; + dynamic) eval $1=static ;; + *) eval $1=static ;; + esac ;; + *) eval $1=n ;; + esac + else + echo -ne "\007" + fi +} + +function l_quad () { + if [ -n "$2" ] + then + eval x=\$$1 + + case "$2" in + static) eval $1=static ;; + dynamic) eval $1=dynamic ;; + debug) eval $1=debug ;; + c) eval x=\$$1 + case $x in + static) eval $1=n ;; + n) eval $1=dynamic ;; + dynamic) eval $1=debug ;; + debug) eval $1=static ;; + *) eval $1=static ;; + esac ;; + *) eval $1=n ;; + esac + else + echo -ne "\007" + fi +} + +# +# Create a dialog for entering an integer into a kernel option. +# +function l_int () { + while true + do + if $DIALOG --title "$1" \ + --backtitle "$backtitle" \ + --inputbox "$inputbox_instructions_int" \ + 10 75 "$4" 2>MCdialog.out + then + answer="`cat MCdialog.out`" + answer="${answer:-$3}" + + # Semantics of + and ? in GNU expr changed, so + # we avoid them: + if expr "$answer" : '0$' '|' "$answer" : '[1-9][0-9]*$' '|' "$answer" : '-[1-9][0-9]*$' >/dev/null + then + eval $2="$answer" + else + eval $2="$3" + echo -en "\007" + ${DIALOG} --backtitle "$backtitle" \ + --infobox "You have made an invalid entry." 3 43 + sleep 2 + fi + + break + fi + + help "$2" "$1" + done +} + +# +# Create a dialog for entering a hexadecimal into a kernel option. +# +function l_hex () { + while true + do + if $DIALOG --title "$1" \ + --backtitle "$backtitle" \ + --inputbox "$inputbox_instructions_hex" \ + 10 75 "$4" 2>MCdialog.out + then + answer="`cat MCdialog.out`" + answer="${answer:-$3}" + answer="${answer##*[x,X]}" + + if expr "$answer" : '[0-9a-fA-F][0-9a-fA-F]*$' >/dev/null + then + eval $2="$answer" + else + eval $2="$3" + echo -en "\007" + ${DIALOG} --backtitle "$backtitle" \ + --infobox "You have made an invalid entry." 3 43 + sleep 2 + fi + + break + fi + + help "$2" "$1" + done +} + +# +# Create a dialog for entering a string into a kernel option. +# +function l_string () { + while true + do + if $DIALOG --title "$1" \ + --backtitle "$backtitle" \ + --inputbox "$inputbox_instructions_string" \ + 10 75 "$4" 2>MCdialog.out + then + answer="`cat MCdialog.out`" + answer="${answer:-$3}" + + # + # Someone may add a nice check for the entered + # string here... + # + eval $2=\"$answer\" + + break + fi + + help "$2" "$1" + done +} + + +# +# Handle a one-of-many choice list. +# +function l_choice () { + # + # Need to remember params cause they're gonna get reset. + # + title="$1" + choices="$2" + current="$3" + chosen= + + # + # Scan current value of choices and set radiolist switches. + # + list= + set -- $choices + firstchoice=$2 + while [ -n "$2" ] + do + case "$1" in + "$current"*) if [ -z "$chosen" ]; then + list="$list $2 $1 ON " + chosen=1 + else + list="$list $2 $1 OFF " + fi ;; + *) list="$list $2 $1 OFF " ;; + esac + + shift ; shift + done + + while true + do + if $DIALOG --title "$title" \ + --backtitle "$backtitle" \ + --radiolist "$radiolist_instructions" \ + 15 70 6 $list 2>MCdialog.out + then + choice=`cat MCdialog.out` + break + fi + + help "$firstchoice" "$title" + done + + # + # Now set the boolean value of each option based on + # the selection made from the radiolist. + # + set -- $choices + while [ -n "$2" ] + do + if [ "$2" = "$choice" ] + then + eval $2="y" + else + eval $2="n" + fi + + shift ; shift + done +} + +# +# Call awk, and watch for error codes, etc. +# +function callawk () { +awk "$1" || echo "Awk died with error code $?. Giving up." || exit 1 +} + +# +# A faster awk based recursive parser. (I hope) +# +function parser1 () { +callawk ' +BEGIN { + menu_no = 0 + comment_is_option = 0 + parser("'$CONFIG_IN'","MCmenu0") +} + +function parser(ifile,menu) { + + while (getline >menu + + newmenu = sprintf("MCmenu%d", menu_no); + printf( "function MCmenu%s () {\n"\ + "default=$1\n"\ + "menu_name %s\n",\ + menu_no, $0) >newmenu + + parser(ifile, newmenu) + } + else if ($0 ~ /^#|\$MAKE|mainmenu_name/) { + printf("") >>menu + } + else if ($1 ~ "endmenu") { + printf("}\n") >>menu + return + } + else if ($1 == "source") { + parser($2,menu) + } + else { + print >>menu + } + } +}' +} + +# +# Secondary parser for single menu mode. +# +function parser2 () { +callawk ' +BEGIN { + parser("'$CONFIG_IN'","MCmenu0") +} + +function parser(ifile,menu) { + + while (getline >menu + } + else if ($1 ~ /mainmenu_option|endmenu/) { + printf("") >>menu + } + else if ($1 == "source") { + parser($2,menu) + } + else { + print >>menu + } + } +}' +} + +# +# Parse all the config.in files into mini scripts. +# +function parse_config_files () { + rm -f MCmenu* + + echo "function MCmenu0 () {" >MCmenu0 + echo 'default=$1' >>MCmenu0 + echo "menu_name 'Main Menu'" >>MCmenu0 + + if [ "_$single_menu_mode" = "_TRUE" ] + then + parser2 + else + parser1 + fi + + echo "comment ''" >>MCmenu0 + echo "g_alt_config" >>MCmenu0 + echo "s_alt_config" >>MCmenu0 + + echo "}" >>MCmenu0 + + # + # These mini scripts must be sourced into the current + # environment in order for all of this to work. Leaving + # them on the disk as executables screws up the recursion + # in activate_menu(), among other things. Once they are + # sourced we can discard them. + # + for i in MCmenu* + do + echo -n "." + source ./$i + done + rm -f MCmenu* +} + +# +# This is the menu tree's bootstrap. +# +# Executes the parsed menus on demand and creates a set of functions, +# one per configuration option. These functions will in turn execute +# dialog commands or recursively call other menus. +# +function activate_menu () { + rm -f lxdialog.scrltmp + while true + do + comment_ctr=0 #So comment lines get unique tags + + $1 "$default" 2> MCerror #Create the lxdialog menu & functions + + if [ "$?" != "0" ] + then + clear + cat < /' MCerror + cat <. You may also +send a problem report to . + +Please indicate the kernel version you are trying to configure and +which menu you were trying to enter when this error occurred. + +EOM + cleanup + exit 1 + fi + rm -f MCerror + + . ./MCradiolists #Source the menu's functions + + . ./MCmenu 2>MCdialog.out #Activate the lxdialog menu + ret=$? + + read selection "*|*"alt_config"*) + show_readme ;; + *) + eval help $selection ;; + esac + ;; + 255|1) + break + ;; + 139) + stty sane + clear + cat < or post a message to + for additional assistance. + +EOM + cleanup + exit 139 + ;; + esac + done +} + +# +# Create a menu item to load an alternate configuration file. +# +g_alt_config () { + echo -n "get_alt_config 'Load software build profile' "\ + >>MCmenu +} + +# +# Get alternate config file name and load the +# configuration from it. +# +get_alt_config () { + set -f ## Switch file expansion OFF + + while true + do + ALT_CONFIG="${ALT_CONFIG:-$DEFAULTS}" + + $DIALOG --backtitle "$backtitle" \ + --inputbox "\ +Enter the name of the profile you wish to load. Leave blank to abort."\ + 11 55 "$ALT_CONFIG" 2>MCdialog.out + + if [ "$?" = "0" ] + then + ALT_CONFIG=`cat MCdialog.out` + + [ "_" = "_$ALT_CONFIG" ] && break + + if eval [ -r "$TARGETS_DIR/$ALT_CONFIG/$ALT_CONFIG" ] + then + eval load_config_file "$TARGETS_DIR/$ALT_CONFIG/$ALT_CONFIG" + break + else + echo -ne "\007" + $DIALOG --backtitle "$backtitle" \ + --infobox "File does not exist!" 3 38 + sleep 2 + fi + else + cat <help.out + +Profiles are saved in a directory with the same name in "targets" directory. + +If you are uncertain, leave this blank. +EOM + $DIALOG --backtitle "$backtitle"\ + --title "Load Profile"\ + --textbox help.out $ROWS $COLS + fi + done + + set +f ## Switch file expansion ON + rm -f help.out MCdialog.out +} + +# +# Create a menu item to store an alternate config file. +# +s_alt_config () { + echo -n "save_alt_config 'Save software build profile as ...' "\ + >>MCmenu +} + +# +# Get an alternate config file name and save the current +# configuration to it. +# +save_alt_config () { + set -f ## Switch file expansion OFF + + while true + do + $DIALOG --backtitle "$backtitle" \ + --inputbox "\ +Enter a profile name to save. Leave blank to abort."\ + 10 55 "$ALT_CONFIG" 2>MCdialog.out + + if [ "$?" = "0" ] + then + ALT_CONFIG=`cat MCdialog.out` + + [ "_" = "_$ALT_CONFIG" ] && break + + if eval 'mkdir -p $TARGETS_DIR/$ALT_CONFIG;touch $TARGETS_DIR/$ALT_CONFIG/$ALT_CONFIG' 2>/dev/null + then + eval save_configuration $TARGETS_DIR/$ALT_CONFIG/$ALT_CONFIG + load_functions ## RELOAD + break + else + echo -ne "\007" + $DIALOG --backtitle "$backtitle" \ + --infobox "Can't create file! Probably a nonexistent directory." 3 60 + sleep 2 + fi + else + cat <help.out +Profiles are saved in a directory with the same name in "targets" directory. + +If you are uncertain, leave this blank. +EOM + $DIALOG --backtitle "$backtitle"\ + --title "Save profile"\ + --textbox help.out $ROWS $COLS + fi + done + + set +f ## Switch file expansion ON + rm -f help.out MCdialog.out +} + +# +# Load config options from a file. +# Converts all "# OPTION is not set" lines to "OPTION=n" lines +# +function load_config_file () { + awk ' + /# .* is not set.*/ { printf("%s=n\n", $2) } + ! /# .* is not set.*/ { print } + ' $1 >.tmpconfig + + source ./.tmpconfig + rm -f .tmpconfig +} + +# +# Just what it says. +# +save_configuration () { + echo + echo -n "Saving your software build profile." + + # + # Now, let's redefine the configuration functions for final + # output to the config files. + # + # Nested function definitions, YIPEE! + # + function bool () { + set_x_info "$2" "n" + eval define_bool "$2" "$x" + } + + function tristate () { + set_x_info "$2" "n" + eval define_tristate "$2" "$x" + } + + function tristate2 () { + set_x_info "$2" "n" + eval define_tristate2 "$2" "$x" + } + + function quad () { + set_x_info "$2" "n" + eval define_quad "$2" "$x" + } + + function dep_tristate () { + set_x_info "$2" "n" + var="$2" + shift 2 + while [ $# -gt 0 ]; do + if [ "$1" = y ]; then + shift + elif [ "$1" = m -a "$x" != n ]; then + x=m; shift + else + x=n; shift $# + fi + done + define_tristate "$var" "$x" + } + + function dep_bool () { + set_x_info "$2" "n" + var="$2" + shift 2 + while [ $# -gt 0 ]; do + if [ "$1" = y ]; then + shift + else + x=n; shift $# + fi + done + define_bool "$var" "$x" + } + + function dep_mbool () { + set_x_info "$2" "n" + var="$2" + shift 2 + while [ $# -gt 0 ]; do + if [ "$1" = y -o "$1" = m ]; then + shift + else + x=n; shift $# + fi + done + define_bool "$var" "$x" + } + + function int () { + set_x_info "$2" "$3" + echo "$2=$x" >>$CONFIG + echo "#define $2 ($x)" >>$CONFIG_H + } + + function hex () { + set_x_info "$2" "$3" + echo "$2=$x" >>$CONFIG + echo "#define $2 0x${x##*[x,X]}" >>$CONFIG_H + } + + function string () { + set_x_info "$2" "$3" + echo "$2=\"$x\"" >>$CONFIG + echo "#define $2 \"$x\"" >>$CONFIG_H + } + + function define_hex () { + eval $1="$2" + echo "$1=$2" >>$CONFIG + echo "#define $1 0x${2##*[x,X]}" >>$CONFIG_H + } + + function define_int () { + eval $1="$2" + echo "$1=$2" >>$CONFIG + echo "#define $1 ($2)" >>$CONFIG_H + } + + function define_string () { + eval $1="$2" + #echo "$1=\"$2\"" >>$CONFIG + echo "$1=$2" >>$CONFIG + echo "#define $1 \"$2\"" >>$CONFIG_H + } + + function define_bool () { + define_tristate "$1" "$2" + } + + function define_tristate () { + eval $1="$2" + + case "$2" in + y) + echo "$1=y" >>$CONFIG + echo "#define $1 1" >>$CONFIG_H + ;; + + m) + if [ "$CONFIG_MODULES" = "y" ] + then + echo "$1=m" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + echo "#define $1_MODULE 1" >>$CONFIG_H + else + echo "$1=y" >>$CONFIG + echo "#define $1 1" >>$CONFIG_H + fi + ;; + + n) + echo "# $1 is not set" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + ;; + esac + } + + function define_tristate2 () { + eval $1="$2" + + case "$2" in + static) + echo "$1=static" >>$CONFIG + echo "#define $1 1" >>$CONFIG_H + ;; + + dynamic) + echo "$1=dynamic" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + ;; + + n) + echo "# $1 is not set" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + ;; + esac + } + + function define_quad () { + eval $1="$2" + + case "$2" in + static) + echo "$1=static" >>$CONFIG + echo "#define $1 1" >>$CONFIG_H + ;; + + dynamic) + echo "$1=dynamic" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + ;; + + debug) + echo "$1=debug" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + ;; + + n) + echo "# $1 is not set" >>$CONFIG + echo "#undef $1" >>$CONFIG_H + ;; + esac + } + + function choice () { + # + # Find the first choice that's already set to 'y' + # + choices="$2" + default="$3" + current= + chosen= + + set -- $choices + while [ -n "$2" ] + do + if eval [ "_\$$2" = "_y" ] + then + current=$1 + break + fi + shift ; shift + done + + # + # Use the default if none were set. + # + : ${current:=$default} + + # + # Output all choices (to be compatible with other configs). + # + set -- $choices + while [ -n "$2" ] + do + case "$1" in + "$current"*) if [ -z "$chosen" ]; then + define_bool "$2" "y" + chosen=1 + else + define_bool "$2" "n" + fi ;; + *) define_bool "$2" "n" ;; + esac + shift ; shift + done + } + + function mainmenu_name () { + : + } + + function mainmenu_option () { + comment_is_option=TRUE + } + + function endmenu () { + : + } + + function comment () { + if [ "$comment_is_option" ] + then + comment_is_option= + echo >>$CONFIG + echo "#" >>$CONFIG + echo "# $1" >>$CONFIG + echo "#" >>$CONFIG + + echo >>$CONFIG_H + echo "/*" >>$CONFIG_H + echo " * $1" >>$CONFIG_H + echo " */" >>$CONFIG_H + fi + } + + echo -n "." + + DEF_CONFIG="${1:-.config}" + DEF_CONFIG_H="include/linux/autoconf.h" + + CONFIG=.tmpconfig + CONFIG_H=.tmpconfig.h + + echo "#" >$CONFIG + echo "# Automatically generated by make menuconfig: don't edit" >>$CONFIG + echo "#" >>$CONFIG + + echo "/*" >$CONFIG_H + echo " * Automatically generated by make menuconfig: don't edit" >>$CONFIG_H + echo " */" >>$CONFIG_H + echo "#define AUTOCONF_INCLUDED" >> $CONFIG_H + + echo -n "." + if . $CONFIG_IN >>.menuconfig.log 2>&1 + then + if [ "$DEF_CONFIG" = ".config" ] + then + mv $CONFIG_H $DEF_CONFIG_H + fi + + if [ -f "$DEF_CONFIG" ] + then + rm -f ${DEF_CONFIG}.old + mv $DEF_CONFIG ${DEF_CONFIG}.old + fi + + mv $CONFIG $DEF_CONFIG + + return 0 + else + return 1 + fi +} + +# +# Remove temporary files +# +cleanup () { + cleanup1 + cleanup2 +} + +cleanup1 () { + rm -f MCmenu* MCradiolists MCdialog.out help.out +} + +cleanup2 () { + rm -f .tmpconfig .tmpconfig.h +} + +set_geometry () { + # Some distributions export these with incorrect values + # which can really screw up some ncurses programs. + LINES= COLUMNS= + + ROWS=${1:-24} COLS=${2:-80} + + # Just in case the nasty rlogin bug returns. + # + [ $ROWS = 0 ] && ROWS=24 + [ $COLS = 0 ] && COLS=80 + + if [ $ROWS -lt 19 -o $COLS -lt 80 ] + then + echo -e "\n\007Your display is too small to run Menuconfig!" + echo "It must be at least 19 lines by 80 columns." + exit 1 + fi + + ROWS=$((ROWS-4)) COLS=$((COLS-5)) +} + + +set_geometry `stty size 2>/dev/null` + +menu_instructions="\ +Arrow keys navigate the menu. \ + selects submenus --->. \ +Highlighted letters are hotkeys. \ +Pressing includes, excludes, modularizes features. \ +Press to exit, for Help. \ +Legend: [*] built-in [ ] excluded module < > module capable" + +radiolist_instructions="\ +Use the arrow keys to navigate this window or \ +press the hotkey of the item you wish to select \ +followed by the . +Press for additional information about this option." + +inputbox_instructions_int="\ +Please enter a decimal value. \ +Fractions will not be accepted. \ +Use the key to move from the input field to the buttons below it." + +inputbox_instructions_hex="\ +Please enter a hexadecimal value. \ +Use the key to move from the input field to the buttons below it." + +inputbox_instructions_string="\ +Please enter a string value. \ +Use the key to move from the input field to the buttons below it." + +DIALOG="$KERNEL_DIR/scripts/lxdialog/lxdialog" + +#kernel_version="${VERSION}.${PATCHLEVEL}.${SUBLEVEL}${EXTRAVERSION}" + +#backtitle="Linux Kernel v$kernel_version Configuration" +backtitle="Broadcom Commengine xDSL Software Configuration" + +CONFIG_MODULES=y + +trap "cleanup ; exit 1" 1 2 15 + + +# +# Locate default files. +# +#CONFIG_IN=./config.in +if [ "$1" != "" ] ; then + CONFIG_IN=$1 +fi + +#DEFAULTS=arch/$ARCH/defconfig +#if [ -f .config ]; then +# DEFAULTS=.config +#fi +# +#if [ -f $DEFAULTS ] +#then +# echo "Using defaults found in" $DEFAULTS +# load_config_file $DEFAULTS +#else +# echo "No defaults found" +#fi + + +# Fresh new log. +>.menuconfig.log + +# Load the functions used by the config.in files. +echo -n "Preparing scripts: functions" +load_functions + +if [ ! -e $CONFIG_IN ] +then + echo "Your main config.in file ($CONFIG_IN) does not exist" + exit 1 +fi + +if [ ! -x $DIALOG ] +then + echo "Your lxdialog utility does not exist" + exit 1 +fi + +# +# Read config.in files and parse them into one shell function per menu. +# +echo -n ", parsing" +parse_config_files $CONFIG_IN + +echo "done." +# +# Start the ball rolling from the top. +# +activate_menu MCmenu0 + +# +# All done! +# +cleanup1 + +# +# Confirm and Save +# +if $DIALOG --backtitle "$backtitle" \ + --yesno "Save your $ALT_CONFIG software build configuration?" 5 60 +then + if [ "$ALT_CONFIG" != "" ]; then + save_configuration $TARGETS_DIR/$ALT_CONFIG/$ALT_CONFIG + else + save_alt_config + fi + echo + echo + echo "*** End of software build configuration." + echo "*** Use make PROFILE= to build your image" + echo +else + echo + echo + echo Your $ALT_CONFIG software build configuration changes were NOT saved. + echo +fi + +# Remove log if empty. +if [ ! -s .menuconfig.log ] ; then + rm -f .menuconfig.log +fi + +exit 0 diff --git a/hostTools/scripts/defconfig-bcm.template b/hostTools/scripts/defconfig-bcm.template new file mode 100644 index 0000000..3b13f78 --- /dev/null +++ b/hostTools/scripts/defconfig-bcm.template @@ -0,0 +1,1066 @@ +# +# A template defconfig file for BRCM xDSL build: don't edit +# +CONFIG_MIPS=y +# CONFIG_MIPS64 is not set +# CONFIG_64BIT is not set +CONFIG_MIPS32=y + +# +# Code maturity level options +# +CONFIG_EXPERIMENTAL=y +CONFIG_CLEAN_COMPILE=y +CONFIG_STANDALONE=y +CONFIG_BROKEN_ON_SMP=y + +# +# General setup +# +# CONFIG_SWAP is not set +CONFIG_SYSVIPC=y +# CONFIG_POSIX_MQUEUE is not set +# CONFIG_BSD_PROCESS_ACCT is not set +CONFIG_SYSCTL=y +# CONFIG_AUDIT is not set +CONFIG_LOG_BUF_SHIFT=14 +# CONFIG_HOTPLUG is not set +# CONFIG_IKCONFIG is not set +CONFIG_EMBEDDED=y +# CONFIG_KALLSYMS is not set +# CONFIG_KALLSYMS_EXTRA_PASS is not set +# CONFIG_FUTEX is not set +# CONFIG_EPOLL is not set +CONFIG_IOSCHED_NOOP=y +# CONFIG_IOSCHED_AS is not set +# CONFIG_IOSCHED_DEADLINE is not set +# CONFIG_IOSCHED_CFQ is not set +CONFIG_CC_OPTIMIZE_FOR_SIZE=y + +# +# Loadable module support +# +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_OBSOLETE_MODPARM=y +# CONFIG_MODVERSIONS is not set +CONFIG_KMOD=y + +# +# Machine selection +# +CONFIG_MIPS_BRCM=y +# CONFIG_BCM96338 is not set +# CONFIG_BCM96348 is not set +# CONFIG_BCM96358 is not set + +# CONFIG_BCM_BOARD is not set +# CONFIG_BCM_SERIAL is not set +# CONFIG_BCM_ENET is not set +# CONFIG_BCM_ATMAPI is not set +# CONFIG_BCM_ATMTEST is not set +# CONFIG_BCM_USB is not set +# CONFIG_BCM_ADSL is not set +# CONFIG_BCM_WLAN is not set +# CONFIG_BCM_PCI is not set +# CONFIG_BCM_ENDPOINT is not set +# CONFIG_BCM_PROCFS is not set +# CONFIG_BCM_VDSL is not set +# CONFIG_BCM_SECURITY is not set +# CONFIG_BCM_HPNA is not set +# CONFIG_BCM_BCMPROF is not set + +# CONFIG_BCM96338_BOARD_IMPL=1 +# CONFIG_BCM96338_SERIAL_IMPL=1 +# CONFIG_BCM96338_ENET_IMPL=2 +# CONFIG_BCM96338_ATMAPI_IMPL=1 +# CONFIG_BCM96338_BLAA_IMPL=1 +# CONFIG_BCM96338_ATMTEST_IMPL=1 +# CONFIG_BCM96338_USB_IMPL=2 +# CONFIG_BCM96338_ADSL_IMPL=1 +# CONFIG_BCM96338_WLAN_IMPL=1 +# CONFIG_BCM96338_ENDPOINT_IMPL=1 +# CONFIG_BCM96338_PROCFS_IMPL=1 +# CONFIG_BCM96338_VDSL_IMPL=1 +# CONFIG_BCM96338_SECURITY_IMPL=1 +# CONFIG_BCM96338_HPNA_IMPL=0 +# CONFIG_BCM96338_BCMPROF_IMPL=1 + +# CONFIG_BCM96348_BOARD_IMPL=1 +# CONFIG_BCM96348_SERIAL_IMPL=1 +# CONFIG_BCM96348_ENET_IMPL=2 +# CONFIG_BCM96348_ATMAPI_IMPL=1 +# CONFIG_BCM96348_BLAA_IMPL=1 +# CONFIG_BCM96348_ATMTEST_IMPL=1 +# CONFIG_BCM96348_USB_IMPL=2 +# CONFIG_BCM96348_ADSL_IMPL=1 +# CONFIG_BCM96348_WLAN_IMPL=1 +# CONFIG_BCM96348_ENDPOINT_IMPL=1 +# CONFIG_BCM96348_PROCFS_IMPL=1 +# CONFIG_BCM96348_VDSL_IMPL=1 +# CONFIG_BCM96348_SECURITY_IMPL=1 +# CONFIG_BCM96348_HPNA_IMPL=0 +# CONFIG_BCM96348_BCMPROF_IMPL=1 + +# CONFIG_BCM96358_BOARD_IMPL=1 +# CONFIG_BCM96358_SERIAL_IMPL=1 +# CONFIG_BCM96358_ENET_IMPL=2 +# CONFIG_BCM96358_ATMAPI_IMPL=1 +# CONFIG_BCM96358_BLAA_IMPL=1 +# CONFIG_BCM96358_ATMTEST_IMPL=1 +# CONFIG_BCM96358_USB_IMPL=2 +# CONFIG_BCM96358_ADSL_IMPL=1 +# CONFIG_BCM96358_WLAN_IMPL=1 +# CONFIG_BCM96358_ENDPOINT_IMPL=1 +# CONFIG_BCM96358_PROCFS_IMPL=1 +# CONFIG_BCM96358_VDSL_IMPL=1 +# CONFIG_BCM96358_SECURITY_IMPL=1 +# CONFIG_BCM96358_HPNA_IMPL=0 +# CONFIG_BCM96358_BCMPROF_IMPL=1 + +# CONFIG_ROOTFS_SQUASHFS is not set +# CONFIG_ROOTFS_CRAMFS is not set +# CONFIG_ROOTFS_JFFS2 is not set +# CONFIG_ROOTFS_NFS is not set +# CONFIG_ROOT_FLASHFS is not set +# CONFIG_ROOT_NFS_DIR is not set + +# CONFIG_BRCM_USING_PTHREADS is not set + +# CONFIG_MACH_JAZZ is not set +# CONFIG_BAGET_MIPS is not set +# CONFIG_MACH_VR41XX is not set +# CONFIG_TOSHIBA_JMR3927 is not set +# CONFIG_MIPS_COBALT is not set +# CONFIG_MACH_DECSTATION is not set +# CONFIG_MIPS_EV64120 is not set +# CONFIG_MIPS_EV96100 is not set +# CONFIG_MIPS_IVR is not set +# CONFIG_LASAT is not set +# CONFIG_MIPS_ITE8172 is not set +# CONFIG_MIPS_ATLAS is not set +# CONFIG_MIPS_MALTA is not set +# CONFIG_MIPS_SEAD is not set +# CONFIG_MOMENCO_OCELOT is not set +# CONFIG_MOMENCO_OCELOT_G is not set +# CONFIG_MOMENCO_OCELOT_C is not set +# CONFIG_MOMENCO_JAGUAR_ATX is not set +# CONFIG_PMC_YOSEMITE is not set +# CONFIG_DDB5074 is not set +# CONFIG_DDB5476 is not set +# CONFIG_DDB5477 is not set +# CONFIG_NEC_OSPREY is not set +# CONFIG_SGI_IP22 is not set +# CONFIG_SGI_IP32 is not set +# CONFIG_SOC_AU1X00 is not set +# CONFIG_SIBYTE_SB1xxx_SOC is not set +# CONFIG_SNI_RM200_PCI is not set +# CONFIG_TOSHIBA_RBTX4927 is not set +CONFIG_RWSEM_GENERIC_SPINLOCK=y +CONFIG_HAVE_DEC_LOCK=y +CONFIG_DMA_NONCOHERENT=y +# CONFIG_CPU_LITTLE_ENDIAN is not set +CONFIG_MIPS_L1_CACHE_SHIFT=5 +# CONFIG_FB is not set + +# +# CPU selection +# +CONFIG_CPU_MIPS32=y +# CONFIG_CPU_MIPS64 is not set +# CONFIG_CPU_R3000 is not set +# CONFIG_CPU_TX39XX is not set +# CONFIG_CPU_VR41XX is not set +# CONFIG_CPU_R4300 is not set +# CONFIG_CPU_R4X00 is not set +# CONFIG_CPU_TX49XX is not set +# CONFIG_CPU_R5000 is not set +# CONFIG_CPU_R5432 is not set +# CONFIG_CPU_R6000 is not set +# CONFIG_CPU_NEVADA is not set +# CONFIG_CPU_R8000 is not set +# CONFIG_CPU_R10000 is not set +# CONFIG_CPU_RM7000 is not set +# CONFIG_CPU_RM9000 is not set +# CONFIG_CPU_SB1 is not set +CONFIG_PAGE_SIZE_4KB=y +# CONFIG_PAGE_SIZE_16KB is not set +# CONFIG_PAGE_SIZE_64KB is not set +# CONFIG_CPU_HAS_PREFETCH is not set +# CONFIG_VTAG_ICACHE is not set +# CONFIG_64BIT_PHYS_ADDR is not set +# CONFIG_CPU_ADVANCED is not set +CONFIG_CPU_HAS_LLSC=y +CONFIG_CPU_HAS_SYNC=y +# CONFIG_PREEMPT is not set +# CONFIG_DEBUG_SPINLOCK_SLEEP is not set + +# +# Bus options (PCI, PCMCIA, EISA, ISA, TC) +# +# CONFIG_PCI is not set +# CONFIG_PCI_NAMES is not set +# CONFIG_PCI_LEGACY_PROC is not set +CONFIG_MMU=y + +# +# PCI Hotplug Support +# +# CONFIG_HOTPLUG_PCI is not set +# CONFIG_HOTPLUG_PCI_FAKE is not set +# CONFIG_HOTPLUG_PCI_ACPI is not set +# CONFIG_HOTPLUG_PCI_CPCI is not set +# CONFIG_HOTPLUG_PCI_PCIE is not set +# CONFIG_HOTPLUG_PCI_SHPC is not set + +# +# PCMCIA/CardBus support +# +# CONFIG_PCMCIA is not set +# +# Executable file formats +# +CONFIG_BINFMT_ELF=y +# CONFIG_BINFMT_MISC is not set +CONFIG_TRAD_SIGNALS=y +# CONFIG_BINFMT_IRIX is not set + +# +# Device Drivers +# + +# +# Generic Driver Options +# +CONFIG_STANDALONE=y +CONFIG_PREVENT_FIRMWARE_BUILD=y +# CONFIG_FW_LOADER is not set +# CONFIG_DEBUG_DRIVER is not set + +# +# Memory Technology Devices (MTD) +# +CONFIG_MTD=y +# CONFIG_MTD_DEBUG is not set +# CONFIG_MTD_PARTITIONS is not set +# CONFIG_MTD_CONCAT is not set + +# +# User Modules And Translation Layers +# +# CONFIG_MTD_CHAR is not set +# CONFIG_MTD_BLOCK is not set +# CONFIG_MTD_BLOCK_RO is not set +# CONFIG_FTL is not set +# CONFIG_NFTL is not set +# CONFIG_INFTL is not set + +# +# RAM/ROM/Flash chip drivers +# +# CONFIG_MTD_CFI is not set +# CONFIG_MTD_JEDECPROBE is not set +# CONFIG_MTD_GEN_PROBE is not set +# CONFIG_MTD_CFI_ADV_OPTIONS is not set +# CONFIG_MTD_CFI_NOSWAP is not set +# CONFIG_MTD_CFI_BE_BYTE_SWAP is not set +# CONFIG_MTD_CFI_LE_BYTE_SWAP is not set +# CONFIG_MTD_CFI_GEOMETRY is not set +# CONFIG_MTD_CFI_B1 is not set +# CONFIG_MTD_CFI_B2 is not set +# CONFIG_MTD_CFI_B4 is not set +# CONFIG_MTD_CFI_I1 is not set +# CONFIG_MTD_CFI_I2 is not set +# CONFIG_MTD_CFI_I4 is not set +# CONFIG_MTD_CFI_INTELEXT is not set +# CONFIG_MTD_CFI_AMDSTD is not set +# CONFIG_MTD_RAM is not set +# CONFIG_MTD_ROM is not set +# CONFIG_MTD_ABSENT is not set +# CONFIG_MTD_OBSOLETE_CHIPS is not set +# CONFIG_MTD_AMDSTD is not set +# CONFIG_MTD_SHARP is not set +# CONFIG_MTD_JEDEC is not set + +# +# Mapping drivers for chip access +# +# CONFIG_MTD_PHYSMAP is not set +# CONFIG_MTD_PHYSMAP_START is not set +# CONFIG_MTD_PHYSMAP_LEN is not set +# CONFIG_MTD_PHYSMAP_BUSWIDTH is not set +# CONFIG_MTD_PB1000 is not set +# CONFIG_MTD_PB1500 is not set +# CONFIG_MTD_CSTM_MIPS_IXX is not set +# CONFIG_MTD_OCELOT is not set +# CONFIG_MTD_BCM963XX is not set +# CONFIG_MTD_COMPLEX_MAPPINGS is not set +# CONFIG_MTD_PCI is not set +# +# Self-contained MTD device drivers +# +# CONFIG_MTD_PMC551 is not set +# CONFIG_MTD_SLRAM is not set +# CONFIG_MTD_PHRAM is not set +# CONFIG_MTD_MTDRAM is not set +# CONFIG_MTD_BLKMTD is not set + +# +# Disk-On-Chip Device Drivers +# +# CONFIG_MTD_DOC2000 is not set +# CONFIG_MTD_DOC2001 is not set +# CONFIG_MTD_DOC2001PLUS is not set + +# +# NAND Flash Device Drivers +# +# CONFIG_MTD_NAND is not set + +# +# Parallel port support +# +# CONFIG_PARPORT is not set + +# +# Plug and Play support +# + +# +# Block devices +# +# CONFIG_BLK_DEV_FD is not set +# CONFIG_BLK_CPQ_DA is not set +# CONFIG_BLK_CPQ_CISS_DA is not set +# CONFIG_BLK_DEV_DAC960 is not set +# CONFIG_BLK_DEV_UMEM is not set +# CONFIG_BLK_DEV_LOOP is not set +# CONFIG_BLK_DEV_NBD is not set +# CONFIG_BLK_DEV_SX8 is not set +# CONFIG_BLK_DEV_RAM is not set +# CONFIG_LBD is not set + +# +# ATA/ATAPI/MFM/RLL support +# +# CONFIG_IDE is not set + +# +# SCSI device support +# +# CONFIG_SCSI is not set + +# +# Multi-device support (RAID and LVM) +# +# CONFIG_MD is not set + +# +# Fusion MPT device support +# + +# +# IEEE 1394 (FireWire) support +# +# CONFIG_IEEE1394 is not set + +# +# I2O device support +# +# CONFIG_I2O is not set + +# +# Networking support +# +CONFIG_NET=y + +# +# Networking options +# +CONFIG_PACKET=y +# CONFIG_PACKET_MMAP is not set +# CONFIG_NETLINK_DEV is not set +CONFIG_UNIX=y +# CONFIG_NET_KEY is not set +CONFIG_INET=y +# CONFIG_IP_MULTICAST is not set +# CONFIG_IP_ADVANCED_ROUTER is not set +# CONFIG_IP_PNP is not set +# CONFIG_IP_PNP_DHCP is not set +# CONFIG_IP_PNP_BOOTP is not set +# CONFIG_IP_PNP_RARP is not set +# CONFIG_NET_IPIP is not set +# CONFIG_NET_IPGRE is not set +# CONFIG_IP_MROUTE is not set +# CONFIG_IP_PIMSM_V1 is not set +# CONFIG_IP_PIMSM_V2 is not set +# CONFIG_ARPD is not set +# CONFIG_SYN_COOKIES is not set +# CONFIG_INET_AH is not set +# CONFIG_INET_ESP is not set +# CONFIG_INET_IPCOMP is not set +# CONFIG_XFRM_USER is not set + +# +# IP: Virtual Server Configuration +# +# CONFIG_IP_VS is not set +# CONFIG_IPV6 is not set +# CONFIG_DECNET is not set +CONFIG_BRIDGE=y +CONFIG_NETFILTER=y +# CONFIG_NETFILTER_DEBUG is not set +# CONFIG_BRIDGE_NETFILTER is not set + +# +# IP: Netfilter Configuration +# +# CONFIG_IP_NF_CONNTRACK is not set +# CONFIG_IP_NF_FTP is not set +# CONFIG_IP_NF_TFTP is not set +# CONFIG_IP_NF_TALK is not set +# CONFIG_IP_NF_AMANDA is not set +# CONFIG_IP_NF_H323 is not set +# CONFIG_IP_NF_IRC is not set +# CONFIG_IP_NF_DTX8 is not set +# CONFIG_IP_NF_PT is not set +# CONFIG_IP_NF_WM is not set +# CONFIG_IP_NF_PPTP is not set +# CONFIG_IP_NF_IPSEC is not set +# CONFIG_IP_NF_RTSP is not set +# CONFIG_IP_NF_PPTP_DEBUG is not set +# CONFIG_IP_NF_QUEUE is not set +# CONFIG_IP_NF_IPTABLES is not set +# CONFIG_IP_NF_MATCH_LIMIT is not set +# CONFIG_IP_NF_MATCH_IPRANGE is not set +# CONFIG_IP_NF_MATCH_MAC is not set +# CONFIG_IP_NF_MATCH_PKTTYPE is not set +# CONFIG_IP_NF_MATCH_MARK is not set +# CONFIG_IP_NF_MATCH_MULTIPORT is not set +# CONFIG_IP_NF_MATCH_TOS is not set +# CONFIG_IP_NF_MATCH_RECENT is not set +# CONFIG_IP_NF_MATCH_ECN is not set +# CONFIG_IP_NF_MATCH_DSCP is not set +# CONFIG_IP_NF_MATCH_AH_ESP is not set +# CONFIG_IP_NF_MATCH_LENGTH is not set +# CONFIG_IP_NF_MATCH_TTL is not set +# CONFIG_IP_NF_MATCH_TCPMSS is not set +# CONFIG_IP_NF_MATCH_HELPER is not set +# CONFIG_IP_NF_MATCH_STATE is not set +CONFIG_IP_NF_MATCH_CONNLIMIT=y +# CONFIG_IP_NF_MATCH_CONNTRACK is not set +# CONFIG_IP_NF_MATCH_UNCLEAN is not set +# CONFIG_IP_NF_MATCH_OWNER is not set +# CONFIG_IP_NF_MATCH_PHYSDEV is not set +# CONFIG_IP_NF_FILTER is not set +# CONFIG_IP_NF_TARGET_REJECT is not set +# CONFIG_IP_NF_TARGET_MIRROR is not set +# CONFIG_IP_NF_NAT is not set +# CONFIG_IP_NF_NAT_NEEDED is not set +# CONFIG_IP_NF_TARGET_MASQUERADE is not set +# CONFIG_IP_NF_TARGET_REDIRECT is not set +# CONFIG_IP_NF_TARGET_NETMAP is not set +# CONFIG_IP_NF_TARGET_SAME is not set +# CONFIG_IP_NF_NAT_LOCAL is not set +# CONFIG_IP_NF_NAT_TALK is not set +# CONFIG_IP_NF_NAT_H323 is not set +# CONFIG_IP_NF_NAT_SNMP_BASIC is not set +# CONFIG_IP_NF_NAT_IRC is not set +# CONFIG_IP_NF_NAT_FTP is not set +# CONFIG_IP_NF_NAT_TFTP is not set +# CONFIG_IP_NF_NAT_IPSEC is not set +# CONFIG_IP_NF_NAT_RTSP is not set +# CONFIG_IP_NF_MANGLE is not set +# CONFIG_IP_NF_TARGET_LOG is not set +# CONFIG_IP_NF_TARGET_ULOG is not set +# CONFIG_IP_NF_TARGET_TCPMSS is not set +# CONFIG_IP_NF_TARGET_TOS is not set +# CONFIG_IP_NF_TARGET_ECN is not set +# CONFIG_IP_NF_TARGET_DSCP is not set +# CONFIG_IP_NF_TARGET_FTOS is not set +# CONFIG_IP_NF_TARGET_MARK is not set +# CONFIG_IP_NF_TARGET_CLASSIFY is not set +# CONFIG_IP_NF_ARPTABLES is not set +# CONFIG_IP_NF_COMPAT_IPCHAINS is not set +# CONFIG_IP_NF_COMPAT_IPFWADM is not set +# CONFIG_IP_NF_RAW is not set +# CONFIG_IP_NF_MATCH_ADDRTYPE is not set +# CONFIG_IP_NF_MATCH_REALM is not set + +# +# Bridge: Netfilter Configuration +# +# CONFIG_ETHWAN is not set +# CONFIG_BRIDGE_NF_EBTABLES is not set +# CONFIG_BRIDGE_EBT_BROUTE is not set +# CONFIG_BRIDGE_EBT_T_FILTER is not set +# CONFIG_BRIDGE_EBT_T_NAT is not set +# CONFIG_BRIDGE_EBT_802_3 is not set +# CONFIG_BRIDGE_EBT_AMONG is not set +# CONFIG_BRIDGE_EBT_ARP is not set +# CONFIG_BRIDGE_EBT_IP is not set +# CONFIG_BRIDGE_EBT_LIMIT is not set +# CONFIG_BRIDGE_EBT_MARK is not set +# CONFIG_BRIDGE_EBT_PKTTYPE is not set +# CONFIG_BRIDGE_EBT_STP is not set +# CONFIG_BRIDGE_EBT_VLAN is not set +# CONFIG_BRIDGE_EBT_TIME is not set +# CONFIG_BRIDGE_EBT_ARPREPLY is not set +# CONFIG_BRIDGE_EBT_DNAT is not set +# CONFIG_BRIDGE_EBT_MARK_T is not set +# CONFIG_BRIDGE_EBT_REDIRECT is not set +# CONFIG_BRIDGE_EBT_SNAT is not set +# CONFIG_BRIDGE_EBT_FTOS_T is not set +# CONFIG_BRIDGE_EBT_LOG is not set + +# +# SCTP Configuration (EXPERIMENTAL) +# +# CONFIG_IP_SCTP is not set +CONFIG_ATM=y +# CONFIG_ATM_CLIP is not set +# CONFIG_ATM_LANE is not set +CONFIG_ATM_BR2684=y +# CONFIG_ATM_BR2684_IPFILTER is not set +CONFIG_ATM_RT2684=y +# CONFIG_VLAN_8021Q is not set +# CONFIG_LLC2 is not set +# CONFIG_IPX is not set +# CONFIG_ATALK is not set +# CONFIG_X25 is not set +# CONFIG_LAPB is not set +# CONFIG_NET_DIVERT is not set +# CONFIG_ECONET is not set +# CONFIG_WAN_ROUTER is not set +# CONFIG_NET_FASTROUTE is not set +# CONFIG_NET_HW_FLOWCONTROL is not set + +# +# QoS and/or fair queueing +# +# CONFIG_NET_SCHED is not set +# CONFIG_NET_SCH_CLK_JIFFIES is not set +# NET_SCH_CLK_GETTIMEOFDAY is not set +# NET_SCH_CLK_CPU is not set +# CONFIG_NET_SCH_CBQ is not sety +# CONFIG_NET_SCH_HTB is not sety +# CONFIG_NET_SCH_HFSC is not set +# CONFIG_NET_SCH_CSZ is not set +# CONFIG_NET_SCH_ATM is not set +# CONFIG_NET_SCH_PRIO is not sety +# CONFIG_NET_SCH_RED is not set +# CONFIG_NET_SCH_SFQ is not sety +# CONFIG_NET_SCH_TEQL is not set +# CONFIG_NET_SCH_TBF is not set +# CONFIG_NET_SCH_GRED is not set +# CONFIG_NET_SCH_DSMARK is not set +# CONFIG_NET_SCH_DELAY is not set +# CONFIG_NET_SCH_INGRESS is not set +# CONFIG_NET_QOS is not set +# CONFIG_NET_ESTIMATOR is not set +# CONFIG_NET_CLS is not sety +# CONFIG_NET_CLS_TCINDEX is not set +# CONFIG_NET_CLS_ROUTE4 is not set +# CONFIG_NET_CLS_FW is not set +# CONFIG_NET_CLS_U32 is not set +# CONFIG_NET_CLS_RSVP is not set +# CONFIG_NET_CLS_RSVP6 is not set +# CONFIG_NET_CLS_POLICE is not set +# CONFIG_NET_SCH_NETEM is not set +# CONFIG_NET_CLS_IND is not set +# CONFIG_NET_CLS_ACT is not set +# CONFIG_CLS_U32_PERF is not set + + +# +# Network testing +# +# CONFIG_NET_PKTGEN is not set +CONFIG_NETDEVICES=y +# CONFIG_DUMMY is not set +# CONFIG_BONDING is not set +# CONFIG_EQUALIZER is not set +# CONFIG_TUN is not set + +# +# ARCnet devices +# +# CONFIG_ARCNET is not set + +# +# Ethernet (10 or 100Mbit) +# +CONFIG_NET_ETHERNET=y +# CONFIG_MII is not set +# CONFIG_HAPPYMEAL is not set +# CONFIG_SUNGEM is not set +# CONFIG_NET_VENDOR_3COM is not set + +# +# Tulip family network device support +# +# CONFIG_NET_TULIP is not set +# CONFIG_HP100 is not set +# CONFIG_NET_PCI is not set + +# +# Ethernet (1000 Mbit) +# +# CONFIG_ACENIC is not set +# CONFIG_DL2K is not set +# CONFIG_E1000 is not set +# CONFIG_NS83820 is not set +# CONFIG_HAMACHI is not set +# CONFIG_YELLOWFIN is not set +# CONFIG_R8169 is not set +# CONFIG_SK98LIN is not set +# CONFIG_TIGON3 is not set + +# +# Ethernet (10000 Mbit) +# +# CONFIG_SLIP is not set +# CONFIG_IXGB is not set +# CONFIG_S2IO is not set + +CONFIG_PPP=y +CONFIG_PPP_MULTILINK=y +CONFIG_PPP_FILTER=y +CONFIG_PPP_ASYNC=y +CONFIG_PPP_SYNC_TTY=y +# CONFIG_PPP_DEFLATE is not set +# CONFIG_PPP_BSDCOMP is not set +CONFIG_PPPOE=y +CONFIG_PPPOATM=y + +# +# Wireless LAN (non-hamradio) +# +# CONFIG_NET_RADIO is not set +# CONFIG_STRIP is not set + +# +# Wireless 802.11b ISA/PCI cards support +# +# CONFIG_HERMES is not set +# CONFIG_ATMEL is not set + +# +# Prism GT/Duette 802.11(a/b/g) PCI/Cardbus support +# CONFIG_PRISM54 is not set + +# +# Token Ring devices +# +# CONFIG_SHAPER is not set +# CONFIG_NETCONSOLE is not set +# CONFIG_TR is not set + +# +# Wan interfaces +# +# CONFIG_WAN is not set + +# +# ATM drivers +# +# CONFIG_ATM_TCP is not set +# CONFIG_ATM_LANAI is not set +# CONFIG_ATM_ENI is not set +# CONFIG_ATM_FIRESTREAM is not set +# CONFIG_ATM_ZATM is not set +# CONFIG_ATM_NICSTAR is not set +# CONFIG_ATM_IDT77252 is not set +# CONFIG_ATM_AMBASSADOR is not set +# CONFIG_ATM_HORIZON is not set +# CONFIG_ATM_IA is not set +# CONFIG_ATM_FORE200E_MAYBE is not set +# CONFIG_ATM_HE is not set +# CONFIG_FDDI is not set +# CONFIG_HIPPI is not set +# + +# Amateur Radio support +# +# CONFIG_HAMRADIO is not set + +# +# IrDA (infrared) support +# +# CONFIG_IRDA is not set + +# +# Bluetooth support +# +# CONFIG_BT is not set +# CONFIG_NETPOLL is not set +# CONFIG_NET_POLL_CONTROLLER is not set + +# +# ISDN subsystem +# +# CONFIG_ISDN is not set + +# +# Telephony Support +# +# CONFIG_PHONE is not set + +# +# Profiling Support +# +# CONFIG_BCMPROF is not set + +# +# Input device support +# +# CONFIG_INPUT is not set + +# +# Userland interfaces +# + +# +# Input I/O drivers +# +# CONFIG_GAMEPORT is not set +# CONFIG_SOUND_GAMEPORT is not set +# CONFIG_SERIO is not set +# CONFIG_SERIO_I8042 is not set + +# +# Input Device Drivers +# + +# +# Character devices +# +# CONFIG_VT is not set +# CONFIG_SERIAL_NONSTANDARD is not set + +# +# Serial drivers +# +# CONFIG_SERIAL_8250 is not set + +# +# Non-8250 serial port support +# +# CONFIG_UNIX98_PTYS is not set +# CONFIG_QIC02_TAPE is not set +CONFIG_LEGACY_PTYS=y +CONFIG_LEGACY_PTY_COUNT=2 + +# +# IPMI +# +# CONFIG_IPMI_HANDLER is not set + +# +# Watchdog Cards +# +# CONFIG_WATCHDOG is not set +# CONFIG_RTC is not set +# CONFIG_GEN_RTC is not set +# CONFIG_DTLK is not set +# CONFIG_R3964 is not set +# CONFIG_APPLICOM is not set + +# +# Ftape, the floppy tape device driver +# +# CONFIG_FTAPE is not set +# CONFIG_AGP is not set +# CONFIG_DRM is not set +# CONFIG_RAW_DRIVER is not set + +# +# I2C support +# +# CONFIG_I2C is not set + +# +# Dallas's 1-wire bus +# +# CONFIG_W1 is not set + +# +# Misc devices +# + +# +# Multimedia devices +# +# CONFIG_VIDEO_DEV is not set + +# +# Digital Video Broadcasting Devices +# +# CONFIG_DVB is not set + +# +# Graphics support +# + +# +# Sound +# +# CONFIG_SOUND is not set + +# +# USB support +# +# CONFIG_USB is not set +# CONFIG_USB_DEBUG is not set +# + +# Miscellaneous USB options +# +# CONFIG_USB_DEVICEFS is not set +# CONFIG_USB_BANDWIDTH is not set +# CONFIG_USB_DYNAMIC_MINORS is not set + +# +# USB Host Controller Drivers +# +# CONFIG_USB_EHCI_HCD is not set +# CONFIG_USB_OHCI_HCD is not set +# CONFIG_USB_UHCI_HCD is not set +# CONFIG_USB_EHCI_SPLIT_ISO is not set +# CONFIG_USB_EHCI_ROOT_HUB_TT is not set + +# +# USB Device Class drivers +# +# CONFIG_USB_BLUETOOTH_TTY is not set +# CONFIG_USB_ACM is not set +# CONFIG_USB_PRINTER is not set +# CONFIG_USB_STORAGE is not set + +# +# USB Human Interface Devices (HID) +# +# CONFIG_USB_HID is not set + +# +# Input core support is needed for USB HID input layer or HIDBP support +# + +# +# USB HID Boot Protocol drivers +# + +# +# USB Imaging devices +# +# CONFIG_USB_MDC800 is not set + +# +# USB Multimedia devices +# +# CONFIG_USB_DABUSB is not set + +# +# Video4Linux support is needed for USB Multimedia device support +# + +# +# USB Network adaptors +# +# CONFIG_USB_CATC is not set +# CONFIG_USB_KAWETH is not set +# CONFIG_USB_PEGASUS is not set +# CONFIG_USB_RTL8150 is not set +# CONFIG_USB_USBNET is not set + +# +# USB Host-to-Host Cables +# +# CONFIG_USB_ALI_M5632 is not set +# CONFIG_USB_AN2720 is not set +# CONFIG_USB_BELKIN is not set +# CONFIG_USB_GENESYS is not set +# CONFIG_USB_NET1080 is not set +# CONFIG_USB_PL2301 is not set + +# +# Intelligent USB Devices/Gadgets +# +# CONFIG_USB_ARMLINUX is not set +# CONFIG_USB_EPSON2888 is not set +# CONFIG_USB_ZAURUS is not set +# CONFIG_USB_CDCETHER is not set + +# +# USB Network Adapters +# +# CONFIG_USB_AX8817X is not set + +# +# USB port drivers +# + +# +# USB Serial Converter support +# +# CONFIG_USB_SERIAL is not set + +# +# USB Miscellaneous drivers +# +# CONFIG_USB_EMI62 is not set +# CONFIG_USB_EMI26 is not set +# CONFIG_USB_TIGL is not set +# CONFIG_USB_AUERSWALD is not set +# CONFIG_USB_RIO500 is not set +# CONFIG_USB_LEGOTOWER is not set +# CONFIG_USB_LCD is not set +# CONFIG_USB_LED is not set +# CONFIG_USB_CYTHERM is not set +# CONFIG_USB_SPEEDTOUCH is not set +# CONFIG_USB_PHIDGETSERVO is not set + +# +# USB Gadget Support +# +# CONFIG_USB_GADGET is not set + +# +# File systems +# +# CONFIG_EXT2_FS is not set +# CONFIG_EXT3_FS is not set +# CONFIG_JBD is not set +# CONFIG_REISERFS_FS is not set +# CONFIG_JFS_FS is not set +# CONFIG_XFS_FS is not set +# CONFIG_MINIX_FS is not set +# CONFIG_ROMFS_FS is not set +# CONFIG_QUOTA is not set +# CONFIG_AUTOFS_FS is not set +# CONFIG_AUTOFS4_FS is not set + +# +# CD-ROM/DVD Filesystems +# +# CONFIG_ISO9660_FS is not set +# CONFIG_UDF_FS is not set + +# +# DOS/FAT/NT Filesystems +# +# CONFIG_MSDOS_FS is not set +# CONFIG_VFAT_FS is not set +# CONFIG_NTFS_FS is not set + +# +# Pseudo filesystems +# +CONFIG_PROC_FS=y +CONFIG_PROC_KCORE=y +# CONFIG_SYSFS is not set +# CONFIG_DEVFS_FS is not set +CONFIG_TMPFS=y +# CONFIG_HUGETLB_PAGE is not set +# CONFIG_RAMFS is not set + +# +# Miscellaneous filesystems +# +# CONFIG_ADFS_FS is not set +# CONFIG_AFFS_FS is not set +# CONFIG_HFS_FS is not set +# CONFIG_HFSPLUS_FS is not set +# CONFIG_BEFS_FS is not set +# CONFIG_BFS_FS is not set +# CONFIG_EFS_FS is not set +# CONFIG_JFFS_FS is not set +# CONFIG_JFFS2_FS is not set +# CONFIG_SQUASHFS is not set +# CONFIG_CRAMFS is not set +# CONFIG_LZMA_FS_INFLATE is not set +# CONFIG_ZLIB_FS_INFLATE is not set +# CONFIG_VXFS_FS is not set +# CONFIG_HPFS_FS is not set +# CONFIG_QNX4FS_FS is not set +# CONFIG_SYSV_FS is not set +# CONFIG_UFS_FS is not set + +# +# Network File Systems +# +# CONFIG_NFS_FS is not set +# CONFIG_NFS_V3 is not set +# CONFIG_NFS_V4 is not set +# CONFIG_NFS_DIRECTIO is not set +# CONFIG_NFSD is not set +# CONFIG_ROOT_NFS is not set +# CONFIG_LOCKD is not set +# CONFIG_LOCKD_V4 is not set +# CONFIG_EXPORTFS is not set +# CONFIG_SUNRPC is not set +# CONFIG_RPCSEC_GSS_KRB5 is not set +# CONFIG_SMB_FS is not set +# CONFIG_CIFS is not set +# CONFIG_NCP_FS is not set +# CONFIG_CODA_FS is not set +# CONFIG_INTERMEZZO_FS is not set +# CONFIG_AFS_FS is not set + +# +# Partition Types +# +# CONFIG_PARTITION_ADVANCED is not set + +# +# Native Language Support +# +# CONFIG_NLS is not set + +# +# Kernel hacking +# +CONFIG_CROSSCOMPILE=y +CONFIG_CMDLINE="" +# CONFIG_DEBUG_KERNEL is not set + +# +# Security options +# +# CONFIG_SECURITY is not set + +# +# Cryptographic options +# +# CONFIG_CRYPTO is not set +# CONFIG_CRYPTO_HMAC is not set +# CONFIG_CRYPTO_NULL is not set +# CONFIG_CRYPTO_MD4 is not set +# CONFIG_CRYPTO_MD5 is not set +# CONFIG_CRYPTO_SHA1 is not set +# CONFIG_CRYPTO_SHA1_Z990 is not set +# CONFIG_CRYPTO_SHA256 is not set +# CONFIG_CRYPTO_SHA512 is not set +# CONFIG_CRYPTO_DES is not set +# CONFIG_CRYPTO_DES_Z990 is not set +# CONFIG_CRYPTO_BLOWFISH is not set +# CONFIG_CRYPTO_TWOFISH is not set +# CONFIG_CRYPTO_SERPENT is not set +# CONFIG_CRYPTO_AES is not set +# CONFIG_CRYPTO_AES_586 is not set +# CONFIG_CRYPTO_CAST5 is not set +# CONFIG_CRYPTO_CAST6 is not set +# CONFIG_CRYPTO_TEA is not set +# CONFIG_CRYPTO_ARC4 is not set +# CONFIG_CRYPTO_KHAZAD is not set +# CONFIG_CRYPTO_DEFLATE is not set +# CONFIG_CRYPTO_MICHAEL_MIC is not set +# CONFIG_CRYPTO_CRC32C is not set +# CONFIG_CRYPTO_TEST is not set + +# +# Library routines +# +# CONFIG_CRC32 is not set +# CONFIG_LIBCRC32C is not set diff --git a/hostTools/scripts/gendefconfig b/hostTools/scripts/gendefconfig new file mode 100644 index 0000000..bb719da --- /dev/null +++ b/hostTools/scripts/gendefconfig @@ -0,0 +1,398 @@ +#!/bin/bash + +#**************************************************************************** +# +# Copyright (c) 2001, 2002, 2003, 2004 Broadcom Corporation +# All Rights Reserved +# No portions of this material may be reproduced in any form without the +# written permission of: +# Broadcom Corporation +# 16251 Laguna Canyon Road +# Irvine, California 92618 +# All information contained in this document is Broadcom Corporation +# company private, proprietary, and trade secret. +# +#**************************************************************************** + +driver_setup () +{ + + BRCM_DRIVER_NAME=BRCM_DRIVER_$1 + eval BRCM_DRIVER_VAL=\$$BRCM_DRIVER_NAME + LINUX_CONFIG_NAME=CONFIG_BCM_"$2" + LINUX_IMPL_NAME=CONFIG_BCM9"$BRCM_CHIP"_"$2"_IMPL + + if [ "$BRCM_DRIVER_VAL" != "" ]; then + SEDCMD="$SEDCMD -e 's/# $LINUX_CONFIG_NAME is not set/$LINUX_CONFIG_NAME="$BRCM_DRIVER_VAL"/'" + fi + SEDCMD="$SEDCMD -e 's/# "$LINUX_IMPL_NAME"/"$LINUX_CONFIG_NAME"_IMPL/'" +} + + +netfilter_setup () +{ + + SEDCMD="$SEDCMD -e 's/# CONFIG_IP_NF_"$1" is not set/CONFIG_IP_NF_"$1"="$2"/'" + +} + +general_setup () +{ + + SEDCMD="$SEDCMD -e 's/# "$1" is not set/"$1"="$2"/'" + +} + +TEMPLATE=$HOSTTOOLS_DIR/scripts/defconfig-bcm.template +#TEMPLATE=defconfig-bcm.template + +. $1 + +############################################################ +# Driver config generation +############################################################ + +SEDCMD="$SEDCMD -e 's/# CONFIG_BCM9"$BRCM_CHIP" is not set/CONFIG_BCM9"$BRCM_CHIP"=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BCM_BOARD is not set/CONFIG_BCM_BOARD=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BCM9"$BRCM_CHIP"_BOARD_IMPL/CONFIG_BCM_BOARD_IMPL/'" + +driver_setup "ATM" "ATMAPI" + +driver_setup "ATM" "BLAA" + +# 63xx Chip ATM Diagnostic +if [ "$BUILD_DIAGAPP" != "" ]; then + general_setup CONFIG_BCM_ATMTEST $BRCM_DRIVER_ATM +fi +SEDCMD="$SEDCMD -e 's/# CONFIG_BCM9"$BRCM_CHIP"_ATMTEST_IMPL/CONFIG_BCM_ATMTEST_IMPL/'" + +driver_setup "ADSL" "ADSL" + +driver_setup "ETHERNET" "ENET" + +driver_setup "USB" "USB" + +driver_setup "WIRELESS" "WLAN" +if [ "$BRCM_DRIVER_WIRELESS" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_NET_RADIO is not set/CONFIG_NET_RADIO=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_HOTPLUG is not set/CONFIG_HOTPLUG=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_SYSFS is not set/CONFIG_SYSFS=y/'" +if [ "$BRCM_CHIP" = "6348" -o "$BRCM_CHIP" = "6358" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_PCI is not set/CONFIG_PCI=y/'" +fi +fi + +if [ "$BRCM_CHIP" = "6348" -o "$BRCM_CHIP" = "6358" ]; then +driver_setup "PCI" "PCI" +if [ "$BRCM_DRIVER_PCI" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_PCI is not set/CONFIG_PCI=y/'" +fi +fi + +if [ "$LINUX_DRIVER_USB_HOST" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_USB is not set/CONFIG_USB=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_USB_OHCI_HCD is not set/CONFIG_USB_OHCI_HCD=y/'" +if [ "$BRCM_CHIP" != "6348" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_USB_EHCI_HCD is not set/CONFIG_USB_EHCI_HCD=y/'" +fi + SEDCMD="$SEDCMD -e 's/# CONFIG_PCI is not set/CONFIG_PCI=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_USB_PRINTER is not set/CONFIG_USB_PRINTER=y/'" +if [ "$LINUX_DRIVER_CDCETHER" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_USB_USBNET is not set/CONFIG_USB_USBNET=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_USB_CDCETHER is not set/CONFIG_USB_CDCETHER=y/'" +fi +fi + +driver_setup "VDSL" "VDSL" + +driver_setup "PHONE" "ENDPOINT" + +driver_setup "BCMPROF" "BCMPROF" + +driver_setup "HPNA" "HPNA" + +driver_setup "SECURITY" "SECURITY" + +driver_setup "PROCFS" "PROCFS" + +driver_setup "SERIAL" "SERIAL" + +# CFI Not needed anymore and all flash sectores used for rootfs are treated as ROM +# CFI only used by JFFS2 +if [ "$BRCM_KERNEL_ROOTFS" = "squashfs" -o "$BRCM_KERNEL_ROOTFS" = "cramfs" ]; then + + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_BLOCK_RO is not set/CONFIG_MTD_BLOCK_RO=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_BCM963XX is not set/CONFIG_MTD_BCM963XX=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_ROM is not set/CONFIG_MTD_ROM=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_COMPLEX_MAPPINGS is not set/CONFIG_MTD_COMPLEX_MAPPINGS=y/'" + +elif [ "$BRCM_KERNEL_ROOTFS" = "jffs2" ]; then + + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_BLOCK is not set/CONFIG_MTD_BLOCK=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_PARTITIONS is not set/CONFIG_MTD_PARTITIONS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI is not set/CONFIG_MTD_CFI=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_JEDECPROBE is not set/CONFIG_MTD_JEDECPROBE=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_GEN_PROBE is not set/CONFIG_MTD_GEN_PROBE=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_ADV_OPTIONS is not set/CONFIG_MTD_CFI_ADV_OPTIONS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_NOSWAP is not set/CONFIG_MTD_CFI_NOSWAP=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_B1 is not set/CONFIG_MTD_CFI_B1=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_B2 is not set/CONFIG_MTD_CFI_B2=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_I2 is not set/CONFIG_MTD_CFI_I1=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_I2 is not set/CONFIG_MTD_CFI_I2=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_CFI_AMDSTD is not set/CONFIG_MTD_CFI_AMDSTD=y/'" +# Change Mbytes to bytes in Hex + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_PHYSMAP is not set/CONFIG_MTD_PHYSMAP=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_PHYSMAP_START is not set/CONFIG_MTD_PHYSMAP_START=BFC00000/'" + BRCM_FLASH_SIZE=$(($BRCM_FLASH_SIZE*100000)) + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_PHYSMAP_LEN is not set/CONFIG_MTD_PHYSMAP_LEN="$BRCM_FLASH_SIZE"/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_PHYSMAP_BUSWIDTH is not set/CONFIG_MTD_PHYSMAP_BUSWIDTH=2/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_MTD_COMPLEX_MAPPINGS is not set/CONFIG_MTD_COMPLEX_MAPPINGS=y/'" +fi + + +############################################################ +# Root file system config generation +############################################################ +if [ "$BRCM_KERNEL_ROOTFS" = "nfs" ]; then + SEDCMD="$SEDCMD -e 's/CONFIG_BCM_ENET=m/CONFIG_BCM_ENET=y/'" + SEDCMD="$SEDCMD -e 's?# CONFIG_ROOT_NFS_DIR is not set?CONFIG_ROOT_NFS_DIR=\"$PROFILE_DIR/fs\"?'" + SEDCMD="$SEDCMD -e 's/# CONFIG_ROOTFS_NFS is not set/CONFIG_ROOTFS_NFS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_NFS_FS is not set/CONFIG_NFS_FS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_ROOT_NFS is not set/CONFIG_ROOT_NFS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_IP_PNP is not set/CONFIG_IP_PNP=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_SUNRPC is not set/CONFIG_SUNRPC=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_LOCKD is not set/CONFIG_LOCKD=y/'" +elif [ "$BRCM_KERNEL_ROOTFS" = "jffs2" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_ROOTFS_JFFS2 is not set/CONFIG_ROOTFS_JFFS2=y/'" + #ROOTDEV="root=/dev/mtdblock2 ro" + ROOTDEV="root=31:2 ro noinitrd" + SEDCMD="$SEDCMD -e 's?# CONFIG_ROOT_FLASHFS is not set?CONFIG_ROOT_FLASHFS=\"$ROOTDEV\"?'" + SEDCMD="$SEDCMD -e 's/# CONFIG_JFFS2_FS is not set/CONFIG_JFFS2_FS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_JFFS2_FS_DEBUG/CONFIG_JFFS2_FS_DEBUG/'" +else + if [ "$BRCM_KERNEL_ROOTFS" = "squashfs" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_ROOTFS_SQUASHFS is not set/CONFIG_ROOTFS_SQUASHFS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_SQUASHFS is not set/CONFIG_SQUASHFS=y/'" + #SEDCMD="$SEDCMD -e 's/# CONFIG_ZLIB_FS_INFLATE is not set/CONFIG_ZLIB_FS_INFLATE=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_LZMA_FS_INFLATE is not set/CONFIG_LZMA_FS_INFLATE=y/'" + fi + if [ "$BRCM_KERNEL_ROOTFS" = "cramfs" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_ROOTFS_CRAMFS is not set/CONFIG_ROOTFS_CRAMFS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_CRAMFS is not set/CONFIG_CRAMFS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_LZMA_FS_INFLATE is not set/CONFIG_LZMA_FS_INFLATE=y/'" + fi + + #ROOTDEV="root=/dev/mtdblock0 ro noinitrd" + ROOTDEV="root=31:0 ro noinitrd" + SEDCMD="$SEDCMD -e 's?# CONFIG_ROOT_FLASHFS is not set?CONFIG_ROOT_FLASHFS=\"$ROOTDEV\"?'" +fi + + +############################################################ +# Kernel Debug config generation +############################################################ + +if [ "$BRCM_KERNEL_DEBUG" = "y" ]; then +SEDCMD="$SEDCMD -e 's/# CONFIG_REMOTE_DEBUG is not set/CONFIG_REMOTE_DEBUG=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_GDB_CONSOLE is not set/CONFIG_GDB_CONSOLE=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_KALLSYMS is not set/CONFIG_KALLSYMS=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_KALLSYMS_EXTRA_PASS is not set/CONFIG_KALLSYMS_EXTRA_PASS=y/'" +fi + + +############################################################ +# Kernel preemption +############################################################ + +if [ "$BRCM_KERNEL_PREEMPT" = "y" ]; then +SEDCMD="$SEDCMD -e 's/# CONFIG_PREEMPT is not set/CONFIG_PREEMPT=y/'" +fi + + +############################################################ +# Netfilter config generation +############################################################ +NETFILTER_MODULES="CONNTRACK FTP TFTP TALK H323 IRC PPTP IPSEC DTX8 WM PT IPTABLES \ + MATCH_STATE MATCH_LIMIT MATCH_MARK RTSP \ + FILTER MANGLE NAT NAT_NEEDED NAT_TALK NAT_H323 NAT_SNMP_BASIC NAT_IRC NAT_FTP NAT_TFTP \ + TARGET_MASQUERADE TARGET_REDIRECT TARGET_LOG TARGET_TCPMSS TARGET_FTOS TARGET_MARK" + +NF_FIREWALL_MODULES="IPTABLES MATCH_STATE MATCH_LIMIT FILTER TARGET_TCPMSS" + +NF_MANGLE_MODULES="MANGLE MATCH_MARK TARGET_FTOS TARGET_MARK" + +NF_NAT_MODULES="IPTABLES CONNTRACK NAT NAT_NEEDED TARGET_MASQUERADE TARGET_REDIRECT" + +NF_PPPREMOTE_MODULES="IPTABLES CONNTRACK NAT NAT_NEEDED FILTER TARGET_TCPMSS MATCH_LIMIT" + +ALGS="FTP TFTP TALK H323 IRC PPTP IPSEC SNMP DTX8 WM PT RTSP" +FTP_ALG_MODULES="FTP NAT_FTP" +TFTP_ALG_MODULES="TFTP NAT_TFTP" +H323_ALG_MODULES="H323 NAT_H323" +IRC_ALG_MODULES="IRC NAT_IRC" +DTX8_ALG_MODULES="DTX8" +WM_ALG_MODULES="WM" +PT_ALG_MODULES="PT" +PPTP_ALG_MODULES="PPTP" +IPSEC_ALG_MODULES="IPSEC" +RTSP_ALG_MODULES="RTSP" +SNMP_ALG_MODULES="NAT_SNMP_BASIC" +TALK_ALG_MODULES="TALK NAT_TALK" + + +NF_LOG_MODULES="TARGET_LOG" + + +if [ "$BRCM_KERNEL_NETFILTER" != "" ]; then +# for mod in $NETFILTER_MODULES; do +# netfilter_setup $mod $BRCM_KERNEL_NETFILTER +# done + # set up firewall related modules + if [ "$BRCM_KERNEL_NF_FIREWALL" != "" ]; then + for mod in $NF_FIREWALL_MODULES; do + netfilter_setup $mod $BRCM_KERNEL_NF_FIREWALL + done + fi + + # set up packet mangling related modules + if [ "$BRCM_KERNEL_NF_MANGLE" != "" ]; then + for mod in $NF_MANGLE_MODULES; do + netfilter_setup $mod $BRCM_KERNEL_NF_MANGLE + done + fi + + # set up NAT related modules + if [ "$BRCM_KERNEL_NF_NAT" != "" ]; then + # set up required NAT modules + for mod in $NF_NAT_MODULES; do + netfilter_setup $mod $BRCM_KERNEL_NF_NAT + done + #set up ALGs + for alg in $ALGS; do + ALG_NAME=BRCM_KERNEL_NF_NAT_ALG_"$alg" + ALG_MODULE_NAME="$alg"_ALG_MODULES + eval ALG_VAL=\$$ALG_NAME + eval ALG_MODULE_VAL=\$$ALG_MODULE_NAME + if [ "$ALG_VAL" != "" ]; then + for mod in $ALG_MODULE_VAL; do + netfilter_setup $mod $ALG_VAL + done + fi + done + fi + + # set up logging module + if [ "$BRCM_KERNEL_NF_LOG" != "" ]; then + for mod in $NF_LOG_MODULES; do + netfilter_setup $mod $BRCM_KERNEL_NF_LOG + done + fi + + # set up special remote access support modules in PPP IP extension mode + if [ "$BRCM_KERNEL_NF_PPPREMOTE" != "" ]; then + for mod in $NF_PPPREMOTE_MODULES; do + netfilter_setup $mod $BRCM_KERNEL_NF_PPPREMOTE + done + fi + +fi + +############################################################ +# Cryptographic config generation +############################################################ +if [ "$BRCM_KERNEL_CRYPTO" != "" ]; then + + # set up networking options + SEDCMD="$SEDCMD -e 's/# CONFIG_NET_KEY is not set/CONFIG_NET_KEY=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_INET_AH is not set/CONFIG_INET_AH=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_INET_ESP is not set/CONFIG_INET_ESP=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_XFRM_USER is not set/CONFIG_XFRM_USER=y/'" + + # set up cryptographic options + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO is not set/CONFIG_CRYPTO=y/'" + if [ "$BRCM_KERNEL_CRYPTO_HMAC" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO_HMAC is not set/CONFIG_CRYPTO_HMAC=y/'" + fi + if [ "$BRCM_KERNEL_CRYPTO_NULL" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO_NULL is not set/CONFIG_CRYPTO_NULL=y/'" + fi + if [ "$BRCM_KERNEL_CRYPTO_MD5" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO_MD5 is not set/CONFIG_CRYPTO_MD5=y/'" + fi + if [ "$BRCM_KERNEL_CRYPTO_SHA1" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO_SHA1 is not set/CONFIG_CRYPTO_SHA1=y/'" + fi + if [ "$BRCM_KERNEL_CRYPTO_DES" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO_DES is not set/CONFIG_CRYPTO_DES=y/'" + fi + if [ "$BRCM_KERNEL_CRYPTO_AES" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_CRYPTO_AES is not set/CONFIG_CRYPTO_AES=y/'" + fi +fi + +############################################################ +# Bridge Layer filter config generation +############################################################ +if [ "$BUILD_EBTABLES" != "" ]; then +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_NF_EBTABLES is not set/CONFIG_BRIDGE_NF_EBTABLES=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_T_FILTER is not set/CONFIG_BRIDGE_EBT_T_FILTER=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_BROUTE is not set/CONFIG_BRIDGE_EBT_BROUTE=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_IP is not set/CONFIG_BRIDGE_EBT_IP=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_VLAN is not set/CONFIG_BRIDGE_EBT_VLAN=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_TIME is not set/CONFIG_BRIDGE_EBT_TIME=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_MARK_T is not set/CONFIG_BRIDGE_EBT_MARK_T=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_BRIDGE_EBT_FTOS_T is not set/CONFIG_BRIDGE_EBT_FTOS_T=y/'" +fi + +############################################################ +# IGMP PROXY filter config generation +############################################################ +if [ "$BUILD_IGMP" != "" ]; then +SEDCMD="$SEDCMD -e 's/# CONFIG_IP_MROUTE is not set/CONFIG_IP_MROUTE=y/'" +SEDCMD="$SEDCMD -e 's/# CONFIG_IP_MULTICAST is not set/CONFIG_IP_MULTICAST=y/'" +fi + +############################################################ +# Network QoS config generation +############################################################ +if [ "$BRCM_KERNEL_NETQOS" != "" ]; then + SEDCMD="$SEDCMD -e 's/# CONFIG_NET_SCHED is not set/CONFIG_NET_SCHED=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_NET_SCH_CLK_JIFFIES is not set/CONFIG_NET_SCH_CLK_JIFFIES=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_NET_CLS is not set/CONFIG_NET_CLS=y/'" + SEDCMD="$SEDCMD -e 's/# CONFIG_NET_CLS_POLICE is not set/CONFIG_NET_CLS_POLICE=y/'" + general_setup CONFIG_NET_SCH_CBQ $BRCM_KERNEL_NETQOS + general_setup CONFIG_NET_SCH_HTB $BRCM_KERNEL_NETQOS + general_setup CONFIG_NET_SCH_PRIO $BRCM_KERNEL_NETQOS + general_setup CONFIG_NET_SCH_SFQ $BRCM_KERNEL_NETQOS + general_setup CONFIG_NET_CLS_FW $BRCM_KERNEL_NETQOS + general_setup CONFIG_NET_CLS_U32 $BRCM_KERNEL_NETQOS + general_setup CONFIG_NET_CLS_POLICE $BRCM_KERNEL_NETQOS +fi + +############################################################ +# VLAN config generation +############################################################ +if [ "$BUILD_VCONFIG" != "" ]; then +SEDCMD="$SEDCMD -e 's/# CONFIG_VLAN_8021Q is not set/CONFIG_VLAN_8021Q=y/'" +fi + +############################################################ +# WAN operation over Ethernet +############################################################ +if [ "$BUILD_ETHWAN" != "" ]; then +SEDCMD="$SEDCMD -e 's/# CONFIG_ETHWAN is not set/CONFIG_ETHWAN=y/'" +fi + +############################################################ +# PTHREADS support +############################################################ +if [ "$BRCM_PTHREADS" != "" ]; then + general_setup CONFIG_BRCM_USING_PTHREADS $BRCM_PTHREADS +fi + +gen="sed $SEDCMD $TEMPLATE" +#echo $gen +#eval $gen +eval $gen > $KERNEL_DIR/arch/mips/defconfig + + diff --git a/hostTools/scripts/nightlybuild/voice/ccLoadRules/CommEngine/cxcLoadRules.txt b/hostTools/scripts/nightlybuild/voice/ccLoadRules/CommEngine/cxcLoadRules.txt new file mode 100644 index 0000000..b1924ed --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/ccLoadRules/CommEngine/cxcLoadRules.txt @@ -0,0 +1,10 @@ +load \CommEngine\cfe +load \CommEngine\docs +load \CommEngine\hostTools +load \CommEngine\Makefile +load \CommEngine\release +load \CommEngine\toolChains +load \CommEngine\sgibcm_2_4_17 +load \CommEngine\targets +load \CommEngine\userapps +load \CommEngine\xChange diff --git a/hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common/cxcLoadRules.txt b/hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common/cxcLoadRules.txt new file mode 100644 index 0000000..b60daae --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common/cxcLoadRules.txt @@ -0,0 +1,108 @@ +load \ldx_apps\apps\buildHostLib.sh +load \ldx_apps\apps\chip3341_common/ +load \ldx_apps\apps\dsl_setenv.bat +load \ldx_apps\apps\dspApp3341 +load \ldx_apps\apps\dspApp3341_7231_729a_726/ +load \ldx_apps\apps\dspApp3341_faxr2 +load \ldx_apps\apps\dspApp3341_fxo_ext +load \ldx_apps\apps\dspApp3341_tdm_ext +load \ldx_apps\apps\dspApp3341_tdm_conf +load \ldx_apps\apps\dspApp3341_tdm_faxr2_ext +load \ldx_apps\apps\refApp_3341_common +load \ldx_apps\apps\refApp_3341_common_dsl_tdm +load \ldx_apps\apps\refApp_3341_common_dsl_fxo +load \ldx_apps\apps\dspApp3368/ +load \ldx_apps\apps\dspApp3368_super/ +load \ldx_apps\apps\dspApp6358/ +load \ldx_apps\apps\hausware_libs_gateway_distdsp/ +load \ldx_apps\apps\hausware_libs_gateway_distdsp_dsl_tdm +load \ldx_apps\apps\hausware_libs_gateway_distdsp_dsl_fxo +load \ldx_apps\apps\hausware_libs_gateway_disthost_33xx/ +load \ldx_apps\apps\hausware_libs_gateway_disthost_63xx/ +load \ldx_apps\apps\hausware_libs_gateway_disthost_6358/ +load \ldx_apps\apps\hausware_libs_gateway_disthost_63xx_tdm +load \ldx_apps\apps\hausware_libs_gateway_disthost_63xx_fxo +load \ldx_apps\apps\hausware_libs_gateway_disthost_63xx_dualdsp +load \ldx_apps\apps\hostApp_6348gw +load \ldx_apps\apps\ldxApps.mk +load \ldx_apps\apps\makefile +load \ldx_apps\apps\readme.txt +load \ldx_apps\apps\refApp_3341_common/ +load \ldx_apps\apps\setenv.bash +load \ldx_apps\apps\setenv.bat +load \ldx_apps\common +load \ldx_apps\makefile + +load \ldx_apps\common\inc +load \ldx_apps\common\source\hal_3341 +load \ldx_apps\common\source\hal_3341tdm +load \ldx_apps\common\source\hal_3341_hybrid +load \ldx_apps\common\source\hal_6358_dsp +load \ldx_apps\common\source\hal_3368_dsp +load \ldx_apps\common\source\resample +load \ldx_apps\common\commoncode.mak +load \ldx_apps\common\commoncode_targets.mak +load \ldx_apps\drivers + +#load \voice_res_gw +load \voice_res_gw\boardHal\inc +load \voice_res_gw\boardHal\src\bcm6348_Le9502FXO +load \voice_res_gw\boardHal\src\bcm6348_Le9500FXO +load \voice_res_gw\boardHal\src\bcm6348gw +load \voice_res_gw\boardHal\src\bcm6348gw_pcm +load \voice_res_gw\boardHal\src\bcm6358vw +load \voice_res_gw\boardHal\src\common +load \voice_res_gw\boardHal\src\common3368 +load \voice_res_gw\boardHal\src\common6358 +load \voice_res_gw\boardHal\src\common6348 +load \voice_res_gw\boardHal\src\sim +load \voice_res_gw\casCtl +load \voice_res_gw\classStm +load \voice_res_gw\cmtdCtl +load \voice_res_gw\codec +load \voice_res_gw\endpt +load \voice_res_gw\hdsp +load \voice_res_gw\heartbeat +load \voice_res_gw\inc +load \voice_res_gw\lhapi +load \voice_res_gw\pstnCtl +load \voice_res_gw\tpdCtl +load \voice_res_gw\voice_res_gw.mk +load \voice_res_gw\voice_res_gw_env.mk +load \voice_res_gw\voice_res_gw_targets.mk + +load \xchg_common\assert +load \xchg_common\blog +load \xchg_common\bos +load \xchg_common\containers +load \xchg_common\crt +load \xchg_common\mem +load \xchg_common\rules +load \xchg_common\sme +load \xchg_common\str +load \xchg_common\tools\bin +load \xchg_common\trace +load \xchg_common\xchg_common.mk +load \xchg_common\xchg_common_app_template.mk +load \xchg_common\xchg_common_env.mk +load \xchg_common\xchg_common_targets.mk + +#load \xchg_drivers +load \xchg_drivers\arch +load \xchg_drivers\bcm3341 +load \xchg_drivers\bcm6348 +load \xchg_drivers\bcm3368 +load \xchg_drivers\bcm6358 +load \xchg_drivers\bcm63xx +load \xchg_drivers\bcm33xx +load \xchg_drivers\inc +load \xchg_drivers\xchg_drivers.mk +load \xchg_drivers\xchg_drivers_env.mk +load \xchg_drivers\xchg_drivers_targets.mk + +load \prot_callctrl + +load \ldx_tools +load \ldx_hausware +#load \zOEMtools_gnu_mips_elf +#load \zOEMtools_zsp \ No newline at end of file diff --git a/hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common_ldx197/cxcLoadRules.txt b/hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common_ldx197/cxcLoadRules.txt new file mode 100644 index 0000000..b59dfcd --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/ccLoadRules/dslx_common_ldx197/cxcLoadRules.txt @@ -0,0 +1,89 @@ +load \ldx_apps\makefile +load \ldx_apps\apps\buildHostLib.sh +load \ldx_apps\apps\dsl_setenv.bat +load \ldx_apps\apps\ldxApps.mk +load \ldx_apps\apps\makefile +load \ldx_apps\apps\readme.txt +load \ldx_apps\apps\setenv.bat +load \ldx_apps\apps\chip3341_common +load \ldx_apps\apps\dspApp3341 +load \ldx_apps\apps\dspApp3341_ext +load \ldx_apps\apps\dspApp3341_faxr +load \ldx_apps\apps\dspApp3341_faxrinternal +load \ldx_apps\apps\dspApp3341_fxo +load \ldx_apps\apps\dspApp3341_fxo_ext +load \ldx_apps\apps\dspApp3341_hybrid +load \ldx_apps\apps\dspApp3341_hybrid_ext +load \ldx_apps\apps\dspApp3341_hybrid_fxo_ext +load \ldx_apps\apps\dspApp3341_tdm +load \ldx_apps\apps\xtp6348_hausware_libs +load \ldx_apps\apps\xtp6348fxo_hausware_libs +load \ldx_apps\apps\xtp6348hybrid_hausware_libs +load \ldx_apps\apps\xtp6348hybrid_fxo_hausware_libs +load \ldx_apps\apps\xtp6348tdm_hausware_libs +load \ldx_apps\common\source\hal_3341 +load \ldx_apps\common\source\hal_3341_hybrid +load \ldx_apps\common\source\hal_3341tdm +load \ldx_apps\common\source\resample +load \ldx_apps\common\inc\dspImageArchive.h +load \ldx_apps\common\inc\hal3341.h +load \ldx_apps\common\inc\hal3341hybrid.h +load \ldx_apps\common\inc\hal3341tdm.h +load \ldx_apps\common\inc\hal_ipc.h +load \ldx_apps\common\inc\hal_ipc_slave.h +load \ldx_apps\common\inc\resample.h +load \ldx_apps\common\commoncode_targets.mak +load \ldx_apps\common\commoncode.mak +load \ldx_apps\drivers\drivers.mak +load \ldx_apps\drivers\drivers_targets.mak + +load \ldx_hausware\ldxhausware.cfg +load \ldx_hausware\makefile +load \ldx_hausware\hausware + +load \ldx_tools + +load \voice_res_gw\battCtl +load \voice_res_gw\boardHal\inc +load \voice_res_gw\boardHal\src\bcm6348gw +load \voice_res_gw\boardHal\src\bcm6345gw +load \voice_res_gw\boardHal\src\bcm6348gw_pcm +load \voice_res_gw\boardHal\src\bcm6348LE9502 +load \voice_res_gw\boardHal\src\bcm6348gw_hybrid +load \voice_res_gw\boardHal\src\bcm6348lv +load \voice_res_gw\boardHal\src\bcm6348_Le9502FXO +load \voice_res_gw\boardHal\src\bcm6348_Le9500FXO +load \voice_res_gw\boardHal\src\common +load \voice_res_gw\casCtl +load \voice_res_gw\classStm +load \voice_res_gw\codec +load \voice_res_gw\cmtdCtl +load \voice_res_gw\endpt +load \voice_res_gw\hdsp +load \voice_res_gw\inc +load \voice_res_gw\lhapi +load \voice_res_gw\pstnCtl +load \voice_res_gw\voice_res_gw.mk +load \voice_res_gw\voice_res_gw_env.mk +load \voice_res_gw\voice_res_gw_targets.mk + +load \xchg_common\assert +load \xchg_common\bos +load \xchg_common\containers +load \xchg_common\crt +load \xchg_common\mem +load \xchg_common\rules +load \xchg_common\sme +load \xchg_common\str +load \xchg_common\tools\bin +load \xchg_common\trace +load \xchg_common\xchg_common.mk +load \xchg_common\xchg_common_app_template.mk +load \xchg_common\xchg_common_env.mk +load \xchg_common\xchg_common_targets.mk + +load \xchg_drivers + +#load \zOEMtools_zsp + + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nb_ce_getsrc.sh b/hostTools/scripts/nightlybuild/voice/cxc_nb_ce_getsrc.sh new file mode 100644 index 0000000..45a24ab --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nb_ce_getsrc.sh @@ -0,0 +1,133 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxc_nb_getsrc.sh +# Purpose: Gets the CxC source code form Source Control Database +# Arguments: $1 = 0 - get minnimum latest source code without labeling +# = 1 - label and get full source code +#------------------------------------------------------------------------------ + +if [ -z "${CXNB_LABEL}" ] +then + echo CXNB_LABEL not defined! + exit +fi + +export CXNB_BASECC_LOG_PATH=$(cygpath -u "${CXNB_BASECC_LOCAL_LOG_PATH}") +export CXNB_BASECC_SRC_PATH=$(cygpath -u "${CXNB_BASECC_LOCAL_SRC_PATH}") +export CXNB_BASECC_VIEW_PATH=$(cygpath -u "${CXNB_BASECC_LOCAL_VIEW_PATH}") +export CXNB_BASECC_CC_LRULES_CYGPATH=$(cygpath -u "${CXNB_BASECC_LRULES_PATH}") + +mkdir -p ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL} + +cxcFullLoad=$1 + +if [ "${cxcFullLoad}" = "0" ] +then + if [ "${CXNB_CFG_PREBUILD}" = "no" -o "${CXNB_CFG_PREBUILD_GETSRC}" = "no" ] + then + echo "Skipping prebuild source update at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + exit + fi +else + if [ "${CXNB_CFG_BUILD}" = "no" -o "${CXNB_CFG_BUILD_GETSRC}" = "no" ] + then + echo "Skipping source update at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + exit + fi +fi + +# echo "Removing directory at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log +# rm -f -R ${CXNB_BASECC_SRC_PATH} +# echo "Finished Removing directory at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + +mkdir -p ${CXNB_BASECC_SRC_PATH} +# echo "Finished makeing new directories at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + +if [ "${cxcFullLoad}" = "1" ] +then + #echo "Labeling code at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + #ss label $/${VSS_PROJ_PATH} -L"CXNB_${CXNB_LABEL}" "-Cnightly build" -I-Y + #echo "Finished Labeling code at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + echo "Starting Full CC update at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log +else + echo "Starting Minimum CC update at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log +fi + +# Get the source from CC. Predefined load rules determine what gets loaded +cd ${CXNB_BASECC_VIEW_PATH} + #record baselines of build + echo " " >${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + echo "------------------------------------------------------------------------" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Nightly build Build Configuration: " >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + echo "------------------------------------------------------------------------" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + echo " " >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + cleartool lsstream -cview -fmt "Project %[project]p\nStream %[name]p\n" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Modifiable Components:" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + cleartool lsproject -cview -fmt "\t%[mod_comps]p\n" | sed -e 's/ /\n\t/g' | sort >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Recommended Baselines:" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + for baseline in `cleartool lsstream -cview -fmt "%[rec_bls]p"`; do cleartool lsbl -fmt "%[5]t(%[component]p) %[30]t$baseline\n" $baseline@\\rmna_projects ; done | sort >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Foundation Baselines:" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + for baseline in `cleartool lsstream -cview -fmt "%[found_bls]p"`; do cleartool lsbl -fmt "%[5]t(%[component]p) %[30]t$baseline\n" $baseline@\\rmna_projects ; done | sort >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + #update load rules in the config spec + + #get the original config spec + cleartool catcs >${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_config_spec.txt + #extract the original load rules + if [ "${CXNB_CFG_CC_CONFIGSPEC_UPDATE}" = "yes" ] + then + loadRulesFile=${CXNB_BASECC_CC_LRULES_CYGPATH}/cxcLoadRules.txt + if [ -e "${loadRulesFile}" ] + then + updateRules=0; + grep '^load' ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_config_spec.txt >${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_orig_lrules.txt + diff -b ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_orig_lrules.txt ${loadRulesFile} || updateRules=1 + rm -f ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_orig_lrules.txt + if [ "${updateRules}" = "1" ] + then + echo "CC Config Spec updated at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + #remove load rules from the original config spec + grep -v '^load' ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_config_spec.txt >${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_new_cs.txt + #append new load rules to the config spec + cat ${loadRulesFile}>>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_new_cs.txt + #update the config spec + mv -f ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_new_cs.txt . + cleartool setcs cc_new_cs.txt << EOF +y +EOF + mv -f ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_config_spec.txt ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_config_spec.old + mv -f ./cc_new_cs.txt ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/cc_ce_config_spec.txt + fi + else + echo "ERROR: Could not find default load rules ${loadRulesFile}" >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + echo + fi + fi + + #update the view + cleartool update -force -overwrite << EOF + y +EOF + +if [ "${cxcFullLoad}" = "1" ] +then + echo "Finished Full CC update at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log +else + echo "Finished Minimum CC update at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log +fi + + +# copy everything to a build location (shorter path name) to avoid +# problems with long paths that some tools might have +echo "Copying files to build location started at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log +cp -f -R * ${CXNB_BASECC_SRC_PATH} +echo "Copying files to build location finished at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + +exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nb_getsrc.sh b/hostTools/scripts/nightlybuild/voice/cxc_nb_getsrc.sh new file mode 100644 index 0000000..64c75d1 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nb_getsrc.sh @@ -0,0 +1,133 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxc_nb_getsrc.sh +# Purpose: Gets the CxC source code form Source Control Database +# Arguments: $1 = 0 - get minnimum latest source code without labeling +# = 1 - label and get full source code +#------------------------------------------------------------------------------ + +if [ -z "${CXNB_LABEL}" ] +then + echo CXNB_LABEL not defined! + exit +fi + +export CXNB_LOG_PATH=$(cygpath -u "${CXNB_LOCAL_LOG_PATH}") +export CXNB_SRC_PATH=$(cygpath -u "${CXNB_LOCAL_SRC_PATH}") +export CXNB_VIEW_PATH=$(cygpath -u "${CXNB_LOCAL_VIEW_PATH}") +export CXNB_CC_LRULES_CYGPATH=$(cygpath -u "${CXNB_CC_LRULES_PATH}") + +mkdir -p ${CXNB_LOG_PATH}/${CXNB_LABEL} + +cxcFullLoad=$1 + +if [ "${cxcFullLoad}" = "0" ] +then + if [ "${CXNB_CFG_PREBUILD}" = "no" -o "${CXNB_CFG_PREBUILD_GETSRC}" = "no" ] + then + echo "Skipping prebuild source update at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + exit + fi +else + if [ "${CXNB_CFG_BUILD}" = "no" -o "${CXNB_CFG_BUILD_GETSRC}" = "no" ] + then + echo "Skipping source update at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + exit + fi +fi + +# echo "Removing directory at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +# rm -f -R ${CXNB_SRC_PATH} +# echo "Finished Removing directory at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +mkdir -p ${CXNB_SRC_PATH} +# echo "Finished makeing new directories at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +if [ "${cxcFullLoad}" = "1" ] +then + #echo "Labeling code at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + #ss label $/${VSS_PROJ_PATH} -L"CXNB_${CXNB_LABEL}" "-Cnightly build" -I-Y + #echo "Finished Labeling code at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + echo "Starting Full CC update at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +else + echo "Starting Minimum CC update at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +fi + +# Get the source from CC. Predefined load rules determine what gets loaded +cd ${CXNB_VIEW_PATH} + #record baselines of build + echo " " >${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + echo "------------------------------------------------------------------------" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Nightly build Build Configuration: " >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + echo "------------------------------------------------------------------------" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + echo " " >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + cleartool lsstream -cview -fmt "Project %[project]p\nStream %[name]p\n" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Modifiable Components:" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + cleartool lsproject -cview -fmt "\t%[mod_comps]p\n" | sed -e 's/ /\n\t/g' | sort >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Recommended Baselines:" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + for baseline in `cleartool lsstream -cview -fmt "%[rec_bls]p"`; do cleartool lsbl -fmt "%[5]t(%[component]p) %[30]t$baseline\n" $baseline@\\rmna_projects ; done | sort >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + echo "Foundation Baselines:" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + for baseline in `cleartool lsstream -cview -fmt "%[found_bls]p"`; do cleartool lsbl -fmt "%[5]t(%[component]p) %[30]t$baseline\n" $baseline@\\rmna_projects ; done | sort >>${CXNB_LOG_PATH}/${CXNB_LABEL}/buildconfig.log 2>&1 + + #update load rules in the config spec + + #get the original config spec + cleartool catcs >${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_config_spec.txt + #extract the original load rules + if [ "${CXNB_CFG_CC_CONFIGSPEC_UPDATE}" = "yes" ] + then + loadRulesFile=${CXNB_CC_LRULES_CYGPATH}/cxcLoadRules.txt + if [ -e "${loadRulesFile}" ] + then + updateRules=0; + grep '^load' ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_config_spec.txt >${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_orig_lrules.txt + diff -b ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_orig_lrules.txt ${loadRulesFile} || updateRules=1 + rm -f ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_orig_lrules.txt + if [ "${updateRules}" = "1" ] + then + echo "CC Config Spec updated at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + #remove load rules from the original config spec + grep -v '^load' ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_config_spec.txt >${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_new_cs.txt + #append new load rules to the config spec + cat ${loadRulesFile}>>${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_new_cs.txt + #update the config spec + mv -f ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_new_cs.txt . + cleartool setcs cc_new_cs.txt << EOF +y +EOF + mv -f ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_config_spec.txt ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_config_spec.old + mv -f ./cc_new_cs.txt ${CXNB_LOG_PATH}/${CXNB_LABEL}/cc_config_spec.txt + fi + else + echo "ERROR: Could not find default load rules ${loadRulesFile}" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + echo + fi + fi + + #update the view + cleartool update -force -overwrite << EOF + y +EOF + +if [ "${cxcFullLoad}" = "1" ] +then + echo "Finished Full CC update at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +else + echo "Finished Minimum CC update at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +fi + + +# copy everything to a build location (shorter path name) to avoid +# problems with long paths that some tools might have +echo "Copying files to build location started at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +cp -f -R * ${CXNB_SRC_PATH} +echo "Copying files to build location finished at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_app.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_app.bat new file mode 100644 index 0000000..4756559 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_app.bat @@ -0,0 +1,13 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_app.bat +:: Purpose: Nightly Build and Regression testing application script +:: (builds target application and runs tests on it) +:: Arguments: %1 - target CxC app to build and test +:: %2 - target OS +::------------------------------------------------------------------------------ + +cmd.exe /c cxc_nbrt_build.bat %1 %2 +:: cmd.exe /c cxc_nbrt_test.bat %1 %2 + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_build.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_build.bat new file mode 100644 index 0000000..5ec6804 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_build.bat @@ -0,0 +1,33 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_build.bat +:: Purpose: CxC Nightly Build script (builds and saves CxC images) +:: Arguments: %1 - target CxC app to build +:: %2 - target OS to build the app for +::------------------------------------------------------------------------------ + +if "%2" == "" goto argsError + +set cxnb_build_target=%1 +set cxnb_build_os=%2 + +:: Setup nbrt environment variables specific for this target +call cxc_nbrt_cfg.bat %cxnb_build_target% %cxnb_build_os% + +if "%CXNB_CFG_BUILD%" == "no" goto theend + +bash -C lnx_nbrt_build.sh >> %CXNB_OUTPUT_DIR%\build.log + +goto theend + +:argsError +echo cxc_nbrt_build error: Insufficient arguments arg1=%1; arg2=%2; >>%CXNB_OUTPUT_DIR%\build.log +goto theend + +:theend +set cxnb_build_target= +set cxnb_build_os= +set cxnb_local_output_dir= +exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_ce_postprocess.sh b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_ce_postprocess.sh new file mode 100644 index 0000000..2689b21 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_ce_postprocess.sh @@ -0,0 +1,129 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxc_nbrt_postprocess.sh +# Purpose: Postprocess the nightly build and test logs +#------------------------------------------------------------------------------ + +if [ -z "${CXNB_LABEL}" ] +then + echo CXNB_LABEL not defined! + exit +fi + +cxcOsType=$1 + +export CXNB_BASECC_LOG_PATH=$(cygpath -u "${CXNB_BASECC_LOCAL_LOG_PATH}") +export CXNB_BASECC_LOCAL_SRC_PATH=$(cygpath -u "${CXNB_BASECC_LOCAL_SRC_PATH}") + +echo "Postprocessing at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + +mkdir -p ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/${cxcOsType} +cd ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/${cxcOsType} + +mkdir -p buildlogs +# mkdir -p deps/buildlogs +# mkdir -p debug +# mv -f *.map debug +# mv -f *_sym.bin debug +# mv -f *.txt buildlogs +# mv -f deps/*.txt deps/buildlogs + +# cp -f -v ${CXNB_BASECC_SRC_PATH}/cablex_tools/dev/callagent/callagent.exe ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/ +# cp -f -v ${CXNB_BASECC_SRC_PATH}/cablex_tools/dev/callagent/hhca.cfg ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/ +cp -f -v ${CXNB_BASECC_SRC_PATH}/update*.updt ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/ +cp -f -v ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build*.log ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/buildlogs/ + + +if [ "${CXNB_CFG_SENDMAIL}" = "yes" ] +then + appendFiles= + attachments="build.log.gz cc_ce_config_spec.txt" + + if [ "${CXNB_CFG_BUILD_GETSRC}" = "yes" ] + then + appendFiles="${appendFiles} buildlogs/buildconfig.log" + fi + + echo $appendFiles + + recipients="vmarkovski@broadcom.com,jnicol@broadcom.com" + + echo "------------------------------------------------------------------------">email.txt + echo "CxC Nightly Build summary " >>email.txt + echo "------------------------------------------------------------------------">>email.txt + echo " " >>email.txt + # NOTE: The following greps are dependent on the printouts + # for the Linux commands through the expect script lnx_nbrt_basic_cmds.exp + total=`grep -c "The Linux command \"make PROFILE=9634.*GWV BRCM" buildlogs/build.log` + fail=`grep -c "The Linux command \"make PROFILE=9634.*GWV BRCM.*failed" buildlogs/build.log` + pass=`grep -c "The Linux command \"make PROFILE=9634.*GWV BRCM.*succeeded" buildlogs/build.log` + echo "Total of $total DSL CommEngine apps built: $pass passed, $fail failed. " >>email.txt + grep "The Linux command \"make PROFILE=9634.*GWV BRCM.*failed" buildlogs/build.log >>email.txt + echo " " >>email.txt + grep "The Linux command \"make PROFILE=9634.*GWV BRCM.*succeeded" buildlogs/build.log >>email.txt + echo " " >>email.txt + + echo "------------------------------------------------------------------------">>email.txt + echo "Nightly build auto-update activity " >>email.txt + echo "------------------------------------------------------------------------">>email.txt + echo " " >>email.txt + grep "CC Config Spec" ./../build.log >> email.txt || echo "No config spec update." >> email.txt + echo " " >>email.txt + + if [ -n "${appendFiles}" ] + then + for file in ${appendFiles} + do + if [ -e "${file}" ] + then + cat ${file} >> email.txt + else + echo "POSTPROCESSING ERROR: could not find ${file}!" >>email.txt + fi + done + fi + echo " " >>email.txt + + if [ -e "./../cc_config_spec.old" ] + then + attachments="${attachments} cc_config_spec.old" + fi + + echo "------------------------------------------------------------------------">>email.txt + echo "Nightly build images, logs and source code for ${CXNB_EMAIL_DATE}:" >> email.txt + echo "------------------------------------------------------------------------">>email.txt + echo "Last night's nightly build source code can be found at:" >> email.txt + echo $(cygpath -w //${CXNB_BASECC_LOCAL_SRC_PATH}) >> email.txt + echo "Nightly build images can be found at" >> email.txt + echo $(cygpath -w //${CXNB_BASECC_LOCAL_SRC_PATH}/images) >> email.txt + # echo "Nightly build map files and symbol table images can be found at" >> email.txt + # echo $(cygpath -w //${UNIX_TESTCOMPUTER}/cxnb/cxlogs/${CXNB_UCMCC_PROJECT}/${CXNB_LABEL}/${cxcOsType}/debug) >> email.txt + echo "Nightly build logs can be found at" >> email.txt + echo $(cygpath -w //${CXNB_BASECC_LOCAL_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/buildlogs) >> email.txt + # echo $(cygpath -w //${CXNB_BASECC_LOCAL_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/deps/buildlogs) >> email.txt + + #note: blat doesnt work from UNC directories so copy everything to a local directory + + echo "Finished postprocessing at: " `date +"%T"` >>${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/buildlogs/build.log + gzip ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/buildlogs/build.log + gzip ${CXNB_BASECC_LOG_PATH}/${CXNB_LABEL}/build.log + + for file in ${attachments} + do + cp -f ./../${file} . + done + + attachList="" + for file in ${attachments} + do + attachList="${attachList} -attach ${file}" + done + + blat email.txt -t ${recipients} -s "${CXNB_BASECC_PROJECT} Nightly Build results for ${CXNB_EMAIL_DATE}" ${attachList} + rm -f ${attachments} +fi + +exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_cfg.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_cfg.bat new file mode 100644 index 0000000..fc288c1 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_cfg.bat @@ -0,0 +1,117 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_cfg.bat +:: Purpose: Configuration file for CablexChange Nightly Build & Regression +:: Testing scripts +:: Arguments: %1 - CxC target to set the environment for +:: %2 - OS to set the environment for +::------------------------------------------------------------------------------ +set cxc_nbrt_cfg_target=%1 +set cxc_nbrt_cfg_os=%2 + +:: List of valid CxC app targets, separated by commas +set cxnb_cxc_target_list=bcm6345gw + +::------------------------------------------------------------------------------ +:: Default Target Build/Test configuration +::------------------------------------------------------------------------------ +:: prebuild - prebuild process (make deps, libraries, ...) +:: if prebuild=yes the following is also configurable +:: getsrc - get source from source control database +:: deps - make CM-side dependencies +:: resolv - build MTA resolver library +:: hoth - build hausware library +:: +:: build - Build the test images +:: if build=yes the following is also configurable +:: getsrc - get source from source control database +:: cmsym - build symbol table image +:: +:: test - Test the test images +:: if test=yes the following is also configurable +:: testlabel - cxlogs subfolder to grab the images from, defaults to CXNB_LABEL +:: (example: CXNB_CFG_TESTLABEL=03_05_07.15_25_56 will force the testing +:: portion to use images form cxlogs/03_05_07.15_25_56 for testing) +:: note: this is normaly used when CXNB_CFG_BUILD=no +:: abacus - Abacus based tests +:: pc - PacketCable test scripts (Tcl) +:: data - data only tests +:: datavoice - simultaneous data and voice tests +set CXNB_CFG_OUTPUTDIR= +set CXNB_CFG_PREBUILD=no +set CXNB_CFG_PREBUILD_GETSRC=yes +set CXNB_CFG_PREBUILD_DEPS=no +set CXNB_CFG_PREBUILD_RESOLV=no +set CXNB_CFG_PREBUILD_HOTH=no +set CXNB_CFG_BUILD=yes +set CXNB_CFG_BUILD_GETSRC=yes +set CXNB_CFG_BUILD_CMSYM=yes +set CXNB_CFG_TEST=no +set CXNB_CFG_TESTLABEL= +set CXNB_CFG_TEST_ABACUS=yes +set CXNB_CFG_TEST_PC=yes +set CXNB_CFG_TEST_DATA=yes +set CXNB_CFG_TEST_DATAVOICE=yes +set CXNB_CFG_SENDMAIL=yes +set CXNB_CFG_CC_CONFIGSPEC_UPDATE=yes + +:: empty target indicates to use default config settings +if "%cxc_nbrt_cfg_target%" == "" goto theend + +:: go to custom target configuration +for %%t in (%cxnb_cxc_target_list%) do if "%%t" == "%cxc_nbrt_cfg_target%" goto config_%%t + +echo "cxc_nbrt_cfg ERROR: Unsuported cxc target=%cxc_nbrt_cfg_target%!" +set errorlevel= +set errorlevel 2>NUL: +goto :eof + +::------------------------------------------------------------------------------ +:: Custom Target Build/Test configurations +:: (overrides default target configuration) +::------------------------------------------------------------------------------ +:config_bcm3348vcm +set CXNB_CFG_OUTPUTDIR=bcm93348_propane +set CXNB_CFG_PREBUILD=yes +set CXNB_CFG_PREBUILD_RESOLV=yes +set CXNB_CFG_PREBUILD_DEPS=no +set CXNB_CFG_TEST=no +goto theend + +:config_bcm3348vcm_euro +set CXNB_CFG_OUTPUTDIR=bcm93348_propane_eu +set CXNB_CFG_TEST=yes +goto theend + +:config_bcm3348vcm_bv16 +:config_bcm3348vcm_g723 +:config_bcm3348vcm_g726 +:config_bcm3348vcm_g729a +:config_bcm3348vcm_bv32 +set CXNB_CFG_OUTPUTDIR=bcm93348_propane +goto theend + +:config_bcm3351vcm +set CXNB_CFG_OUTPUTDIR=bcm93351_propane +set CXNB_CFG_PREBUILD_DEPS=yes +goto theend + +:config_bcm3351svl +set CXNB_CFG_OUTPUTDIR=bcm93351_propane +goto theend + +:config_bcm3352v_g723 +:config_bcm3352v +set CXNB_CFG_OUTPUTDIR=bcm93352_propane +set CXNB_CFG_PREBUILD_DEPS=yes +goto theend + +:theend +set cxc_nbrt_cfg_target= +set cxc_nbrt_cfg_os= + + + + + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_genlabel.sh b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_genlabel.sh new file mode 100644 index 0000000..51a79ba --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_genlabel.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxc_nbrt_genlabel.sh +# Purpose: Generates a set file for the CXNB_LABEL and CXNB_OUTPUT_DIR env. +# variables used by Nightly build and Regression testing scripts +#------------------------------------------------------------------------------ + +setFileName=cxc_nbrt_setlabel.bat + +#remove old setfile +rm -f ${setFileName} + +#Take snapshot of date and time to set up logging. +DATE=`date +"%y_%m_%d"` +TIME=`date +"%H_%M_%S"` +LABEL=${DATE}.${TIME}.${CXNB_LABEL_SUFFIX} +EMAIL_LABEL=`date +"%A, %B %e %Y, %T"` + +echo ":: ------------------------------------------------------------------------------" >>${setFileName} +echo ":: Broadcom Canada Ltd., Copyright 2001 - 2003" >>${setFileName} +echo ":: " >>${setFileName} +echo ":: Filename: cxc_nbrt_setlabel.bat" >>${setFileName} +echo ":: Purpose: Sets CXNB_LABEL and CXNB_OUTPUT_DIR environment variables" >>${setFileName} +echo ":: needed for the Nightly Build and Regression Testing scripts" >>${setFileName} +echo ":: NOTE: This is an automatically generated file." >>${setFileName} +echo ":: Do *NOT* edit manually!" >>${setFileName} +echo ":: ------------------------------------------------------------------------------" >>${setFileName} +echo " " >>${setFileName} +echo "set CXNB_LABEL=${LABEL}" >>${setFileName} +echo "set CXNB_EMAIL_DATE=${EMAIL_LABEL}" >>${setFileName} +echo "set CXNB_OUTPUT_DIR=${CXNB_BASECC_LOCAL_LOG_PATH}\\${LABEL}" >>${setFileName} + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_main.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_main.bat new file mode 100644 index 0000000..43f62fd --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_main.bat @@ -0,0 +1,37 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_main.bat +:: Purpose: Main script for CxC Nightly Build and Regression Testing +::------------------------------------------------------------------------------ + + +:: Generate a set file to setup logging and output folder +bash -C cxc_nbrt_genlabel.sh + +:: Run the set file to setup logging and output folder +call cxc_nbrt_setlabel.bat + +::Load the Default Config settings for CxC Nightly build/regression testing +call cxc_nbrt_cfg.bat + +:: Linux pre-build +bash -C lnx_nbrt_prepare.sh + +::Build: label&download source code from Source Control (CommEngine) +bash -C cxc_nb_ce_getsrc.sh 1 + +::Build: label&download source code from Source Control (dslx_common) +bash -C cxc_nb_getsrc.sh 1 + +::Build&test: make target apps and launch the tests +for %%t in (%cxnb_cxc_target_list%) do call cxc_nbrt_app.bat %%t linux + +:: Postprocess the nightly and regression test logs (CommEngine) +bash -C cxc_nbrt_ce_postprocess.sh linux + +:: Postprocess the nightly and regression test logs (dslx_common) +bash -C cxc_nbrt_postprocess.sh linux + +::exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_main_test.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_main_test.bat new file mode 100644 index 0000000..0a8831b --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_main_test.bat @@ -0,0 +1,12 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_main.bat +:: Purpose: Main script for CxC Nightly Build and Regression Testing +::------------------------------------------------------------------------------ + + +call cxc_nbrt_setenv.bat + +bash -C lnx_nbrt_test.sh 1 + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_postprocess.sh b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_postprocess.sh new file mode 100644 index 0000000..95b6e24 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_postprocess.sh @@ -0,0 +1,125 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxc_nbrt_postprocess.sh +# Purpose: Postprocess the nightly build and test logs +#------------------------------------------------------------------------------ + +if [ -z "${CXNB_LABEL}" ] +then + echo CXNB_LABEL not defined! + exit +fi + +cxcOsType=$1 + +export CXNB_LOG_PATH=$(cygpath -u "${CXNB_LOCAL_LOG_PATH}") +export CXNB_SRC_PATH=$(cygpath -u "${CXNB_LOCAL_SRC_PATH}") + +echo "Postprocessing at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +mkdir -p ${CXNB_LOG_PATH}/${CXNB_LABEL}/${cxcOsType} +cd ${CXNB_LOG_PATH}/${CXNB_LABEL}/${cxcOsType} + +mkdir -p buildlogs +# mkdir -p deps/buildlogs +# mkdir -p debug +# mv -f *.map debug +# mv -f *_sym.bin debug +# mv -f *.txt buildlogs +# mv -f deps/*.txt deps/buildlogs + +# cp -f -v ${CXNB_SRC_PATH}/cablex_tools/dev/callagent/callagent.exe ${CXNB_LOG_PATH}/${CXNB_LABEL}/ +# cp -f -v ${CXNB_SRC_PATH}/cablex_tools/dev/callagent/hhca.cfg ${CXNB_LOG_PATH}/${CXNB_LABEL}/ +cp -f -v ${CXNB_SRC_PATH}/update*.updt ${CXNB_LOG_PATH}/${CXNB_LABEL}/ +cp -f -v ${CXNB_LOG_PATH}/${CXNB_LABEL}/build*.log ${CXNB_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/buildlogs/ + + +if [ "${CXNB_CFG_SENDMAIL}" = "yes" ] +then + appendFiles= + attachments="build.log cc_config_spec.txt" + + if [ "${CXNB_CFG_BUILD_GETSRC}" = "yes" ] + then + appendFiles="${appendFiles} buildlogs/buildconfig.log" + fi + + echo $appendFiles + + recipients="vmarkovski@broadcom.com,jnicol@broadcom.com" + + echo "------------------------------------------------------------------------">email.txt + echo "CxC Nightly Build summary " >>email.txt + echo "------------------------------------------------------------------------">>email.txt + echo " " >>email.txt + # total=`grep -c "cxc build" buildlogs/build_summary.txt` + # fail=`grep -c FAILED buildlogs/build_summary.txt` + # pass=`grep -c successfull buildlogs/build_summary.txt` + # echo "Total of $total cxc apps built: $pass pass, $fail failed. " >>email.txt + # echo " " >>email.txt + # grep FAILED buildlogs/build_summary.txt >>email.txt + # echo " " >>email.txt + # grep successfull buildlogs/build_summary.txt >>email.txt + # echo " " >>email.txt + + echo "------------------------------------------------------------------------">>email.txt + echo "Nightly build auto-update activity " >>email.txt + echo "------------------------------------------------------------------------">>email.txt + echo " " >>email.txt + grep "CC Config Spec" ./../build.log >> email.txt || echo "No config spec update." >> email.txt + echo " " >>email.txt + + if [ -n "${appendFiles}" ] + then + for file in ${appendFiles} + do + if [ -e "${file}" ] + then + cat ${file} >> email.txt + else + echo "POSTPROCESSING ERROR: could not find ${file}!" >>email.txt + fi + done + fi + echo " " >>email.txt + + if [ -e "./../cc_config_spec.old" ] + then + attachments="${attachments} cc_config_spec.old" + fi + + echo "------------------------------------------------------------------------">>email.txt + echo "Nightly build images, logs and source code for ${CXNB_EMAIL_DATE}:" >> email.txt + echo "------------------------------------------------------------------------">>email.txt + echo "Last night's nightly build source code can be found at:" >> email.txt + echo $(cygpath -w //${CXNB_LOCAL_SRC_PATH}) >> email.txt + echo "Nightly build images can be found at" >> email.txt + echo $(cygpath -w //${CXNB_BASECC_LOCAL_SRC_PATH}/images) >> email.txt + # echo "Nightly build map files and symbol table images can be found at" >> email.txt + # echo $(cygpath -w //${UNIX_TESTCOMPUTER}/cxnb/cxlogs/${CXNB_UCMCC_PROJECT}/${CXNB_LABEL}/${cxcOsType}/debug) >> email.txt + echo "Nightly build logs can be found at" >> email.txt + echo $(cygpath -w //${CXNB_LOCAL_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/buildlogs) >> email.txt + # echo $(cygpath -w //${CXNB_LOCAL_LOG_PATH}/${CXNB_LABEL}/${cxcOsType}/deps/buildlogs) >> email.txt + + #note: blat doesnt work from UNC directories so copy everything to a local directory + for file in ${attachments} + do + cp -f ./../${file} . + done + + attachList="" + for file in ${attachments} + do + attachList="${attachList} -attacht ${file}" + done + + blat email.txt -t ${recipients} -s "${CXNB_UCMCC_PROJECT} Nightly Build results for ${CXNB_EMAIL_DATE}" ${attachList} + rm -f ${attachments} +fi + +echo "Finished postprocessing at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_prebuild.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_prebuild.bat new file mode 100644 index 0000000..fd7e6a1 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_prebuild.bat @@ -0,0 +1,140 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_prebuild.bat +:: Purpose: CxC Nightly pre-build script (makes dependancies and libraries) +:: Arguments: %1 - target CxC app to pre-build +:: %2 - target OS to pre-build the app for +::------------------------------------------------------------------------------ + +if "%2" == "" goto argsError + +set cxnb_preapp_target=%1 +set cxnb_preapp_os=%2 + +:: Setup nbrt environment variables specific for this target +call cxc_nbrt_cfg.bat %cxnb_preapp_target% %cxnb_preapp_os% + +if "%CXNB_CFG_PREBUILD%" == "no" goto skipprebuild + +set cxnb_prebuild_output_dir=%CXNB_OUTPUT_DIR%\%cxnb_preapp_os%\deps +mkdir %cxnb_prebuild_output_dir% + +:: Set the CxC build environment +cd /D %CXNB_LOCAL_SRC_PATH%\cablex +call gen.bat %cxnb_preapp_os% %CXNB_TORNADO_PATH% +call set_%cxnb_preapp_os%.bat + +if "%CXNB_CFG_PREBUILD_DEPS%" == "no" goto skipdeps + +:: Check-out target dependancies +cd /D %CXNB_LOCAL_SRC_PATH%\cablex\src\cm_v2\CMAPP_DOCSIS1.0\%cxnb_preapp_os% +cd /D %CXNB_CFG_OUTPUTDIR% +copy /Y makefile.deps makefile.old +del /F makefile.deps + +:: *** Make the CM deps for CxC app *** +cd /D %CXNB_LOCAL_SRC_PATH%\cablex\apps\%cxnb_preapp_target% +echo Started dependancy make for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log +make cm_deps >%cxnb_prebuild_output_dir%\%cxnb_preapp_target%_propane_deps.txt 2>&1 +echo Completed dependancy make for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log + +:: Save and check-in the new target dependancies +cd /D %CXNB_LOCAL_SRC_PATH%\cablex\src\cm_v2\CMAPP_DOCSIS1.0\%cxnb_preapp_os%\%CXNB_CFG_OUTPUTDIR% +copy /Y makefile.deps %cxnb_prebuild_output_dir%\makefile_%cxnb_preapp_target%_propane.deps +if not exist makefile.old goto skipDepsCheckin +if not exist makefile.deps goto skipDepsCheckin + +:: Assume the dependencies have not changed +set cxnb_makefile_deps_different=no + +:: Compare the new deps with the old ones, if they differ set the flag +cleardiff -status_only -blank_ignore makefile.deps makefile.old || set cxnb_makefile_deps_different=yes + +:: Checkin the new deps if its different +if "%cxnb_makefile_deps_different%" == "no" goto skipDepsCheckin + +:: Checkin the new CxC app dependencies +mkdir %CXNB_LOCAL_VIEW_PATH%\cablex\src\cm_v2\CMAPP_DOCSIS1.0\%cxnb_preapp_os%\%CXNB_CFG_OUTPUTDIR% +cd /D %CXNB_LOCAL_VIEW_PATH%\cablex\src\cm_v2\CMAPP_DOCSIS1.0\%cxnb_preapp_os%\%CXNB_CFG_OUTPUTDIR% +cleartool setactivity -nc Nightly_CXNB_%CXNB_UCMCC_PROJECT%_Updates +cleartool checkout -nc -nquery makefile.deps +cp -f %CXNB_LOCAL_SRC_PATH%\cablex\src\cm_v2\CMAPP_DOCSIS1.0\%cxnb_preapp_os%\%CXNB_CFG_OUTPUTDIR%\makefile.deps . +cleartool checkin -nc makefile.deps + +:: Set increment baseline flag +set CXNB_UPDATE_BASELINE="yes" +:skipDepsCheckin + +:: Post-process dependancies +cd /D %cxnb_prebuild_output_dir% +bash -C cxwarnerrorgrep.sh %cxnb_preapp_target%_propane_deps makefile_%cxnb_preapp_target%_propane.deps +goto resolv + +:skipdeps +echo Skiping dependancy build for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log + +:resolv +if "%CXNB_CFG_PREBUILD_RESOLV%" == "no" goto skipresolv + +:: Check-out resolver library +cd /D %CXNB_LOCAL_SRC_PATH%\cablex\lib\%cxnb_preapp_os% +copy /Y libresolv.a libresolv.old + +:: Build the resolver library +cd /D %CXNB_LOCAL_SRC_PATH%\cablex\apps\%cxnb_preapp_target% +make clean_resolv >%cxnb_prebuild_output_dir%\mta_resolv.txt 2>&1 +echo Started resolver library make for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log +make resolv >>%cxnb_prebuild_output_dir%\mta_resolv.txt 2>&1 +echo Completed resolver library make for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log + +:: Save and check-in the new resolver library +cd /D %CXNB_LOCAL_SRC_PATH%\cablex\lib\%cxnb_preapp_os% +copy /Y libresolv.a %cxnb_prebuild_output_dir%\libresolv.a +if not exist libresolv.a goto skipResolvCheckin +if not exist libresolv.old goto skipResolvCheckin + +:: Assume the library has not changed +set cxnb_resolv_different=no + +:: Compare the new lib with the old one, if they differ set the flag +rawcmp.exe libresolv.a libresolv.old || set cxnb_resolv_different=yes + +:: Checkin the new lib if its different +if "%cxnb_resolv_different%" == "no" goto skipResolvCheckin + +:: Checkin the new resolver library +mkdir %CXNB_LOCAL_VIEW_PATH%\cablex\lib\VxWorks +cd /D %CXNB_LOCAL_VIEW_PATH%\cablex\lib\VxWorks +cleartool setactivity -nc Nightly_CXNB_%CXNB_UCMCC_PROJECT%_Updates +cleartool checkout -nc -nquery libresolv.a +cp -f %CXNB_LOCAL_SRC_PATH%\cablex\lib\%cxnb_preapp_os%\libresolv.a . +cleartool checkin -nc libresolv.a + +:: Set increment baseline flag +set CXNB_UPDATE_BASELINE="yes" +:skipResolvCheckin + +:: Post-process resolver build +cd /D %cxnb_prebuild_output_dir% +bash -C cxwarnerrorgrep.sh mta_resolv libresolv.a +goto theend + +:argsError +echo cxc_nbrt_prebuild error: Insufficient arguments arg1=%1; arg2=%2; >>%CXNB_OUTPUT_DIR%\build.log +goto theend + +:skipprebuild +echo Skiping prebuild for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log +goto theend + +:skipresolv +echo Skiping resolver build for %cxnb_preapp_target% %cxnb_preapp_os% >>%CXNB_OUTPUT_DIR%\build.log +goto theend + +:theend +set cxnb_preapp_target= +set cxnb_preapp_os= +set cxnb_prebuild_output_dir= +set cxnb_resolv_different= +exit diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_setenv.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_setenv.bat new file mode 100644 index 0000000..6a3fab2 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_setenv.bat @@ -0,0 +1,65 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_setenv.bat +:: Purpose: Setting up of environment variables used in the other +:: batch files and shell scripts. +:: +:: NOTE: This is probably the only file that you will need +:: to modify to enable nightly builds. +::------------------------------------------------------------------------------ + +set CXNB_UCMCC_PROJECT=dslx_common +set CXNB_BASECC_PROJECT=CommEngine +set CXNB_LABEL_SUFFIX=dslx + +:: CUSTOMIZE +:: Enter the name or the IP address of the nightly build computer, +:: the username and password, +:: and the root password for the test computer +SET UNIX_TESTCOMPUTER=ip_address_of_the_unix_box +set UNIX_USERNAME=username_for_the_unix_box +set UNIX_USERPASS=password_for_the_unix_box +set UNIX_ROOTPASS=root_password_for_the_unix_box + +:: CUSTOMIZE +:: Enter the root of the CommEngine view (one level above CommEngine directory) in MSDOS style +set CXNB_BASECC_VIEW_ROOT=\\%UNIX_TESTCOMPUTER%\vmarkovski\views\nb_view + +:: CUSTOMIZE +:: Enter the path to the nightly build tools (blat, psexec, etc.) +set CXNB_TOOLS_PATH=c:\nb\tools + +:: CUSTOMIZE +:: Enter the path of the directory from where +:: the nightly build scripts are launched +set CXNB_RUN_PATH=V:\views\irvine_view\%CXNB_BASECC_PROJECT%\hostTools\scripts\nightlybuild\voice + +:: CUSTOMIZE +:: Make sure bash, build scripts and ldx_tools are in the PATH +set PATH=c:\cygwin\bin;%PATH%;%CXNB_RUN_PATH%;%CXNB_LDX_TOOLS_PATH%;%CXNB_TOOLS_PATH% + +:: CUSTOMIZE +:: Enter the root of the CommEngine view (one level above CommEngine directory) in UNIX style +set UNIX_CXNB_BASECC_VIEW_ROOT=/home/vmarkovski/views/nb_view + +set UNIX_CXNB_BASECC_LOCAL_VIEW_PATH=%UNIX_CXNB_BASECC_VIEW_ROOT%/%CXNB_BASECC_PROJECT% +set UNIX_CXNB_BASECC_LOCAL_SRC_PATH=%UNIX_CXNB_BASECC_VIEW_ROOT%/%CXNB_BASECC_PROJECT%_build + +:: Variables for base ClearCase projects +set CXNB_BASECC_LOCAL_VIEW_PATH=%CXNB_BASECC_VIEW_ROOT%\%CXNB_BASECC_PROJECT% +set CXNB_BASECC_LOCAL_SRC_PATH=%CXNB_BASECC_VIEW_ROOT%\%CXNB_BASECC_PROJECT%_build +set CXNB_BASECC_LRULES_PATH=%CXNB_RUN_PATH%\ccLoadRules\%CXNB_BASECC_PROJECT% +set CXNB_BASECC_LOCAL_LOG_PATH=%CXNB_BASECC_VIEW_ROOT%\log\%CXNB_BASECC_PROJECT% + +:: Variables for UCM ClearCase projects +:: CUSTOMIZE +:: For the variable CXNB_LOCAL_VIEW_PATH, enter to root of the UCM view +set CXNB_LOCAL_VIEW_PATH=%CXNB_BASECC_LOCAL_VIEW_PATH%\xChange\rmna_nb_%CXNB_UCMCC_PROJECT%_integration_sv +set CXNB_LOCAL_SRC_PATH=%CXNB_BASECC_LOCAL_SRC_PATH%\xChange\%CXNB_UCMCC_PROJECT% +set CXNB_CC_LRULES_PATH=%CXNB_RUN_PATH%\ccLoadRules\%CXNB_UCMCC_PROJECT% +set CXNB_LOCAL_LOG_PATH=%CXNB_BASECC_VIEW_ROOT%\log\%CXNB_UCMCC_PROJECT% + +:: Initialize baseline update flag to no +set CXNB_UPDATE_BASELINE="no" + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_start.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_start.bat new file mode 100644 index 0000000..11af566 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_start.bat @@ -0,0 +1,28 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_start.bat +:: Purpose: Startup script for CxC Nightly Build and Regression Testing +::------------------------------------------------------------------------------ + +echo ON + +::Set the environment for CxC Nightly build/regression testing +call cxc_nbrt_setenv.bat + +mkdir %CXNB_BASECC_VIEW_ROOT%\log +set CXNB_MAIN_LOGFILE=%CXNB_BASECC_VIEW_ROOT%\log\cxc_nbrt_main.log + +echo "cxc_nbrt_start.bat started" > %CXNB_MAIN_LOGFILE% +echo. >> %CXNB_MAIN_LOGFILE% +call cxc_nbrt_main.bat >> %CXNB_MAIN_LOGFILE% 2>&1 +echo ON +echo. >> %CXNB_MAIN_LOGFILE% +echo "cxc_nbrt_start.bat finished" >> %CXNB_MAIN_LOGFILE% + +echo "" +echo "UPDATE & BUILD FINISHED !!!" +echo "" +sleep 10 + +exit \ No newline at end of file diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_test.bat b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_test.bat new file mode 100644 index 0000000..9a558fb --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_test.bat @@ -0,0 +1,62 @@ +::------------------------------------------------------------------------------ +:: Broadcom Canada Ltd., Copyright 2001 - 2003 +:: +:: Filename: cxc_nbrt_test.bat +:: Purpose: Entry point for CxC Nightly Regression Testing +:: Arguments: %1 - target platform to run the tests on +:: %2 - target OS to use for testing +::------------------------------------------------------------------------------ + +if "%1" == "" goto argsError +if "%2" == "" goto argsError + +set cxrt_test_target=%1 +set cxrt_test_os=%2 + +if "%cxrt_test_os%" == "vxworks" goto vxworks +echo "CXC_NBRT_TEST ERROR: Unsupported OS=%cxrt_test_os%" +goto theend + +:vxworks + +:: Setup nbrt environment variables specific for this target +call cxc_nbrt_cfg.bat %cxrt_test_target% %cxrt_test_os% + +if "%CXNB_CFG_TEST%" == "no" goto theend + +set TEST_SRV_COMPUTER_NAME=LBRMNA-CEXCH02 + +del /F /Q \\%TEST_SRV_COMPUTER_NAME%\cxnt\incoming\* + +:: Check if there is an image for this platform ready after the build + +if "%CXNB_CFG_TESTLABEL%" == "" set CXNB_CFG_TESTLABEL=%CXNB_LABEL% + +if exist %CXNB_LOCAL_LOG_PATH%\%CXNB_CFG_TESTLABEL%\%cxrt_test_os%\vxram_sto_%cxrt_test_target%.bin goto copyToTestServer +echo "CXC_NBRT_TEST ERROR: No image %CXNB_OUTPUT_DIR%\%cxrt_test_os%\vxram_sto_%cxrt_test_target%.bin. Could not run Regression tests!" +goto theend + +:copyToTestServer + +:: copy the test image +copy /Y %CXNB_LOCAL_LOG_PATH%\%CXNB_CFG_TESTLABEL%\%cxrt_test_os%\vxram_sto_%cxrt_test_target%.bin \\%TEST_SRV_COMPUTER_NAME%\cxnt\incoming + +:: create and copy the configuration file for regression testing +more cxc_nbrt_cfg.bat >cxc_rt_cfg.bat +more cxc_nbrt_setlabel.bat >>cxc_rt_cfg.bat +copy /Y cxc_rt_cfg.bat \\%TEST_SRV_COMPUTER_NAME%\cxnt\incoming + +:: spawn the test on the remote test server +psexec \\%TEST_SRV_COMPUTER_NAME% -s -i c:\cxnt\spawnTestServer.bat %cxrt_test_target% vxram_sto_%cxrt_test_target%.bin + +goto theend + +:argsError +echo cxc_nbrt_test error: Insufficient arguments arg1=%1; arg2=%2; +goto theend + +:theend +set cxrt_test_target= +set cxrt_test_os= +exit + diff --git a/hostTools/scripts/nightlybuild/voice/cxc_nbrt_updatebaseline.sh b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_updatebaseline.sh new file mode 100644 index 0000000..c50e72b --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxc_nbrt_updatebaseline.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxc_nbrt_updatebaseline.sh +# Purpose: Postprocess the nightly build and test logs +#------------------------------------------------------------------------------ + +#make the date and time look like a CCDelivery +CCDATE=`date +"%Y_%m_%d_%H%M_%S.0000"` + + +# Update the baselines if necessary + cd ${CXNB_LOCAL_VIEW_PATH} + cleartool mkbl -all -inc "BL_INC_nightlybuild_${CXNB_UCMCC_PROJECT}_${CCDATE}" + +# Exit this shell +exit diff --git a/hostTools/scripts/nightlybuild/voice/cxwarnerrorgrep.sh b/hostTools/scripts/nightlybuild/voice/cxwarnerrorgrep.sh new file mode 100644 index 0000000..82913e1 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/cxwarnerrorgrep.sh @@ -0,0 +1,37 @@ +#/bin/bash +#------------------------------------------------------------------------------ +# Broadcom Canada Ltd., Copyright 2001-2003 +# +# Filename: cxwarnerrorgrep.sh +# Purpose: Grep for errors and warnings in the build logs +# Arguments: $1 - CxC build target log file to parse +# $2 - File resulted from a build process (library, dependancy file, +# target image, ...). Indicates success or failure of the build +#------------------------------------------------------------------------------ + + +export CXNB_LOG_PATH=$(cygpath -u "${CXNB_LOCAL_LOG_PATH}") +echo "Started error warning grep for $1 $2 at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +if [ -e "$1.txt" ] +then + export CXNB_LDX_TOOLS_PATH_U=$(cygpath -u "${CXNB_LDX_TOOLS_PATH}") + mkdir -p buildlogs + echo "***************************************************************" >buildlogs/warn_err_$1.txt + echo "CablexChange Warnings and Errors for $1 " >>buildlogs/warn_err_$1.txt + echo "***************************************************************" >>buildlogs/warn_err_$1.txt + echo >>buildlogs/warn_err_$1.txt + gawk -f ${CXNB_LDX_TOOLS_PATH_U}/log.awk $1.txt >>buildlogs/warn_err_$1.txt + if [ -e "$2" ] + then + echo "$1 cxc build successfull." >>buildlogs/build_summary.txt + else + echo "$1 cxc build FAILED !!!!!" >>buildlogs/build_summary.txt + fi +else + echo "cxwarnerrorgrep error: $1.txt does not exits" >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log +fi + +echo "Completed error warning grep for $1 $2 at: " `date +"%T"` >>${CXNB_LOG_PATH}/${CXNB_LABEL}/build.log + +exit diff --git a/hostTools/scripts/nightlybuild/voice/lnx_nbrt_basic_cmds.exp b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_basic_cmds.exp new file mode 100644 index 0000000..c67b74e --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_basic_cmds.exp @@ -0,0 +1,115 @@ +# +# Login to a Linux box +# +proc lnxLogin {ip username pass} { + + send_user "Login to Linux : $ip\n" + set timeout 20 + spawn ssh -l $username $ip + global ssh_spawn_id + set ssh_spawn_id $spawn_id + expect { + "password:" {} + "refused" { + send_user "\n Cannot connect to host $ip !!! \n" + exit 1 + } + timeout { + send_user "\n Timeout while connecting to host $ip !!! \n" + exit 1 + } + } + + send "$pass\r" + expect { + "$username@*\\$" {} + "denied" { + send_user "\n Password not correct for user $username !!! \n" + exit 1 + } + timeout { + send_user "\n Timeout while waiting for user passord !!! \n" + exit 1 + } + } + + send_user "Login Complete\n" + return 0 +} + + +# +# Switch to superuser (user already logged in) +# +proc lnxSu {pass} { + global ssh_spawn_id + set spawn_id $ssh_spawn_id + send_user "\n Switching to superuser\n" + + send "su\r" + expect { + "Password:" {} + timeout { + send_user "\n Timeout. Couldn't switch to superuser !!! \n" + exit 1 + } + } + + send "$pass\r" + expect { + "root@" {} + "incorrect password" { + send_user "\n Invalid password for superuser !!! \n" + exit 1 + } + timeout { + send_user "\n Timeout. Couldn't switch to superuser !!! \n" + exit 1 + } + } + + send_user "\n Switched to superuser\n" +} + + +# +# Generic Linux command +# parameter 1: the actual command +# parameter 2: timeout value for the command (default = 10 sec) +# +proc lnxCmd {cmd timeoutvalue} { + global ssh_spawn_id + set spawn_id $ssh_spawn_id + set timeout $timeoutvalue + + send_user "\n \n $cmd \n" + + send "$cmd \r" + send "if \[ $? -ne 0 \]; then echo 'cmd''failure'; else echo 'cmd''success';fi \r" + + if {$timeout < 20} { + set timeout 20 + } + send_user "Timeout for the command \"$cmd\" set to $timeout sec\n" + expect { + "cmdfailure" { + send_user "\n The Linux command \"$cmd\" has failed !!!\n" + } + "cmdsuccess" { + send_user "\n The Linux command \"$cmd\" has succeeded \n" + } + timeout { + send_user "\n Timeout for Linux command \"$cmd\" \n" + exit 1 + } + } +} + + +proc lnxTest {params} { + set test [lindex $params 1] + puts $test + puts [exec ls] +} + +#lnxTest "ab cd de ef" diff --git a/hostTools/scripts/nightlybuild/voice/lnx_nbrt_build.sh b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_build.sh new file mode 100644 index 0000000..5127091 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +expect -c "set EXP_BASECC_PATH $UNIX_CXNB_BASECC_LOCAL_VIEW_PATH; \ + set EXP_SRC_PATH $UNIX_CXNB_BASECC_LOCAL_SRC_PATH; \ + set EXP_USERNAME $UNIX_USERNAME; \ + set EXP_USERPASS $UNIX_USERPASS; \ + set EXP_ROOTPASS $UNIX_ROOTPASS; \ + set EXP_TESTCOMPUTER $UNIX_TESTCOMPUTER" \ + -f lnx_nbrt_view_build.exp \ No newline at end of file diff --git a/hostTools/scripts/nightlybuild/voice/lnx_nbrt_prepare.sh b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_prepare.sh new file mode 100644 index 0000000..14df0af --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_prepare.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +expect -c "set EXP_BASECC_PATH $UNIX_CXNB_BASECC_LOCAL_VIEW_PATH; \ + set EXP_SRC_PATH $UNIX_CXNB_BASECC_LOCAL_SRC_PATH; \ + set EXP_USERNAME $UNIX_USERNAME; \ + set EXP_USERPASS $UNIX_USERPASS; \ + set EXP_ROOTPASS $UNIX_ROOTPASS; \ + set EXP_TESTCOMPUTER $UNIX_TESTCOMPUTER" \ + -f lnx_nbrt_view_prepare.exp \ No newline at end of file diff --git a/hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_build.exp b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_build.exp new file mode 100644 index 0000000..aa7d7ce --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_build.exp @@ -0,0 +1,53 @@ +source lnx_nbrt_basic_cmds.exp + +lnxLogin $EXP_TESTCOMPUTER $EXP_USERNAME $EXP_USERPASS +lnxCmd "cd ${EXP_SRC_PATH}" 0 +lnxSu $EXP_ROOTPASS + +# NOTE: +# The following make commands assume that the default build in the profile is G729. +# Therefore, we don't setup the values for all the vocoder variables when we build +# a vocoder-specific build. For non-G729 loads, we only need to set the G729 +# vocoder variable to 'n' (i.e. reset the default G729 variable) +# and set the appropriate vocoder variable of choice. + +# MGCP, G723 load +lnxCmd "make PROFILE=96345GWV clean" 180 +lnxCmd "make PROFILE=96345GWV BRCM_APP_PHONE=mgcp BRCM_DSP_CODEC_G729=n BRCM_DSP_CODEC_G723=y" 3600 + +# MGCP, G729 load +lnxCmd "make PROFILE=96345GWV clean" 180 +lnxCmd "make PROFILE=96345GWV BRCM_APP_PHONE=mgcp BRCM_DSP_CODEC_G729=y" 3600 + +# MGCP, extended SRAM load +lnxCmd "make PROFILE=96345GWV clean" 180 +lnxCmd "make PROFILE=96345GWV BRCM_APP_PHONE=mgcp BRCM_DSP_CODEC_G729=n BRCM_DSP_CODEC_G7xx=y" 3600 + +# MGCP, IOM2 load +lnxCmd "make PROFILE=96345GWV clean" 180 +lnxCmd "make PROFILE=96345GWV BRCM_APP_PHONE=mgcp BRCM_DSP_CODEC_G729=n BRCM_DSP_IOM2=y" 3600 + +# SIP, G726 load +lnxCmd "make PROFILE=96348GWV clean" 180 +lnxCmd "make PROFILE=96348GWV BRCM_APP_PHONE=sip BRCM_DSP_CODEC_G729=n BRCM_DSP_CODEC_G726=y" 3600 + +# SIP, G729 load +lnxCmd "make PROFILE=96348GWV clean" 180 +lnxCmd "make PROFILE=96348GWV BRCM_APP_PHONE=sip BRCM_DSP_CODEC_G729=y" 3600 + +# SIP, extended SRAM load +lnxCmd "make PROFILE=96348GWV clean" 180 +lnxCmd "make PROFILE=96348GWV BRCM_APP_PHONE=sip BRCM_DSP_CODEC_G729=n BRCM_DSP_CODEC_G7xx=y" 3600 + +# SIP, extended SRAM load (Japan) +lnxCmd "make PROFILE=96348GWV clean" 180 +lnxCmd "make PROFILE=96348GWV BRCM_APP_PHONE=sip BRCM_DSP_CODEC_G729=n BRCM_DSP_CODEC_G7xx=y BRCM_VOICE_COUNTRY_JAPAN=y" 3600 + +# SIP, IOM2 load +lnxCmd "make PROFILE=96348GWV clean" 180 +lnxCmd "make PROFILE=96348GWV BRCM_APP_PHONE=sip BRCM_DSP_CODEC_G729=n BRCM_DSP_IOM2=y" 3600 + +lnxCmd "exit" 0 +lnxCmd "exit" 0 + + diff --git a/hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_prepare.exp b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_prepare.exp new file mode 100644 index 0000000..15d1047 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/lnx_nbrt_view_prepare.exp @@ -0,0 +1,31 @@ +source lnx_nbrt_basic_cmds.exp + +lnxLogin $EXP_TESTCOMPUTER $EXP_USERNAME $EXP_USERPASS +lnxSu $EXP_ROOTPASS + +lnxCmd "rm -rf ${EXP_BASECC_PATH}/cfe" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/docs" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/hostTools" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/images" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/release" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/sgibcm_2_4_17" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/targets" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/userapps" 30 +lnxCmd "rm -rf ${EXP_BASECC_PATH}/xChange/dslx" 30 + + +lnxCmd "rm -rf ${EXP_SRC_PATH}/cfe" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/docs" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/hostTools" 30 +# Don't delete the created images +# lnxCmd "rm -rf ${EXP_SRC_PATH}/images" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/release" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/sgibcm_2_4_17" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/targets" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/userapps" 30 +lnxCmd "rm -rf ${EXP_SRC_PATH}/xChange" 30 + +lnxCmd "exit" 0 +lnxCmd "exit" 0 + + diff --git a/hostTools/scripts/nightlybuild/voice/readme.txt b/hostTools/scripts/nightlybuild/voice/readme.txt new file mode 100644 index 0000000..74b7ec1 --- /dev/null +++ b/hostTools/scripts/nightlybuild/voice/readme.txt @@ -0,0 +1,19 @@ +This directory contains scripts for enabling nightly builds. + +The following prerequisites are needed for enabling nightly builds: +- one Windows and one Linux PC +- the Windows PC should have ClearCase client +- the Windows PC should have cygwin with support for bash shell and expect scripts +- the user can login to the Linux PC using ssh +- the ClearCase bin directory should be in the cygwin bash path +- base ClearCase view (CommEngine) should be created on the Linux PC and UCM ClearCase +view (dslx_common) should be created in the CommEngine/xChange directory + +The script initiation is performed from a Windows PC, usually through the Windows task scheduler. The batch file cxc_nbrt_start.bat starts the overall build process. + +IMPORTANT: +- the user should copy the content of this directory to a directory outside of the designated nightly build view +- the content of the file cxc_nbrt_setenv.bat should be modified to correspond to the user settings. The user should modify all the variables preceded with the comment "CUSTOMIZE". + + + diff --git a/hostTools/squashfs/Makefile b/hostTools/squashfs/Makefile new file mode 100644 index 0000000..c8472c1 --- /dev/null +++ b/hostTools/squashfs/Makefile @@ -0,0 +1,12 @@ +INCLUDEDIR = . + +CFLAGS := -I$(INCLUDEDIR) -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -O2 + +mksquashfs: mksquashfs.o read_fs.o sort.o + $(CC) mksquashfs.o read_fs.o sort.o -lz -o $@ + +mksquashfs.o: mksquashfs.c mksquashfs.h + +read_fs.o: read_fs.c read_fs.h + +sort.o: sort.c diff --git a/hostTools/squashfs/mksquashfs.c b/hostTools/squashfs/mksquashfs.c new file mode 100644 index 0000000..e327a01 --- /dev/null +++ b/hostTools/squashfs/mksquashfs.c @@ -0,0 +1,1971 @@ +/* + * Create a squashfs filesystem. This is a highly compressed read only filesystem. + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * mksquashfs.c + */ + +#define TRUE 1 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mksquashfs.h" +#include +/* BRCM begin */ +#include "7zapi.h" +/* BRCM end */ + +#ifdef SQUASHFS_TRACE +#define TRACE(s, args...) printf("mksquashfs: "s, ## args) +#else +#define TRACE(s, args...) +#endif + +#define INFO(s, args...) do { if(!silent) printf("mksquashfs: "s, ## args); } while(0) +#define ERROR(s, args...) do { fprintf(stderr, s, ## args); } while(0) +#define EXIT_MKSQUASHFS() do { if(restore)\ + restorefs();\ + exit(1); } while(0) +#define BAD_ERROR(s, args...) do {\ + fprintf(stderr, "FATAL ERROR:" s, ##args);\ + EXIT_MKSQUASHFS();\ + } while(0) + +int duplicate_checking = 1, noF = 0, no_fragments = 0, always_use_fragments = 0; +int total_compressed = 0, total_uncompressed = 0; + +int fd; + +/* superblock attributes */ +int noI = 0, noD = 0, check_data = 0, block_size = SQUASHFS_FILE_SIZE, block_log; +unsigned short uid_count = 0, guid_count = 0; +squashfs_uid uids[SQUASHFS_UIDS], guids[SQUASHFS_GUIDS]; +int block_offset; + +/* write position within data section */ +unsigned int bytes = 0, total_bytes = 0; + +/* in memory directory table - possibly compressed */ +char *directory_table = NULL; +unsigned int directory_bytes = 0, directory_size = 0, total_directory_bytes = 0; + +/* cached directory table */ +char *directory_data_cache = NULL; +unsigned int directory_cache_bytes = 0, directory_cache_size = 0; + +/* in memory inode table - possibly compressed */ +char *inode_table = NULL; +unsigned int inode_bytes = 0, inode_size = 0, total_inode_bytes = 0; + +/* cached inode table */ +char *data_cache = NULL; +unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; + +/* in memory directory header */ +struct directory { + unsigned int start_block; + unsigned int size; + unsigned char *buff; + unsigned char *p; + unsigned int entry_count; + squashfs_dir_header *entry_count_p; +}; + +struct file_info *dupl[65536], *frag_dups[65536]; +int dup_files = 0; + +int swap, silent = TRUE; +int file_count = 0, sym_count = 0, dev_count = 0, dir_count = 0, fifo_count = 0, sock_count = 0; + +/* list of exclude dirs/files */ +struct exclude_info { + dev_t st_dev; + ino_t st_ino; +}; + +#define EXCLUDE_SIZE 8192 +int exclude = 0; +struct exclude_info *exclude_paths = NULL; +int excluded(char *filename, struct stat *buf); + +/* fragment block data structures */ +int fragments = 0; +static char *fragment_data; +static int fragment_size = 0; +struct fragment { + unsigned int index; + int offset; + int size; +}; + + +#define FRAG_SIZE 32768 +squashfs_fragment_entry *fragment_table = NULL; + +/* list of source dirs/files */ +int source = 0; +char **source_path; + +/* list of root directory entries read from original filesystem */ +int old_root_entries = 0; +struct old_root_entry_info { + char name[SQUASHFS_NAME_LEN + 1]; + squashfs_inode inode; + int type; +}; + +/* in memory file info */ +struct file_info { + unsigned int bytes; + unsigned short checksum; + unsigned int start; + unsigned int *block_list; + struct file_info *next; + struct fragment *fragment; + unsigned short fragment_checksum; +}; + +/* count of how many times SIGINT or SIGQUIT has been sent */ +int interrupted = 0; + +/* restore orignal filesystem state if appending to existing filesystem is cancelled */ +jmp_buf env; +char *sdata_cache, *sdirectory_data_cache; +unsigned int sbytes, sinode_bytes, scache_bytes, sdirectory_bytes, + sdirectory_cache_bytes, suid_count, sguid_count, + stotal_bytes, stotal_inode_bytes, stotal_directory_bytes, + sinode_count, sfile_count, ssym_count, sdev_count, + sdir_count, sdup_files; +int sfragments; +int restore = 0; + +/*flag whether destination file is a block device */ +int block_device = 0; + +/* flag indicating whether files are sorted using sort list(s) */ +int sorted = 0; + +long long global_uid = -1, global_gid = -1; + +/* structure to used to pass in a pointer or an integer + * to duplicate buffer read helper functions. + */ +struct duplicate_buffer_handle { + unsigned char *ptr; + unsigned int start; +}; + +struct old_root_entry_info *old_root_entry; +void add_old_root_entry(char *name, squashfs_inode inode, int type); +extern int read_super(int fd, squashfs_super_block *sBlk, int *be, char *source); +extern int read_filesystem(char *root_name, int fd, squashfs_super_block *sBlk, char **inode_table, int *inode_bytes, + char **data_cache, int *cache_bytes, int *cache_size, char **directory_table, int *directory_bytes, + char **directory_data_cache, int *directory_cache_bytes, int *directory_cache_size, + int *file_count, int *sym_count, int *dev_count, int *dir_count, int *fifo_count, int *sock_count, + squashfs_uid *uids, unsigned short *uid_count, squashfs_uid *guids, unsigned short *guid_count, + unsigned int *uncompressed_file, unsigned int *uncompressed_inode, unsigned int *uncompressed_directory, + void (push_directory_entry)(char *, squashfs_inode, int), squashfs_fragment_entry **fragment_table); +squashfs_inode get_sorted_inode(struct stat *buf); +int read_sort_file(char *filename, int source, char *source_path[]); +void sort_files_and_write(int source, char *source_path[]); +struct file_info *duplicate(unsigned char *(get_next_file_block)(struct duplicate_buffer_handle *, unsigned int), struct duplicate_buffer_handle *file_start, int bytes, unsigned int **block_list, int *start, int blocks, struct fragment **fragment, char *frag_data, int frag_bytes); + +#define FALSE 0 + +#define MKINODE(A) ((squashfs_inode)(((squashfs_inode) inode_bytes << 16) + (((char *)A) - data_cache))) + +/* BRCM begin */ +#define LZMA 0 +#define GZIP 1 +int compress_algorithm = LZMA; +/* BRCM end */ + +void restorefs() +{ + ERROR("Exiting - restoring original filesystem!\n\n"); + bytes = sbytes; + memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes); + memcpy(directory_data_cache, sdirectory_data_cache, directory_cache_bytes = sdirectory_cache_bytes); + inode_bytes = sinode_bytes; + directory_bytes = sdirectory_bytes; + uid_count = suid_count; + guid_count = sguid_count; + total_bytes = stotal_bytes; + total_inode_bytes = stotal_inode_bytes; + total_directory_bytes = stotal_directory_bytes; + inode_count = sinode_count; + file_count = sfile_count; + sym_count = ssym_count; + dev_count = sdev_count; + dir_count = sdir_count; + dup_files = sdup_files; + fragments = sfragments; + fragment_size = 0; + longjmp(env, 1); +} + + +void sighandler() +{ + if(interrupted == 1) + restorefs(); + else { + ERROR("Interrupting will restore original filesystem!\n"); + ERROR("Interrupt again to quit\n"); + interrupted ++; + } +} + + +unsigned int mangle(char *d, char *s, int size, int block_size, int uncompressed, int data_block) +{ + unsigned long c_byte; + unsigned int res; + + /* BRCM begin */ + if (compress_algorithm == GZIP) { + c_byte = block_size << 1; + if(!uncompressed && (res = compress2(d, &c_byte, s, size, 9)) != Z_OK) { + if(res == Z_MEM_ERROR) + BAD_ERROR("zlib::compress failed, not enough memory\n"); + else if(res == Z_BUF_ERROR) + BAD_ERROR("zlib::compress failed, not enough room in output buffer\n"); + else + BAD_ERROR("zlib::compress failed, unknown error %d\n", res); + return 0; + } + } + + if (compress_algorithm == LZMA) { + unsigned lzma_algo; + unsigned lzma_dictsize; + unsigned lzma_fastbytes; + int opt_compression_level = 1; + + c_byte = block_size << 3; + switch (opt_compression_level) { + case 1 : + lzma_algo = 1; + lzma_dictsize = 1 << 20; + lzma_fastbytes = 64; + break; + case 2 : + lzma_algo = 2; + lzma_dictsize = 1 << 22; + lzma_fastbytes = 128; + break; + case 3 : + lzma_algo = 2; + lzma_dictsize = 1 << 24; + lzma_fastbytes = 255; + break; + default : + BAD_ERROR("Invalid LZMA compression level. Must be 1,2,3."); + return 0; + } + if(!uncompressed && !(res = compress_lzma_7zapi((const unsigned char*)s, + (unsigned)size, + (unsigned char*)d, + (unsigned *)&c_byte, + lzma_algo, lzma_dictsize, lzma_fastbytes))) { + /* this should NEVER happen */ + BAD_ERROR("Internal error - LZMA compression failed.\n"); + return 0; + } + //printf("LZMA: block_size=%d, in_size=%d, out_size=%d\n", block_size, size, c_byte); + } + /* BRCM end */ + + if(uncompressed || c_byte >= size) { + memcpy(d, s, size); + return size | (data_block ? SQUASHFS_COMPRESSED_BIT_BLOCK : SQUASHFS_COMPRESSED_BIT); + } + + return (unsigned int) c_byte; +} + + +squashfs_base_inode_header *get_inode(int req_size) +{ + int data_space; + unsigned short c_byte; + + while(cache_bytes >= SQUASHFS_METADATA_SIZE) { + if((inode_size - inode_bytes) < ((SQUASHFS_METADATA_SIZE << 1)) + 2) { + if((inode_table = (char *) realloc(inode_table, inode_size + (SQUASHFS_METADATA_SIZE << 1) + 2)) + == NULL) { + goto failed; + } + inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; + } + + c_byte = mangle(inode_table + inode_bytes + block_offset, data_cache, + SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, noI, 0); + TRACE("Inode block @ %x, size %d\n", inode_bytes, c_byte); + if(!swap) + memcpy((void *) (inode_table + inode_bytes), (void *) &c_byte, sizeof(unsigned short)); + else + SQUASHFS_SWAP_SHORTS((&c_byte), (inode_table + inode_bytes), 1); + if(check_data) + *((unsigned char *)(inode_table + inode_bytes + block_offset - 1)) = SQUASHFS_MARKER_BYTE; + inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + block_offset; + total_inode_bytes += SQUASHFS_METADATA_SIZE + block_offset; + memcpy(data_cache, data_cache + SQUASHFS_METADATA_SIZE, cache_bytes - SQUASHFS_METADATA_SIZE); + cache_bytes -= SQUASHFS_METADATA_SIZE; + } + + data_space = (cache_size - cache_bytes); + if(data_space < req_size) { + int realloc_size = cache_size == 0 ? ((req_size + SQUASHFS_METADATA_SIZE) & ~(SQUASHFS_METADATA_SIZE - 1)) : req_size - data_space; + + if((data_cache = (char *) realloc(data_cache, cache_size + realloc_size)) == NULL) { + goto failed; + } + cache_size += realloc_size; + } + + cache_bytes += req_size; + + return (squashfs_base_inode_header *)(data_cache + (cache_bytes - req_size)); + +failed: + BAD_ERROR("Out of memory in inode table reallocation!\n"); +} + + +void read_bytes(int fd, unsigned int byte, int bytes, char *buff) +{ + off_t off = byte; + + if(lseek(fd, off, SEEK_SET) == -1) { + perror("Lseek on destination failed"); + EXIT_MKSQUASHFS(); + } + + if(read(fd, buff, bytes) == -1) { + perror("Read on destination failed"); + EXIT_MKSQUASHFS(); + } +} + + +void write_bytes(int fd, unsigned int byte, int bytes, char *buff) +{ + off_t off = byte; + + if(off + bytes > ((long long)1<<32) - 1 ) + BAD_ERROR("Filesystem greater than maximum size 2^32 - 1\n"); + + if(lseek(fd, off, SEEK_SET) == -1) { + perror("Lseek on destination failed"); + EXIT_MKSQUASHFS(); + } + + if(write(fd, buff, bytes) == -1) { + perror("Write on destination failed"); + EXIT_MKSQUASHFS(); + } +} + + +unsigned int write_inodes() +{ + unsigned short c_byte; + int avail_bytes; + char *datap = data_cache; + unsigned int start_bytes = bytes; + + while(cache_bytes) { + if(inode_size - inode_bytes < ((SQUASHFS_METADATA_SIZE << 1) + 2)) { + if((inode_table = (char *) realloc(inode_table, inode_size + ((SQUASHFS_METADATA_SIZE << 1) + 2))) == NULL) { + BAD_ERROR("Out of memory in inode table reallocation!\n"); + } + inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; + } + avail_bytes = cache_bytes > SQUASHFS_METADATA_SIZE ? SQUASHFS_METADATA_SIZE : cache_bytes; + c_byte = mangle(inode_table + inode_bytes + block_offset, datap, avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); + TRACE("Inode block @ %x, size %d\n", inode_bytes, c_byte); + if(!swap) + memcpy((void *) (inode_table + inode_bytes), (void *) &c_byte, sizeof(unsigned short)); + else + SQUASHFS_SWAP_SHORTS((&c_byte), (inode_table + inode_bytes), 1); + if(check_data) + *((unsigned char *)(inode_table + inode_bytes + block_offset - 1)) = SQUASHFS_MARKER_BYTE; + inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + block_offset; + total_inode_bytes += avail_bytes + block_offset; + datap += avail_bytes; + cache_bytes -= avail_bytes; + } + + write_bytes(fd, bytes, inode_bytes, (char *) inode_table); + bytes += inode_bytes; + + return start_bytes; +} + + +unsigned int write_directories() +{ + unsigned short c_byte; + int avail_bytes; + char *directoryp = directory_data_cache; + unsigned int start_bytes = bytes; + + while(directory_cache_bytes) { + if(directory_size - directory_bytes < ((SQUASHFS_METADATA_SIZE << 1) + 2)) { + if((directory_table = (char *) realloc(directory_table, directory_size + + ((SQUASHFS_METADATA_SIZE << 1) + 2))) == NULL) { + BAD_ERROR("Out of memory in directory table reallocation!\n"); + } + directory_size += (SQUASHFS_METADATA_SIZE << 1) + 2; + } + avail_bytes = directory_cache_bytes > SQUASHFS_METADATA_SIZE ? SQUASHFS_METADATA_SIZE : directory_cache_bytes; + c_byte = mangle(directory_table + directory_bytes + block_offset, directoryp, avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); + TRACE("Directory block @ %x, size %d\n", directory_bytes, c_byte); + if(!swap) + memcpy((void *) (directory_table + directory_bytes), (void *) &c_byte, sizeof(unsigned short)); + else + SQUASHFS_SWAP_SHORTS((&c_byte), (directory_table + directory_bytes), 1); + if(check_data) + *((unsigned char *)(directory_table + directory_bytes + block_offset - 1)) = SQUASHFS_MARKER_BYTE; + directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + block_offset; + total_directory_bytes += avail_bytes + block_offset; + directoryp += avail_bytes; + directory_cache_bytes -= avail_bytes; + } + write_bytes(fd, bytes, directory_bytes, (char *) directory_table); + bytes += directory_bytes; + + return start_bytes; +} + + +unsigned int get_uid(squashfs_uid uid) +{ + int i; + + for(i = 0; (i < uid_count) && uids[i] != uid; i++); + if(i == uid_count) { + if(uid_count == SQUASHFS_UIDS) { + ERROR("Out of uids! - using uid 0 - probably not what's wanted!\n"); + i = 0; + } else + uids[uid_count++] = uid; + } + + return i; +} + + +unsigned int get_guid(squashfs_uid uid, squashfs_uid guid) +{ + int i; + + if(uid == guid) + return SQUASHFS_GUIDS; + + for(i = 0; (i < guid_count) && guids[i] != guid; i++); + if(i == guid_count) { + if(guid_count == SQUASHFS_GUIDS) { + ERROR("Out of gids! - using gid 0 - probably not what's wanted!\n"); + return SQUASHFS_GUIDS; + } else + guids[guid_count++] = guid; + } + + return i; +} + + +squashfs_inode create_inode(char *filename, int type, int byte_size, +squashfs_block start_block, unsigned int offset, unsigned int *block_list, struct fragment *fragment) +{ + squashfs_inode i_no; + struct stat buf; + squashfs_inode_header inode_header; + squashfs_base_inode_header *inode, *base = &inode_header.base; + + if(filename[0] == '\0') { + /* dummy top level directory, if multiple sources specified on command line */ + buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO; + buf.st_uid = getuid(); + buf.st_gid = getgid(); + buf.st_mtime = time(NULL); + } else if(lstat(filename, &buf) == -1) { + char buffer[8192]; + sprintf(buffer, "Cannot stat dir/file %s, ignoring", filename); + perror(buffer); + return SQUASHFS_INVALID; + } + + base->mode = SQUASHFS_MODE(buf.st_mode); + base->uid = get_uid((squashfs_uid) global_uid == -1 ? buf.st_uid : global_uid); + base->inode_type = type; + base->guid = get_guid((squashfs_uid) global_uid == -1 ? buf.st_uid : global_uid, (squashfs_uid) global_gid == -1 ? buf.st_gid : global_gid); + + if(type == SQUASHFS_FILE_TYPE) { + int i; + squashfs_reg_inode_header *reg = &inode_header.reg, *inodep; + + inodep = (squashfs_reg_inode_header *) inode = get_inode(sizeof(*reg) + + offset * sizeof(unsigned int)); + reg->mtime = buf.st_mtime; + reg->file_size = byte_size; + reg->start_block = start_block; + reg->fragment = fragment->index; + reg->offset = fragment->offset; + if(!swap) { + memcpy((void *) inodep, (void *) reg, sizeof(*reg)); + memcpy((void *) inodep->block_list, block_list, offset * sizeof(unsigned int)); + } else { + SQUASHFS_SWAP_REG_INODE_HEADER(reg, inodep); + SQUASHFS_SWAP_INTS(block_list, inodep->block_list, offset); + } + TRACE("File inode, file_size %d, start_block %x, blocks %d, fragment %d, offset %d, size %d\n", byte_size, + start_block, offset, fragment->index, fragment->offset, fragment->size); + for(i = 0; i < offset; i++) + TRACE("Block %d, size %d\n", i, block_list[i]); + } + else if(type == SQUASHFS_DIR_TYPE) { + squashfs_dir_inode_header *dir = &inode_header.dir; + + inode = get_inode(sizeof(*dir)); + dir->mtime = buf.st_mtime; + dir->file_size = byte_size; + dir->offset = offset; + dir->start_block = start_block; + if(!swap) + memcpy((void *) inode, (void *) dir, sizeof(*dir)); + else + SQUASHFS_SWAP_DIR_INODE_HEADER(dir, inode); + TRACE("Directory inode, file_size %d, start_block %x, offset %x\n", byte_size, + start_block, offset); + } + else if(type == SQUASHFS_CHRDEV_TYPE || type == SQUASHFS_BLKDEV_TYPE) { + squashfs_dev_inode_header *dev = &inode_header.dev; + + inode = get_inode(sizeof(*dev)); + dev->rdev = (unsigned short) ((major(buf.st_rdev) << 8) | + (minor(buf.st_rdev) & 0xff)); + if(!swap) + memcpy((void *) inode, (void *) dev, sizeof(*dev)); + else + SQUASHFS_SWAP_DEV_INODE_HEADER(dev, inode); + TRACE("Device inode, rdev %x\n", dev->rdev); + } + else if(type == SQUASHFS_SYMLINK_TYPE) { + squashfs_symlink_inode_header *symlink = &inode_header.symlink, *inodep; + int byte; + char buff[65536]; + + if((byte = readlink(filename, buff, 65536)) == -1) { + perror("Error in reading symbolic link, skipping..."); + return SQUASHFS_INVALID; + } + + if(byte == 65536) { + ERROR("Symlink is greater than 65536 bytes! skipping..."); + return SQUASHFS_INVALID; + } + + inodep = (squashfs_symlink_inode_header *) inode = get_inode(sizeof(*symlink) + byte); + symlink->symlink_size = byte; + if(!swap) + memcpy((void *) inode, symlink, sizeof(*symlink)); + else + SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); + strncpy(inodep->symlink, buff, byte); + TRACE("Symbolic link inode, symlink_size %d\n", byte); + } + else if(type == SQUASHFS_FIFO_TYPE || type == SQUASHFS_SOCKET_TYPE) { + squashfs_ipc_inode_header *ipc = &inode_header.ipc; + + inode = get_inode(sizeof(*ipc)); + if(!swap) + memcpy((void *) inode, (void *) ipc, sizeof(*ipc)); + else + SQUASHFS_SWAP_IPC_INODE_HEADER(ipc, inode); + TRACE("ipc inode, type %s %d\n", type == SQUASHFS_FIFO_TYPE ? "fifo" : "socket"); + } else + return SQUASHFS_INVALID; + + i_no = MKINODE(inode); + inode_count ++; + + TRACE("Created inode 0x%Lx, type %d, uid %d, guid %d\n", i_no, type, base->uid, base->guid); + + return i_no; +} + + +void init_dir(struct directory *dir) +{ + if((dir->buff = (char *)malloc(SQUASHFS_METADATA_SIZE)) == NULL) { + BAD_ERROR("Out of memory allocating directory buffer\n"); + } + + dir->size = SQUASHFS_METADATA_SIZE; + dir->p = dir->buff; + dir->entry_count = 256; + dir->entry_count_p = NULL; +} + + +void add_dir(squashfs_inode inode, char *name, int type, struct directory *dir) +{ + char *buff; + squashfs_dir_entry idir, *idirp; + unsigned int start_block = inode >> 16; + unsigned int offset = inode & 0xffff; + unsigned int size; + + if((size = strlen(name)) > SQUASHFS_NAME_LEN) { + size = SQUASHFS_NAME_LEN; + ERROR("Filename is greater than %d characters, truncating! ...\n", SQUASHFS_NAME_LEN); + } + + if(dir->p + sizeof(squashfs_dir_entry) + size + 6 >= dir->buff + dir->size) { + if((buff = (char *) realloc(dir->buff, dir->size += SQUASHFS_METADATA_SIZE)) == NULL) { + BAD_ERROR("Out of memory reallocating directory buffer\n"); + } + + dir->p = (dir->p - dir->buff) + buff; + if(dir->entry_count_p) + dir->entry_count_p = (squashfs_dir_header *) (((unsigned char *) dir->entry_count_p) - + dir->buff + buff); + dir->buff = buff; + } + + if(dir->entry_count == 256 || start_block != dir->start_block) { + if(dir->entry_count_p) { + squashfs_dir_header dir_header; + + dir_header.count = dir->entry_count - 1; + dir_header.start_block = dir->start_block; + if(!swap) + memcpy((void *) dir->entry_count_p, (void *) &dir_header, sizeof(dir_header)); + else + SQUASHFS_SWAP_DIR_HEADER((&dir_header), dir->entry_count_p); + } + + dir->entry_count_p = (squashfs_dir_header *) dir->p; + dir->start_block = start_block; + dir->entry_count = 0; + dir->p += sizeof(squashfs_dir_header); + } + + idirp = (squashfs_dir_entry *) dir->p; + idir.offset = offset; + idir.type = type; + idir.size = size - 1; + if(!swap) + memcpy((void *) idirp, (void *) &idir, sizeof(idir)); + else + SQUASHFS_SWAP_DIR_ENTRY((&idir), idirp); + strncpy(idirp->name, name, size); + dir->p += sizeof(squashfs_dir_entry) + size; + dir->entry_count ++; +} + + +squashfs_inode write_dir(char *filename, struct directory *dir) +{ + squashfs_inode inode; + unsigned int dir_size; + int data_space; + unsigned short c_byte; + + while(directory_cache_bytes >= SQUASHFS_METADATA_SIZE) { + if((directory_size - directory_bytes) < ((SQUASHFS_METADATA_SIZE << 1) + 2)) { + if((directory_table = (char *) realloc(directory_table, + directory_size + (SQUASHFS_METADATA_SIZE << 1) + 2)) == NULL) { + goto failed; + } + directory_size += SQUASHFS_METADATA_SIZE << 1; + } + + c_byte = mangle(directory_table + directory_bytes + block_offset, directory_data_cache, + SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, noI, 0); + TRACE("Directory block @ %x, size %d\n", directory_bytes, c_byte); + if(!swap) + memcpy((void *) directory_table + directory_bytes, (void *) &c_byte, sizeof(unsigned short)); + else + SQUASHFS_SWAP_SHORTS((&c_byte), (directory_table + directory_bytes), 1); + if(check_data) + *((unsigned char *)(directory_table + directory_bytes + block_offset - 1)) = SQUASHFS_MARKER_BYTE; + directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + block_offset; + total_directory_bytes += SQUASHFS_METADATA_SIZE + block_offset; + memcpy(directory_data_cache, directory_data_cache + SQUASHFS_METADATA_SIZE, directory_cache_bytes - SQUASHFS_METADATA_SIZE); + directory_cache_bytes -= SQUASHFS_METADATA_SIZE; + } + + dir_size = dir->p - dir->buff; + data_space = (directory_cache_size - directory_cache_bytes); + if(data_space < dir_size) { + int realloc_size = directory_cache_size == 0 ? ((dir_size + SQUASHFS_METADATA_SIZE) & ~(SQUASHFS_METADATA_SIZE - 1)) : dir_size - data_space; + + if((directory_data_cache = (char *) realloc(directory_data_cache, directory_cache_size + realloc_size)) == NULL) { + goto failed; + } + directory_cache_size += realloc_size; + } + + if(dir_size) { + squashfs_dir_header dir_header; + + dir_header.count = dir->entry_count - 1; + dir_header.start_block = dir->start_block; + if(!swap) + memcpy((void *) dir->entry_count_p, (void *) &dir_header, sizeof(dir_header)); + else + SQUASHFS_SWAP_DIR_HEADER((&dir_header), dir->entry_count_p); + memcpy(directory_data_cache + directory_cache_bytes, dir->buff, dir_size); + } + + inode = create_inode(filename, SQUASHFS_DIR_TYPE, dir_size, directory_bytes, directory_cache_bytes, NULL, NULL); + + directory_cache_bytes += dir_size; + +#ifdef SQUASHFS_TRACE + if(!swap) { + unsigned char *dirp; + int count; + + TRACE("Directory contents of inode 0x%Lx\n", inode); + dirp = dir->buff; + while(dirp < dir->p) { + char buffer[SQUASHFS_NAME_LEN + 1]; + squashfs_dir_entry idir, *idirp; + squashfs_dir_header *dirh = (squashfs_dir_header *) dirp; + count = dirh->count + 1; + dirp += sizeof(squashfs_dir_header); + + TRACE("\tStart block 0x%x, count %d\n", dirh->start_block, count); + + while(count--) { + idirp = (squashfs_dir_entry *) dirp; + memcpy((char *) &idir, (char *) idirp, sizeof(idir)); + strncpy(buffer, idirp->name, idir.size + 1); + buffer[idir.size + 1] = '\0'; + TRACE("\t\tname %s, inode offset 0x%x, type %d\n", buffer, + idir.offset, idir.type); + dirp += sizeof(squashfs_dir_entry) + idir.size + 1; + } + } + } +#endif + dir_count ++; + + return inode; + +failed: + BAD_ERROR("Out of memory in directory table reallocation!\n"); +} + + +char *get_fragment(char *buffer, struct fragment *fragment) +{ + if(fragment->index == fragments || fragment->index == SQUASHFS_INVALID_BLK) + return fragment_data + fragment->offset; + else { + squashfs_fragment_entry *disk_fragment = &fragment_table[fragment->index]; + int size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size), res; + long bytes = block_size; + + if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) { + char cbuffer[block_size]; + + read_bytes(fd, disk_fragment->start_block, size, cbuffer); + + if((res = uncompress(buffer, &bytes, (const char *) cbuffer, size)) != Z_OK) { + if(res == Z_MEM_ERROR) + BAD_ERROR("zlib::uncompress failed, not enough memory\n"); + else if(res == Z_BUF_ERROR) + BAD_ERROR("zlib::uncompress failed, not enough room in output buffer\n"); + else + BAD_ERROR("zlib::uncompress failed, unknown error %d\n", res); + } + } else + read_bytes(fd, disk_fragment->start_block, size, buffer); + return buffer + fragment->offset; + } +} + + +void write_fragment() +{ + int compressed_size; + unsigned char buffer[block_size << 1]; + + if(fragment_size == 0) + return; + + if(fragments % FRAG_SIZE == 0) + if((fragment_table = (squashfs_fragment_entry *) realloc(fragment_table, (fragments + FRAG_SIZE) * sizeof(squashfs_fragment_entry))) == NULL) + BAD_ERROR("Out of memory in fragment table\n"); + fragment_table[fragments].size = mangle(buffer, fragment_data, fragment_size, block_size, noF, 1); + fragment_table[fragments].start_block = bytes; + compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[fragments].size); + write_bytes(fd, bytes, compressed_size, buffer); + bytes += compressed_size; + total_uncompressed += fragment_size; + total_compressed += compressed_size; + TRACE("Writing fragment %d, uncompressed size %d, compressed size %d\n",fragments, fragment_size, compressed_size); + fragments ++; + fragment_size = 0; +} + + +static struct fragment empty_fragment = {SQUASHFS_INVALID_BLK, 0, 0}; +struct fragment *get_and_fill_fragment(char *buff, int size) +{ + struct fragment *ffrg; + + if(size == 0) + return &empty_fragment; + + if(fragment_size + size > block_size) + write_fragment(); + + if((ffrg = (struct fragment *) malloc(sizeof(struct fragment))) == NULL) + BAD_ERROR("Out of memory in fragment block allocation!\n"); + + ffrg->index = fragments; + ffrg->offset = fragment_size; + ffrg->size = size; + memcpy(fragment_data + fragment_size, buff, size); + fragment_size += size; + + return ffrg; +} + + +unsigned int write_fragment_table() +{ + unsigned int start_bytes, frag_bytes = SQUASHFS_FRAGMENT_BYTES(fragments), + meta_blocks = SQUASHFS_FRAGMENT_INDEXES(fragments); + char cbuffer[(SQUASHFS_METADATA_SIZE << 2) + 2], buffer[frag_bytes]; + squashfs_fragment_entry *p = (squashfs_fragment_entry *) buffer; + unsigned short c_byte; + int i, compressed_size; + squashfs_fragment_index list[meta_blocks]; + + TRACE("write_fragment_table: fragments %d, frag_bytes %d, meta_blocks %d\n", fragments, frag_bytes, meta_blocks); + for(i = 0; i < fragments; i++, p++) { + TRACE("write_fragment_table: fragment %d, start_block %x, size %d\n", i, fragment_table[i].start_block, fragment_table[i].size); + if(!swap) + memcpy((void *) p, &fragment_table[i], sizeof(squashfs_fragment_entry)); + else + SQUASHFS_SWAP_FRAGMENT_ENTRY(&fragment_table[i], p); + } + + for(i = 0; i < meta_blocks; i++) { + int avail_bytes = i == meta_blocks - 1 ? frag_bytes % SQUASHFS_METADATA_SIZE : SQUASHFS_METADATA_SIZE; + c_byte = mangle(cbuffer + block_offset, buffer + i * SQUASHFS_METADATA_SIZE , avail_bytes, SQUASHFS_METADATA_SIZE, noF, 0); + if(!swap) + memcpy((void *) cbuffer, (void *) &c_byte, sizeof(unsigned short)); + else + SQUASHFS_SWAP_SHORTS((&c_byte), cbuffer, 1); + if(check_data) + *((unsigned char *)(cbuffer + block_offset - 1)) = SQUASHFS_MARKER_BYTE; + list[i] = bytes; + compressed_size = SQUASHFS_COMPRESSED_SIZE(c_byte) + block_offset; + write_bytes(fd, bytes, compressed_size, cbuffer); + bytes += compressed_size; + } + + if(!swap) + write_bytes(fd, bytes, sizeof(list), (char *) list); + else { + squashfs_fragment_index slist[meta_blocks]; + SQUASHFS_SWAP_FRAGMENT_INDEXES(list, slist, meta_blocks); + write_bytes(fd, bytes, sizeof(list), (char *) slist); + } + + start_bytes = bytes; + bytes += sizeof(list); + + return start_bytes; +} + + +unsigned char *read_from_buffer(struct duplicate_buffer_handle *handle, unsigned int avail_bytes) +{ + unsigned char *v = handle->ptr; + handle->ptr += avail_bytes; + return v; +} + + +char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE]; +unsigned char *read_from_file(struct duplicate_buffer_handle *handle, unsigned int avail_bytes) +{ + read_bytes(fd, handle->start, avail_bytes, read_from_file_buffer); + handle->start += avail_bytes; + return read_from_file_buffer; +} + + +/* + * Compute 16 bit BSD checksum over the data + */ +unsigned short get_checksum(unsigned char *(get_next_file_block)(struct duplicate_buffer_handle *, unsigned int), struct duplicate_buffer_handle *handle, int l) +{ + unsigned short chksum = 0; + unsigned int bytes = 0; + unsigned char *b; + struct duplicate_buffer_handle position = *handle; + + while(l) { + bytes = l > SQUASHFS_FILE_MAX_SIZE ? SQUASHFS_FILE_MAX_SIZE : l; + l -= bytes; + b = get_next_file_block(&position, bytes); + while(bytes--) { + chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1; + chksum += *b++; + } + } + + return chksum; +} + + +static unsigned int cached_frag = -1; +void add_file(int start, int file_bytes, unsigned int *block_listp, int blocks, unsigned int fragment, int offset, int bytes) +{ + struct fragment *frg; + struct file_info *dupl_ptr; + char *datap; + struct duplicate_buffer_handle handle; + + if(!duplicate_checking) + return; + + if((frg = (struct fragment *) malloc(sizeof(struct fragment))) == NULL) + BAD_ERROR("Out of memory in fragment block allocation!\n"); + + frg->index = fragment; + frg->offset = offset; + frg->size = bytes; + if(cached_frag == fragment) + datap = fragment_data + offset; + else + datap = get_fragment(fragment_data, frg); + handle.start = start; + if((dupl_ptr = duplicate(read_from_file, &handle, file_bytes, &block_listp, &start, blocks, &frg, datap, bytes)) != NULL) + dupl_ptr->fragment = frg; + cached_frag = fragment; +} + + +struct file_info *duplicate(unsigned char *(get_next_file_block)(struct duplicate_buffer_handle *, unsigned int), struct duplicate_buffer_handle *file_start, int bytes, unsigned int **block_list, int *start, int blocks, struct fragment **fragment, char *frag_data, int frag_bytes) +{ + unsigned short checksum = get_checksum(get_next_file_block, file_start, bytes); + struct duplicate_buffer_handle handle = { frag_data, 0 }; + unsigned short fragment_checksum = get_checksum(read_from_buffer, &handle, frag_bytes); + struct file_info *dupl_ptr = bytes ? dupl[checksum] : frag_dups[fragment_checksum]; + + + for(; dupl_ptr; dupl_ptr = dupl_ptr->next) + if(bytes == dupl_ptr->bytes && frag_bytes == dupl_ptr->fragment->size && fragment_checksum == dupl_ptr->fragment_checksum) { + unsigned char buffer1[SQUASHFS_FILE_MAX_SIZE]; + unsigned int dup_bytes = dupl_ptr->bytes, dup_start = dupl_ptr->start; + struct duplicate_buffer_handle position = *file_start; + unsigned char *buffer; + while(dup_bytes) { + int avail_bytes = dup_bytes > SQUASHFS_FILE_MAX_SIZE ? SQUASHFS_FILE_MAX_SIZE : dup_bytes; + + buffer = get_next_file_block(&position, avail_bytes); + read_bytes(fd, dup_start, avail_bytes, buffer1); + if(memcmp(buffer, buffer1, avail_bytes) != 0) + break; + dup_bytes -= avail_bytes; + dup_start += avail_bytes; + } + if(dup_bytes == 0) { + char frag_buffer1[block_size]; + char *fragment_buffer1 = get_fragment(frag_buffer1, dupl_ptr->fragment); + if(frag_bytes == 0 || memcmp(frag_data, fragment_buffer1, frag_bytes) == 0) { + TRACE("Found duplicate file, start 0x%x, size %d, checksum 0x%x, fragment %d, size %d, offset %d, checksum 0x%x\n", dupl_ptr->start, + dupl_ptr->bytes, dupl_ptr->checksum, dupl_ptr->fragment->index, frag_bytes, dupl_ptr->fragment->offset, fragment_checksum); + *block_list = dupl_ptr->block_list; + *start = dupl_ptr->start; + *fragment = dupl_ptr->fragment; + return 0; + } + } + } + + + if((dupl_ptr = (struct file_info *) malloc(sizeof(struct file_info))) == NULL) { + BAD_ERROR("Out of memory in dup_files allocation!\n"); + } + + dupl_ptr->bytes = bytes; + dupl_ptr->checksum = checksum; + dupl_ptr->start = *start; + dupl_ptr->fragment_checksum = fragment_checksum; + if((dupl_ptr->block_list = (unsigned int *) malloc(blocks * sizeof(unsigned int))) == NULL) { + BAD_ERROR("Out of memory allocating block_list\n"); + } + + memcpy(dupl_ptr->block_list, *block_list, blocks * sizeof(unsigned int)); + dup_files ++; + if(bytes) { + dupl_ptr->next = dupl[checksum]; + dupl[checksum] = dupl_ptr; + } else { + dupl_ptr->next = frag_dups[fragment_checksum]; + frag_dups[fragment_checksum] = dupl_ptr; + } + + return dupl_ptr; +} + + +#define MINALLOCBYTES (1024 * 1024) +squashfs_inode write_file(char *filename, long long size, int *duplicate_file) +{ + unsigned int frag_bytes, file, start, file_bytes = 0, block = 0; + unsigned int c_byte; + long long read_size = (size > SQUASHFS_MAX_FILE_SIZE) ? SQUASHFS_MAX_FILE_SIZE : size; + unsigned int blocks = (read_size + block_size - 1) >> block_log; + unsigned int block_list[blocks], *block_listp = block_list; + char buff[block_size], *c_buffer; + int allocated_blocks = blocks, i, bbytes, whole_file = 1; + struct fragment *fragment; + struct file_info *dupl_ptr; + struct duplicate_buffer_handle handle; + + if(!no_fragments && (read_size < block_size || always_use_fragments)) { + allocated_blocks = blocks = read_size >> block_log; + frag_bytes = read_size % block_size; + } else + frag_bytes = 0; + + if(size > read_size) + ERROR("file %s truncated to %Ld bytes\n", filename, SQUASHFS_MAX_FILE_SIZE); + + total_bytes += read_size; + if((file = open(filename, O_RDONLY)) == -1) { + perror("Error in opening file, skipping..."); + return SQUASHFS_INVALID; + } + + do { + if((c_buffer = (char *) malloc((allocated_blocks + 1) << block_log)) == NULL) { + TRACE("Out of memory allocating write_file buffer, allocated_blocks %d, blocks %d\n", allocated_blocks, blocks); + whole_file = 0; + if((allocated_blocks << (block_log - 1)) < MINALLOCBYTES) + BAD_ERROR("Out of memory allocating write_file buffer, could not allocate %d blocks (%d Kbytes)\n", allocated_blocks, allocated_blocks << (block_log - 10)); + allocated_blocks >>= 1; + } + } while(!c_buffer); + + for(start = bytes; block < blocks; file_bytes += bbytes) { + for(i = 0, bbytes = 0; (i < allocated_blocks) && (block < blocks); i++) { + int available_bytes = read_size - (block * block_size) > block_size ? block_size : read_size - (block * block_size); + if(read(file, buff, available_bytes) == -1) + goto read_err; + c_byte = mangle(c_buffer + bbytes, buff, available_bytes, block_size, noD, 1); + block_list[block ++] = c_byte; + bbytes += SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); + } + if(!whole_file) { + write_bytes(fd, bytes, bbytes, c_buffer); + bytes += bbytes; + } + } + + if(frag_bytes != 0) + if(read(file, buff, frag_bytes) == -1) + goto read_err; + + close(file); + if(whole_file) { + handle.ptr = c_buffer; + if(duplicate_checking && (dupl_ptr = duplicate(read_from_buffer, &handle, file_bytes, &block_listp, &start, blocks, &fragment, buff, frag_bytes)) == NULL) { + *duplicate_file = TRUE; + goto wr_inode; + } + write_bytes(fd, bytes, file_bytes, c_buffer); + bytes += file_bytes; + } else { + handle.start = start; + if(duplicate_checking && (dupl_ptr = duplicate(read_from_file, &handle, file_bytes, &block_listp, &start, blocks, &fragment, buff, frag_bytes)) == NULL) { + bytes = start; + if(!block_device) + ftruncate(fd, bytes); + *duplicate_file = TRUE; + goto wr_inode; + } + } + + fragment = get_and_fill_fragment(buff, frag_bytes); + if(duplicate_checking) + dupl_ptr->fragment = fragment; + + *duplicate_file = FALSE; + +wr_inode: + free(c_buffer); + file_count ++; + return create_inode(filename, SQUASHFS_FILE_TYPE, read_size, start, blocks, block_listp, fragment); + +read_err: + perror("Error in reading file, skipping..."); + free(c_buffer); + return SQUASHFS_INVALID; +} + + +struct linuxdir { + DIR *linuxdir; + char pathname[8192]; +}; + + +void *linux_opendir(char *pathname, struct directory *dir) +{ + struct linuxdir *linuxdir; + if((linuxdir = malloc(sizeof(struct linuxdir))) == NULL) + return NULL; + if((linuxdir->linuxdir = opendir(pathname)) == NULL) { + free(linuxdir); + return NULL; + } + strcpy(linuxdir->pathname, pathname); + return (void *) linuxdir; +} + + +int linux_readdir(void *l, char *filename, char *dir_name) +{ + struct dirent *d_name; + struct linuxdir *linuxdir = (struct linuxdir *)l; + + do { + if((d_name = readdir(linuxdir->linuxdir)) == NULL) + return FALSE; + } while(strcmp(d_name->d_name, ".") == 0 || strcmp(d_name->d_name, "..") == 0); + strcat(strcat(strcpy(filename, linuxdir->pathname), "/"), d_name->d_name); + strcpy(dir_name, d_name->d_name); + return TRUE; +} + + +void linux_closedir(void *linuxdir) +{ + closedir(((struct linuxdir *)linuxdir)->linuxdir); + free(linuxdir); +} + + +char b_buffer[8192]; +char *name; +char *basename_r(); + +char *getbase(char *pathname) +{ + char *result; + + if(*pathname != '/') { + result = getenv("PWD"); + strcat(strcat(strcpy(b_buffer, result), "/"), pathname); + } else + strcpy(b_buffer, pathname); + name = b_buffer; + if(((result = basename_r()) == NULL) || (strcmp(result, "..") == 0)) + return NULL; + else + return result; +} + + +char *basename_r() +{ + char *s; + char *p; + int n = 1; + + for(;;) { + s = name; + if(*name == '\0') + return NULL; + if(*name != '/') { + while(*name != '\0' && *name != '/') name++; + n = name - s; + } + while(*name == '/') name++; + if(strncmp(s, ".", n) == 0) + continue; + if((*name == '\0') || (strncmp(s, "..", n) == 0) || ((p = basename_r()) == NULL)) { + s[n] = '\0'; + return s; + } + if(strcmp(p, "..") == 0) + continue; + return p; + } +} + + +char encomp_pathname[8192]; +int encomp_entry = 0; + +void *encomp_opendir(char *pathname, struct directory *dir) +{ + int i; + + for(i = 0; i < old_root_entries; i++) + add_dir(old_root_entry[i].inode, old_root_entry[i].name, old_root_entry[i].type, dir); + + return (void *)source_path; +} + + +int encomp_readdir(void *l, char *filename, char *dir_name) +{ + char *basename; + int n, pass = 1; + + while(encomp_entry < source) { + if((basename = getbase(source_path[encomp_entry])) == NULL) { + ERROR("Bad source directory %s - skipping ...\n", source_path[encomp_entry]); + encomp_entry++; + continue; + } + strcpy(filename, source_path[encomp_entry]); + strcpy(dir_name, basename); + for(;;) { + for(n = 0; n < old_root_entries && strcmp(old_root_entry[n].name, dir_name) != 0; n++); + if(n == old_root_entries) { + add_old_root_entry(dir_name, 0, 0); + encomp_entry++; + return TRUE; + } + ERROR("Source directory entry %s already used! - trying ", dir_name); + sprintf(dir_name, "%s_%d", basename, pass++); + ERROR("%s\n", dir_name); + } + } + return FALSE; +} + + +void encomp_closedir(void *linuxdir) +{ +} + + +void *single_opendir(char *pathname, struct directory *dir) +{ + encomp_opendir(pathname, dir); + return linux_opendir(pathname, dir); +} + + +int single_readdir(void *l, char *filename, char *dir_name) +{ + int i, pass = 1; + char name[SQUASHFS_NAME_LEN + 1]; + + if(linux_readdir(l, filename, dir_name) == FALSE) + return FALSE; + + strcpy(name, dir_name); + for(;;) { + for(i = 0; i < old_root_entries && strcmp(old_root_entry[i].name, dir_name) != 0; i++); + if(i == old_root_entries) { + add_old_root_entry(dir_name, 0, 0); + return TRUE; + } + ERROR("Source directory entry %s already used! - trying ", dir_name); + sprintf(dir_name, "%s_%d", name, pass++); + ERROR("%s\n", dir_name); + } +} + + +squashfs_inode dir_scan(char *pathname, void* (_opendir)(char *, struct directory *), int (_readdir)(void *, char *, char *), + void (_closedir)(void *)) +{ + void *linuxdir; + struct stat buf; + char filename[8192], dir_name[8192]; + int squashfs_type; + squashfs_inode inode = SQUASHFS_INVALID; + struct directory dir; + + init_dir(&dir); + if((linuxdir = _opendir(pathname, &dir)) == NULL) { + ERROR("Could not open %s, skipping...\n", pathname); + goto error; + } + + while(_readdir(linuxdir, filename, dir_name) != FALSE) { + + if(lstat(filename, &buf) == -1) { + char buffer[8192]; + sprintf(buffer, "Cannot stat dir/file %s, ignoring", filename); + perror(buffer); + continue; + } + if(excluded(filename, &buf)) + continue; + + switch(buf.st_mode & S_IFMT) { + case S_IFREG: { + int duplicate_file; + + squashfs_type = SQUASHFS_FILE_TYPE; + if(!sorted) { + inode = write_file(filename, buf.st_size, &duplicate_file); + INFO("file %s, uncompressed size %Ld bytes, %s\n", filename, buf.st_size, duplicate_file ? "DUPLICATE" : ""); + } else + inode = get_sorted_inode(&buf); + break; + } + case S_IFDIR: + squashfs_type = SQUASHFS_DIR_TYPE; + inode = dir_scan(filename, linux_opendir, linux_readdir, linux_closedir); + break; + + case S_IFLNK: + squashfs_type = SQUASHFS_SYMLINK_TYPE; + inode = create_inode(filename, squashfs_type, 0, 0, 0, NULL, NULL); + INFO("symbolic link %s inode 0x%Lx\n", dir_name, inode); + sym_count ++; + break; + + case S_IFCHR: + squashfs_type = SQUASHFS_CHRDEV_TYPE; + inode = create_inode(filename, squashfs_type, 0, 0, 0, NULL, NULL); + INFO("character device %s inode 0x%Lx\n", dir_name, inode); + dev_count ++; + break; + + case S_IFBLK: + squashfs_type = SQUASHFS_BLKDEV_TYPE; + inode = create_inode(filename, squashfs_type, 0, 0, 0, NULL, NULL); + INFO("block device %s inode 0x%Lx\n", dir_name, inode); + dev_count ++; + break; + + case S_IFIFO: + squashfs_type = SQUASHFS_FIFO_TYPE; + inode = create_inode(filename, squashfs_type, 0, 0, 0, NULL, NULL); + INFO("fifo %s inode 0x%Lx\n", dir_name, inode); + fifo_count ++; + break; + + case S_IFSOCK: + squashfs_type = SQUASHFS_SOCKET_TYPE; + inode = create_inode(filename, squashfs_type, 0, 0, 0, NULL, NULL); + INFO("unix domain socket %s inode 0x%Lx\n", dir_name, inode); + sock_count ++; + break; + + default: + ERROR("%s unrecognised file type, mode is %x\n", filename, buf.st_mode); + continue; + } + + if(inode != SQUASHFS_INVALID) + add_dir(inode, dir_name, squashfs_type, &dir); + } + + _closedir(linuxdir); + inode = write_dir(pathname, &dir); + INFO("directory %s inode 0x%Lx\n", pathname, inode); + +error: + free(dir.buff); + + return inode; +} + + +unsigned int slog(unsigned int block) +{ + int i; + + for(i = 9; i <= 16; i++) + if(block == (1 << i)) + return i; + return 0; +} + + +int excluded(char *filename, struct stat *buf) +{ + int i; + + for(i = 0; i < exclude; i++) + if((exclude_paths[i].st_dev == buf->st_dev) && (exclude_paths[i].st_ino == buf->st_ino)) + return TRUE; + return FALSE; +} + + +#define ADD_ENTRY(buf) \ + if(exclude % EXCLUDE_SIZE == 0) {\ + if((exclude_paths = (struct exclude_info *) realloc(exclude_paths, (exclude + EXCLUDE_SIZE) * sizeof(struct exclude_info))) == NULL)\ + BAD_ERROR("Out of memory in exclude dir/file table\n");\ + }\ + exclude_paths[exclude].st_dev = buf.st_dev;\ + exclude_paths[exclude++].st_ino = buf.st_ino; +int add_exclude(char *path) +{ + int i; + char buffer[4096], filename[4096]; + struct stat buf; + + if(path[0] == '/' || strncmp(path, "./", 2) == 0 || strncmp(path, "../", 3) == 0) { + if(lstat(path, &buf) == -1) { + sprintf(buffer, "Cannot stat exclude dir/file %s, ignoring", path); + perror(buffer); + return TRUE; + } + ADD_ENTRY(buf); + return TRUE; + } + + for(i = 0; i < source; i++) { + strcat(strcat(strcpy(filename, source_path[i]), "/"), path); + if(lstat(filename, &buf) == -1) { + if(!(errno == ENOENT || errno == ENOTDIR)) { + sprintf(buffer, "Cannot stat exclude dir/file %s, ignoring", filename); + perror(buffer); + } + continue; + } + ADD_ENTRY(buf); + } + return TRUE; +} + + +void add_old_root_entry(char *name, squashfs_inode inode, int type) +{ + if((old_root_entry = (struct old_root_entry_info *) realloc(old_root_entry, sizeof(struct old_root_entry_info) + * (old_root_entries + 1))) == NULL) + BAD_ERROR("Out of memory in old root directory entries reallocation\n"); + + strcpy(old_root_entry[old_root_entries].name, name); + old_root_entry[old_root_entries].inode = inode; + old_root_entry[old_root_entries++].type = type; +} + + +#define VERSION() \ + printf("mksquashfs version 2.0\n");\ + printf("copyright (C) 2004 Phillip Lougher (plougher@users.sourceforge.net)\n\n"); \ + printf("This program is free software; you can redistribute it and/or\n");\ + printf("modify it under the terms of the GNU General Public License\n");\ + printf("as published by the Free Software Foundation; either version 2,\n");\ + printf("or (at your option) any later version.\n\n");\ + printf("This program is distributed in the hope that it will be useful,\n");\ + printf("but WITHOUT ANY WARRANTY; without even the implied warranty of\n");\ + printf("MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n");\ + printf("GNU General Public License for more details.\n"); +int main(int argc, char *argv[]) +{ + struct stat buf; + int i; + squashfs_super_block sBlk; + char *b, *root_name = NULL; + int be, nopad = FALSE, delete = FALSE, keep_as_directory = FALSE, orig_be; + +#if __BYTE_ORDER == __BIG_ENDIAN + be = TRUE; +#else + be = FALSE; +#endif + + block_log = slog(block_size); + if(argc > 1 && strcmp(argv[1], "-version") == 0) { + VERSION(); + exit(0); + } + for(i = 1; i < argc && argv[i][0] != '-'; i++); + if(i < 3) + goto printOptions; + source_path = argv + 1; + source = i - 2; + for(; i < argc; i++) { + if(strcmp(argv[i], "-b") == 0) { + if((++i == argc) || (block_size = strtol(argv[i], &b, 10), *b !='\0')) { + ERROR("%s: -b missing or invalid block size\n", argv[0]); + exit(1); + } + + if((block_log = slog(block_size)) == 0) { + ERROR("%s: -b block size not power of two or not between 512 and 64K\n", argv[0]); + exit(1); + } + } else if(strcmp(argv[i], "-ef") == 0) { + if(++i == argc) { + ERROR("%s: -ef missing filename\n", argv[0]); + exit(1); + } + } else if(strcmp(argv[i], "-no-duplicates") == 0) + duplicate_checking = FALSE; + + else if(strcmp(argv[i], "-no-fragments") == 0) + no_fragments = TRUE; + + else if(strcmp(argv[i], "-always-use-fragments") == 0) + always_use_fragments = TRUE; + + else if(strcmp(argv[i], "-sort") == 0) { + if(++i == argc) { + ERROR("%s: -sort missing filename\n", argv[0]); + exit(1); + } + } else if(strcmp(argv[i], "-all-root") == 0 || + strcmp(argv[i], "-root-owned") == 0) + global_uid = global_gid = 0; + + else if(strcmp(argv[i], "-force-uid") == 0) { + if(++i == argc) { + ERROR("%s: -force-uid missing uid or user\n", argv[0]); + exit(1); + } + if((global_uid = strtoll(argv[i], &b, 10)), *b =='\0') { + if(global_uid < 0 || global_uid > (((long long) 1 << 32) - 1)) { + ERROR("%s: -force-uid uid out of range\n", argv[0]); + exit(1); + } + } else { + struct passwd *uid = getpwnam(argv[i]); + if(uid) + global_uid = uid->pw_uid; + else { + ERROR("%s: -force-uid invalid uid or unknown user\n", argv[0]); + exit(1); + } + } + } else if(strcmp(argv[i], "-force-gid") == 0) { + if(++i == argc) { + ERROR("%s: -force-gid missing gid or group\n", argv[0]); + exit(1); + } + if((global_gid = strtoll(argv[i], &b, 10)), *b =='\0') { + if(global_gid < 0 || global_gid > (((long long) 1 << 32) - 1)) { + ERROR("%s: -force-gid gid out of range\n", argv[0]); + exit(1); + } + } else { + struct group *gid = getgrnam(argv[i]); + if(gid) + global_gid = gid->gr_gid; + else { + ERROR("%s: -force-gid invalid gid or unknown group\n", argv[0]); + exit(1); + } + } + } else if(strcmp(argv[i], "-noI") == 0 || + strcmp(argv[i], "-noInodeCompression") == 0) + noI = TRUE; + + else if(strcmp(argv[i], "-noD") == 0 || + strcmp(argv[i], "-noDataCompression") == 0) + noD = TRUE; + + else if(strcmp(argv[i], "-noF") == 0 || + strcmp(argv[i], "-noFragmentCompression") == 0) + noF = TRUE; + + else if(strcmp(argv[i], "-nopad") == 0) + nopad = TRUE; + + else if(strcmp(argv[i], "-check_data") == 0) + check_data = TRUE; + + else if(strcmp(argv[i], "-info") == 0) + silent = 0; + + else if(strcmp(argv[i], "-be") == 0) + be = TRUE; + + else if(strcmp(argv[i], "-le") == 0) + be = FALSE; + /* BRCM begin */ + else if(strcmp(argv[i], "-gzip") == 0) + compress_algorithm = GZIP; + + else if(strcmp(argv[i], "-lzma") == 0) + compress_algorithm = LZMA; + /* BRCM end */ + + else if(strcmp(argv[i], "-e") == 0) + break; + + else if(strcmp(argv[i], "-noappend") == 0) + delete = TRUE; + + else if(strcmp(argv[i], "-keep-as-directory") == 0) + keep_as_directory = TRUE; + + else if(strcmp(argv[i], "-root-becomes") == 0) { + if(++i == argc) { + ERROR("%s: -root-becomes: missing name\n", argv[0]); + exit(1); + } + root_name = argv[i]; + } else if(strcmp(argv[i], "-version") == 0) { + VERSION(); + } else { + ERROR("%s: invalid option\n\n", argv[0]); +printOptions: + ERROR("SYNTAX:%s source1 source2 ... dest [options] [-e list of exclude dirs/files]\n", argv[0]); + ERROR("\nOptions are\n"); + ERROR("\t-info\t\t\t\tprint files written to filesystem\n"); + ERROR("\t-sort sort file\t\t\tsort files according to priorities in sort file. One file or dir\n"); + ERROR("\t\t\t\t\twith priority per line. Priority -32768 to 32767, default priority 0\n"); + ERROR("\t-b block size\t\t\tsize of blocks in "); + ERROR("filesystem, default %d\n", SQUASHFS_FILE_SIZE); + ERROR("\t-noappend\t\t\tDo not append to existing filesystem on dest, write a new filesystem\n"); + ERROR("\t\t\t\t\tThis is the default action if dest does not exist, or if no filesystem is on it\n"); + ERROR("\t-keep-as-directory\t\tIf one source directory is specified, create a root directory\n"); + ERROR("\t\t\t\t\tcontaining that directory, rather than the contents of the directory\n"); + ERROR("\t-root-becomes name\t\tWhen appending source files/directories, make the original\n"); + ERROR("\t\t\t\t\troot become a subdirectory in the new root called name, rather\n"); + ERROR("\t\t\t\t\tthan adding the new source items to the original root\n"); + ERROR("\t-noI -noInodeCompression\tdo not compress inode table\n"); + ERROR("\t-noD -noDataCompression\t\tdo not compress data blocks\n"); + ERROR("\t-noF -noFragmentCompression\tdo not compress fragment blocks\n"); + ERROR("\t-no-duplicates\t\t\tdo not perform duplicate checking\n"); + ERROR("\t-no-fragments\t\t\tdo not use fragments\n"); + ERROR("\t-always-use-fragments\t\tuse fragment blocks for files larger than block size\n"); + ERROR("\t-nopad\t\t\t\tdo not pad filesystem to a multiple of 4K\n"); + ERROR("\t-check_data\t\t\tadd checkdata for greater filesystem checks\n"); + ERROR("\t-le\t\t\t\tcreate a little endian filesystem\n"); + ERROR("\t-be\t\t\t\tcreate a big endian filesystem\n"); + /* BRCM begin */ + ERROR("\t-gzip\t\t\t\tuse gzip compression\n"); + ERROR("\t-lzma\t\t\t\tuse lzma compression(default)\n"); + /* BRCM end */ + ERROR("\t-ef exclude file\t\tfile is a list of exclude dirs/files - one per line\n"); + ERROR("\t-all-root\t\t\toverride file uid/gid and make all file uid/gids owned by root\n"); + ERROR("\t-root-owned\t\t\talternative name for -all-root\n"); + ERROR("\t-force-uid uid\t\t\tset all file uids to uid\n"); + ERROR("\t-force-gid gid\t\t\tset all file gids to gid\n"); + ERROR("\t-version\t\t\tprint version, licence and copyright message\n"); + exit(1); + } + } + + if(stat(argv[source + 1], &buf) == -1) { + if(errno == ENOENT) { /* Does not exist */ + if((fd = open(argv[source + 1], O_CREAT | O_TRUNC | O_RDWR, S_IRWXU)) == -1) { + perror("Could not create destination file"); + exit(1); + } + delete = TRUE; + } else { + perror("Could not stat destination file"); + exit(1); + } + + } else { + if(S_ISBLK(buf.st_mode)) { + if((fd = open(argv[source + 1], O_RDWR)) == -1) { + perror("Could not open block device as destination"); + exit(1); + } + block_device = 1; + + } else if(S_ISREG(buf.st_mode)) { + if((fd = open(argv[source + 1], (delete ? O_TRUNC : 0) | O_RDWR)) == -1) { + perror("Could not open regular file for writing as destination"); + exit(1); + } + } + else { + ERROR("Destination not block device or regular file\n"); + exit(1); + } + + if(!delete) { + if(read_super(fd, &sBlk, &orig_be, argv[source + 1]) == 0) { + if(S_ISREG(buf.st_mode)) { /* reopen truncating file */ + close(fd); + if((fd = open(argv[source + 1], O_TRUNC | O_RDWR)) == -1) { + perror("Could not open regular file for writing as destination"); + exit(1); + } + } + delete = TRUE; + } + + } + } + + /* process the exclude files - must be done afer destination file has been possibly created */ + for(i = source + 2; i < argc; i++) + if(strcmp(argv[i], "-ef") == 0) { + FILE *fd; + char filename[16385]; + if((fd = fopen(argv[++i], "r")) == NULL) { + perror("Could not open exclude file..."); + exit(1); + } + while(fscanf(fd, "%16384[^\n]\n", filename) != EOF) + add_exclude(filename); + fclose(fd); + } else if(strcmp(argv[i], "-e") == 0) + break; + else if(strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "-root-becomes") == 0 || strcmp(argv[i], "-sort") == 0) + i++; + + if(i != argc) { + if(++i == argc) { + ERROR("%s: -e missing arguments\n", argv[0]); + exit(1); + } + while(i < argc && add_exclude(argv[i++])); + } + + /* process the sort files - must be done afer the exclude files */ + for(i = source + 2; i < argc; i++) + if(strcmp(argv[i], "-sort") == 0) { + read_sort_file(argv[++i], source, source_path); + sorted ++; + } else if(strcmp(argv[i], "-e") == 0) + break; + else if(strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "-root-becomes") == 0 || strcmp(argv[i], "-ef") == 0) + i++; + + if((fragment_data = (char *) malloc(block_size)) == NULL) + BAD_ERROR("Out of memory allocating fragment_data"); + + if(delete) { + printf("Creating %s filesystem on %s, block size %d.\n", + be ? "big endian" : "little endian", argv[source + 1], block_size); + bytes = sizeof(squashfs_super_block); + } else { + be = orig_be; + block_log = slog(block_size = sBlk.block_size); + noI = SQUASHFS_UNCOMPRESSED_INODES(sBlk.flags); + noD = SQUASHFS_UNCOMPRESSED_DATA(sBlk.flags); + noF = SQUASHFS_UNCOMPRESSED_FRAGMENTS(sBlk.flags); + check_data = SQUASHFS_CHECK_DATA(sBlk.flags); + no_fragments = SQUASHFS_NO_FRAGMENTS(sBlk.flags); + always_use_fragments = SQUASHFS_ALWAYS_FRAGMENTS(sBlk.flags); + duplicate_checking = SQUASHFS_DUPLICATES(sBlk.flags); + + fragments = SQUASHFS_INVALID_BLK; + if((bytes = read_filesystem(root_name, fd, &sBlk, &inode_table, &inode_bytes, &data_cache, + &cache_bytes, &cache_size, &directory_table, &directory_bytes, + &directory_data_cache, &directory_cache_bytes, &directory_cache_size, + &file_count, &sym_count, &dev_count, &dir_count, &fifo_count, &sock_count, (squashfs_uid *) uids, &uid_count, + (squashfs_uid *) guids, &guid_count, + &total_bytes, &total_inode_bytes, &total_directory_bytes, add_old_root_entry, &fragment_table)) == 0) { + ERROR("Failed to read existing filesystem - will not overwrite - ABORTING!\n"); + exit(1); + } + if((fragments = sBlk.fragments)) + fragment_table = (squashfs_fragment_entry *) realloc((char *) fragment_table, ((fragments + FRAG_SIZE - 1) & ~(FRAG_SIZE - 1)) * sizeof(squashfs_fragment_entry)); + + printf("Appending to existing %s squashfs filesystem on %s, block size %d\n", be ? "big endian" : + "little endian", argv[source + 1], block_size); + printf("All -be, -le, -b, -noI, noD, noF, -check_data, no-duplicates, no-fragments and always-use-fragments options ignored\n"); + printf("\nIf appending is not wanted, please re-run with -noappend specified!\n\n"); + + inode_size = inode_bytes; + directory_size = directory_bytes; + + /* save original filesystem state for restoring ... */ + sfragments = fragments; + sbytes = bytes; + sinode_count = sBlk.inodes; + inode_count = file_count + dir_count + sym_count + dev_count; + sdata_cache = (char *)malloc(scache_bytes = cache_size); + sdirectory_data_cache = (char *)malloc(sdirectory_cache_bytes = directory_cache_size); + memcpy(sdata_cache, data_cache, scache_bytes); + memcpy(sdirectory_data_cache, directory_data_cache, sdirectory_cache_bytes); + sinode_bytes = inode_bytes; + sdirectory_bytes = directory_bytes; + suid_count = uid_count; + sguid_count = guid_count; + stotal_bytes = total_bytes; + stotal_inode_bytes = total_inode_bytes; + stotal_directory_bytes = total_directory_bytes; + sfile_count = file_count; + ssym_count = sym_count; + sdev_count = dev_count; + sdir_count = dir_count; + sdup_files = dup_files; + restore = TRUE; + if(setjmp(env)) + goto restore_filesystem; + signal(SIGTERM, sighandler); + signal(SIGINT, sighandler); + write_bytes(fd, SQUASHFS_START, 4, "\0\0\0\0"); + } + +#if __BYTE_ORDER == __BIG_ENDIAN + swap = !be; +#else + swap = be; +#endif + + block_offset = check_data ? 3 : 2; + + if(stat(source_path[0], &buf) == -1) { + perror("Cannot stat source directory"); + EXIT_MKSQUASHFS(); + } + + if(sorted) + sort_files_and_write(source, source_path); + + if(delete && !keep_as_directory && source == 1 && S_ISDIR(buf.st_mode)) + sBlk.root_inode = dir_scan(source_path[0], linux_opendir, linux_readdir, linux_closedir); + else if(!keep_as_directory && source == 1 && S_ISDIR(buf.st_mode)) + sBlk.root_inode = dir_scan(source_path[0], single_opendir, single_readdir, linux_closedir); + else + sBlk.root_inode = dir_scan("", encomp_opendir, encomp_readdir, encomp_closedir); + sBlk.inodes = inode_count; + sBlk.s_magic = SQUASHFS_MAGIC; + sBlk.s_major = SQUASHFS_MAJOR; + sBlk.s_minor = SQUASHFS_MINOR; + sBlk.block_size = block_size; + sBlk.block_log = block_log; + sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, check_data, noF, no_fragments, always_use_fragments, duplicate_checking); + sBlk.mkfs_time = time(NULL); + +restore_filesystem: + write_fragment(); + sBlk.fragments = fragments; + sBlk.inode_table_start = write_inodes(); + sBlk.directory_table_start = write_directories(); + sBlk.fragment_table_start = write_fragment_table(); + + TRACE("sBlk->inode_table_start 0x%x\n", sBlk.inode_table_start); + TRACE("sBlk->directory_table_start 0x%x\n", sBlk.directory_table_start); + TRACE("sBlk->fragment_table_start 0x%x\n", sBlk.fragment_table_start); + + if(sBlk.no_uids = uid_count) { + if(!swap) + write_bytes(fd, bytes, uid_count * sizeof(squashfs_uid), (char *) uids); + else { + squashfs_uid uids_copy[uid_count]; + + SQUASHFS_SWAP_DATA(uids, uids_copy, uid_count, sizeof(squashfs_uid) * 8); + write_bytes(fd, bytes, uid_count * sizeof(squashfs_uid), (char *) uids_copy); + } + sBlk.uid_start = bytes; + bytes += uid_count * sizeof(squashfs_uid); + } else + sBlk.uid_start = 0; + + if(sBlk.no_guids = guid_count) { + if(!swap) + write_bytes(fd, bytes, guid_count * sizeof(squashfs_uid), (char *) guids); + else { + squashfs_uid guids_copy[guid_count]; + + SQUASHFS_SWAP_DATA(guids, guids_copy, guid_count, sizeof(squashfs_uid) * 8); + write_bytes(fd, bytes, guid_count * sizeof(squashfs_uid), (char *) guids_copy); + } + sBlk.guid_start = bytes; + bytes += guid_count * sizeof(squashfs_uid); + } else + sBlk.guid_start = 0; + + sBlk.bytes_used = bytes; + + if(!swap) + write_bytes(fd, SQUASHFS_START, sizeof(squashfs_super_block), (char *) &sBlk); + else { + squashfs_super_block sBlk_copy; + + SQUASHFS_SWAP_SUPER_BLOCK((&sBlk), &sBlk_copy); + write_bytes(fd, SQUASHFS_START, sizeof(squashfs_super_block), (char *) &sBlk_copy); + } + + if(!nopad && (i = bytes & (4096 - 1))) { + unsigned char temp[4096] = {0}; + write_bytes(fd, bytes, 4096 - i, temp); + } + + total_bytes += total_inode_bytes + total_directory_bytes + uid_count + * sizeof(unsigned short) + guid_count * sizeof(unsigned short) + + sizeof(squashfs_super_block); + + printf("\n%s filesystem, data block size %d, %s data, %s metadata, %s fragments\n", be ? + "Big endian" : "Little endian", block_size, noI ? "uncompressed" : "compressed", noD ? + "uncompressed" : "compressed", no_fragments ? "no" : noF ? "uncompressed" : "compressed"); + printf("Filesystem size %.2f Kbytes (%.2f Mbytes)\n", bytes / 1024.0, bytes / (1024.0 * 1024.0)); + printf("\t%.2f%% of uncompressed filesystem size (%.2f Kbytes)\n", + ((float) bytes / total_bytes) * 100.0, total_bytes / 1024.0); + printf("Inode table size %d bytes (%.2f Kbytes)\n", + inode_bytes, inode_bytes / 1024.0); + printf("\t%.2f%% of uncompressed inode table size (%d bytes)\n", + ((float) inode_bytes / total_inode_bytes) * 100.0, total_inode_bytes); + printf("Directory table size %d bytes (%.2f Kbytes)\n", + directory_bytes, directory_bytes / 1024.0); + printf("\t%.2f%% of uncompressed directory table size (%d bytes)\n", + ((float) directory_bytes / total_directory_bytes) * 100.0, total_directory_bytes); + if(duplicate_checking) + printf("Number of duplicate files found %d\n", file_count - dup_files); + else + printf("No duplicate files removed\n"); + printf("Number of inodes %d\n", inode_count); + printf("Number of files %d\n", file_count); + if(!no_fragments) + printf("Number of fragments %d\n", fragments); + printf("Number of symbolic links %d\n", sym_count); + printf("Number of device nodes %d\n", dev_count); + printf("Number of fifo nodes %d\n", fifo_count); + printf("Number of socket nodes %d\n", sock_count); + printf("Number of directories %d\n", dir_count); + printf("Number of uids %d\n", uid_count); + + for(i = 0; i < uid_count; i++) { + struct passwd *user = getpwuid(uids[i]); + printf("\t%s (%d)\n", user == NULL ? "unknown" : user->pw_name, uids[i]); + } + + printf("Number of gids %d\n", guid_count); + + for(i = 0; i < guid_count; i++) { + struct group *group = getgrgid(guids[i]); + printf("\t%s (%d)\n", group == NULL ? "unknown" : group->gr_name, guids[i]); + } + close(fd); + return 0; +} diff --git a/hostTools/squashfs/mksquashfs.h b/hostTools/squashfs/mksquashfs.h new file mode 100644 index 0000000..b3d2e44 --- /dev/null +++ b/hostTools/squashfs/mksquashfs.h @@ -0,0 +1,47 @@ +/* + * macros to convert each packed bitfield structure from little endian to big + * endian and vice versa. These are needed when creating a filesystem on a + * machine with different byte ordering to the target architecture. + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * mksquashfs.h + * + */ + +/* + * macros used to swap each structure entry, taking into account + * bitfields and different bitfield placing conventions on differing architectures + */ +#if __BYTE_ORDER == __BIG_ENDIAN + /* convert from big endian to little endian */ +#define SQUASHFS_SWAP(value, p, pos, tbits) _SQUASHFS_SWAP(value, p, pos, tbits, b_pos) +#else + /* convert from little endian to big endian */ +#define SQUASHFS_SWAP(value, p, pos, tbits) _SQUASHFS_SWAP(value, p, pos, tbits, 64 - tbits - b_pos) +#endif + +#define _SQUASHFS_SWAP(value, p, pos, tbits, SHIFT) {\ + int bits;\ + int b_pos = pos % 8;\ + unsigned long long val = (long long) value << (SHIFT);\ + unsigned char *s = ((unsigned char *) &val) + 7;\ + unsigned char *d = ((unsigned char *)p) + (pos / 8);\ + for(bits = 0; bits < (tbits + b_pos); bits += 8) \ + *d++ |= *s--;\ +} +#define SQUASHFS_MEMSET(s, d, n) memset(d, 0, n); diff --git a/hostTools/squashfs/read_fs.c b/hostTools/squashfs/read_fs.c new file mode 100644 index 0000000..cdb62a7 --- /dev/null +++ b/hostTools/squashfs/read_fs.c @@ -0,0 +1,498 @@ +/* + * Read a squashfs filesystem. This is a highly compressed read only filesystem. + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * read_fs.c + */ + +extern void read_bytes(int, unsigned int, int, char *); +extern int add_file(int, int, unsigned int *, int, unsigned int, int, int); + +#define TRUE 1 +#define FALSE 0 +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "read_fs.h" +#include + +#include + +#ifdef SQUASHFS_TRACE +#define TRACE(s, args...) printf("mksquashfs: "s, ## args) +#else +#define TRACE(s, args...) +#endif + +#define ERROR(s, args...) fprintf(stderr, s, ## args) + +int swap; + +int read_block(int fd, int start, int *next, unsigned char *block, squashfs_super_block *sBlk) +{ + unsigned short c_byte; + int offset = 2; + + if(swap) { + read_bytes(fd, start, 2, block); + ((unsigned char *) &c_byte)[1] = block[0]; + ((unsigned char *) &c_byte)[0] = block[1]; + } else + read_bytes(fd, start, 2, (char *)&c_byte); + + if(SQUASHFS_CHECK_DATA(sBlk->flags)) + offset = 3; + if(SQUASHFS_COMPRESSED(c_byte)) { + unsigned char buffer[SQUASHFS_METADATA_SIZE]; + int res; + long bytes = SQUASHFS_METADATA_SIZE; + + c_byte = SQUASHFS_COMPRESSED_SIZE(c_byte); + read_bytes(fd, start + offset, c_byte, buffer); + + if((res = uncompress(block, &bytes, (const char *) buffer, c_byte)) != Z_OK) { + if(res == Z_MEM_ERROR) + ERROR("zlib::uncompress failed, not enough memory\n"); + else if(res == Z_BUF_ERROR) + ERROR("zlib::uncompress failed, not enough room in output buffer\n"); + else + ERROR("zlib::uncompress failed, unknown error %d\n", res); + return 0; + } + if(next) + *next = start + offset + c_byte; + return bytes; + } else { + c_byte = SQUASHFS_COMPRESSED_SIZE(c_byte); + read_bytes(fd, start + offset, c_byte, block); + if(next) + *next = start + offset + c_byte; + return c_byte; + } +} + + +int scan_inode_table(int fd, int start, int end, int root_inode_start, squashfs_super_block *sBlk, + squashfs_dir_inode_header *dir_inode, unsigned char **inode_table, unsigned int *root_inode_block, + unsigned int *uncompressed_file, unsigned int *uncompressed_directory, int *file_count, int *sym_count, + int *dev_count, int *dir_count, int *fifo_count, int *sock_count) +{ + unsigned char *cur_ptr; + int bytes = 0, size = 0, files = 0; + squashfs_reg_inode_header inode; + unsigned int directory_start_block; + + TRACE("scan_inode_table: start 0x%x, end 0x%x, root_inode_start 0x%x\n", start, end, root_inode_start); + while(start < end) { + if(start == root_inode_start) { + TRACE("scan_inode_table: read compressed block 0x%x containing root inode\n", start); + *root_inode_block = bytes; + } + if((size - bytes < SQUASHFS_METADATA_SIZE) && + ((*inode_table = realloc(*inode_table, size += SQUASHFS_METADATA_SIZE)) == NULL)) + return FALSE; + TRACE("scan_inode_table: reading block 0x%x\n", start); + if((bytes += read_block(fd, start, &start, *inode_table + bytes, sBlk)) == 0) { + free(*inode_table); + return FALSE; + } + } + + /* + * Read last inode entry which is the root directory inode, and obtain the last + * directory start block index. This is used when calculating the total uncompressed + * directory size. The directory bytes in the last block will be counted as normal. + * + * The root inode is ignored in the inode scan. This ensures there is + * always enough bytes left to read a regular file inode entry + */ + bytes -= sizeof(squashfs_dir_inode_header); + if(swap) { + squashfs_dir_inode_header sinode; + memcpy((void *)&sinode, (void *)(*inode_table + bytes), sizeof(*dir_inode)); + SQUASHFS_SWAP_DIR_INODE_HEADER(dir_inode, &sinode); + } else + memcpy((void *)dir_inode, (void *)(*inode_table + bytes), sizeof(*dir_inode)); + directory_start_block = dir_inode->start_block; + + for(cur_ptr = *inode_table; cur_ptr < *inode_table + bytes; files ++) { + if(swap) { + squashfs_reg_inode_header sinode; + memcpy((void *)&sinode, (void *)cur_ptr, sizeof(inode)); + SQUASHFS_SWAP_REG_INODE_HEADER(&inode, &sinode); + } else + memcpy((void *)&inode, (void *)cur_ptr, sizeof(inode)); + + TRACE("scan_inode_table: processing inode @ byte position 0x%x, type 0x%x\n", cur_ptr - *inode_table, + inode.inode_type); + switch(inode.inode_type) { + case SQUASHFS_FILE_TYPE: { + int frag_bytes = inode.fragment == SQUASHFS_INVALID_BLK ? 0 : inode.file_size % sBlk->block_size; + int blocks = inode.fragment == SQUASHFS_INVALID_BLK ? (inode.file_size + + sBlk->block_size - 1) >> sBlk->block_log : inode.file_size >> + sBlk->block_log; + int file_bytes = 0, i, start = inode.start_block; + unsigned int block_list[blocks]; + + TRACE("scan_inode_table: regular file, file_size %d, blocks %d\n", inode.file_size, blocks); + + cur_ptr += sizeof(inode); + if(swap) { + unsigned int sblock_list[blocks]; + memcpy((void *)sblock_list, (void *)cur_ptr, blocks * sizeof(unsigned int)); + SQUASHFS_SWAP_INTS(block_list, sblock_list, blocks); + } else + memcpy((void *)block_list, (void *)cur_ptr, blocks * sizeof(unsigned int)); + + *uncompressed_file += inode.file_size; + (*file_count) ++; + + for(i = 0; i < blocks; i++) + file_bytes += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); + + add_file(start, file_bytes, block_list, blocks, inode.fragment, inode.offset, frag_bytes); + cur_ptr += blocks * sizeof(unsigned int); + break; + } + case SQUASHFS_SYMLINK_TYPE: { + squashfs_symlink_inode_header inodep; + + if(swap) { + squashfs_symlink_inode_header sinodep; + memcpy((void *)&sinodep, (void *)cur_ptr, sizeof(sinodep)); + SQUASHFS_SWAP_SYMLINK_INODE_HEADER(&inodep, &sinodep); + } else + memcpy((void *)&inodep, (void *)cur_ptr, sizeof(inodep)); + (*sym_count) ++; + cur_ptr += sizeof(inodep) + inodep.symlink_size; + break; + } + case SQUASHFS_DIR_TYPE: { + squashfs_dir_inode_header dir_inode; + + if(swap) { + squashfs_dir_inode_header sinode; + memcpy((void *)&sinode, (void *)cur_ptr, sizeof(dir_inode)); + SQUASHFS_SWAP_DIR_INODE_HEADER(&dir_inode, &sinode); + } else + memcpy((void *)&dir_inode, (void *)cur_ptr, sizeof(dir_inode)); + if(dir_inode.start_block < directory_start_block) + *uncompressed_directory += dir_inode.file_size; + (*dir_count) ++; + cur_ptr += sizeof(squashfs_dir_inode_header); + break; + } + case SQUASHFS_BLKDEV_TYPE: + case SQUASHFS_CHRDEV_TYPE: + (*dev_count) ++; + cur_ptr += sizeof(squashfs_dev_inode_header); + break; + + case SQUASHFS_FIFO_TYPE: + (*fifo_count) ++; + cur_ptr += sizeof(squashfs_ipc_inode_header); + break; + case SQUASHFS_SOCKET_TYPE: + (*sock_count) ++; + cur_ptr += sizeof(squashfs_ipc_inode_header); + break; + default: + ERROR("Unknown inode type %d in scan_inode_table!\n", inode.inode_type); + free(*inode_table); + return FALSE; + } + } + + return files; +} + + +int read_super(int fd, squashfs_super_block *sBlk, int *be, char *source) +{ + read_bytes(fd, SQUASHFS_START, sizeof(squashfs_super_block), (char *) sBlk); + + /* Check it is a SQUASHFS superblock */ + swap = 0; + if(sBlk->s_magic != SQUASHFS_MAGIC) { + if(sBlk->s_magic == SQUASHFS_MAGIC_SWAP) { + squashfs_super_block sblk; + ERROR("Reading a different endian SQUASHFS filesystem on %s - ignoring -le/-be options\n", source); + SQUASHFS_SWAP_SUPER_BLOCK(&sblk, sBlk); + memcpy(sBlk, &sblk, sizeof(squashfs_super_block)); + swap = 1; + } else { + ERROR("Can't find a SQUASHFS superblock on %s\n", source); + goto failed_mount; + } + } + + /* Check the MAJOR & MINOR versions */ + if(sBlk->s_major != SQUASHFS_MAJOR || sBlk->s_minor > SQUASHFS_MINOR) { + if(sBlk->s_major == 1) + ERROR("Filesystem on %s is a SQUASHFS 1.x filesystem. Appending\nto SQUASHFS 1.x filesystems is not supported. Please convert it to a SQUASHFS 2.0 filesystem...n", source); + else + ERROR("Major/Minor mismatch, filesystem on %s is (%d:%d), I support (%d: <= %d)\n", + source, sBlk->s_major, sBlk->s_minor, SQUASHFS_MAJOR, SQUASHFS_MINOR); + goto failed_mount; + } + +#if __BYTE_ORDER == __BIG_ENDIAN + *be = !swap; +#else + *be = swap; +#endif + + printf("Found a valid SQUASHFS superblock on %s.\n", source); + printf("\tInodes are %scompressed\n", SQUASHFS_UNCOMPRESSED_INODES(sBlk->flags) ? "un" : ""); + printf("\tData is %scompressed\n", SQUASHFS_UNCOMPRESSED_DATA(sBlk->flags) ? "un" : ""); + printf("\tFragments are %scompressed\n", SQUASHFS_UNCOMPRESSED_FRAGMENTS(sBlk->flags) ? "un" : ""); + printf("\tCheck data is %s present in the filesystem\n", SQUASHFS_CHECK_DATA(sBlk->flags) ? "" : "not"); + printf("\tFragments are %s present in the filesystem\n", SQUASHFS_NO_FRAGMENTS(sBlk->flags) ? "not" : ""); + printf("\tAlways_use_fragments option is %s specified\n", SQUASHFS_ALWAYS_FRAGMENTS(sBlk->flags) ? "" : "not"); + printf("\tDuplicates are %s removed\n", SQUASHFS_DUPLICATES(sBlk->flags) ? "" : "not"); + printf("\tFilesystem size %d bytes\n", sBlk->bytes_used); + printf("\tBlock size %d\n", sBlk->block_size); + printf("\tNumber of fragments %d\n", sBlk->fragments); + printf("\tNumber of inodes %d\n", sBlk->inodes); + printf("\tNumber of uids %d\n", sBlk->no_uids); + printf("\tNumber of gids %d\n", sBlk->no_guids); + TRACE("sBlk->inode_table_start %x\n", sBlk->inode_table_start); + TRACE("sBlk->directory_table_start %x\n", sBlk->directory_table_start); + TRACE("sBlk->uid_start %x\n", sBlk->uid_start); + TRACE("sBlk->fragment_table_start %x\n", sBlk->fragment_table_start); + printf("\n"); + + return TRUE; + +failed_mount: + return FALSE; +} + + +unsigned char *squashfs_readdir(int fd, int root_entries, int start, int offset, int size, squashfs_super_block *sBlk, + void (push_directory_entry)(char *, squashfs_inode, int)) +{ + squashfs_dir_header dirh; + char buffer[sizeof(squashfs_dir_entry) + SQUASHFS_NAME_LEN + 1]; + squashfs_dir_entry *dire = (squashfs_dir_entry *) buffer; + unsigned char *directory_table = NULL; + int bytes = 0, dir_count; + + size += offset; + if((directory_table = malloc((size + SQUASHFS_METADATA_SIZE * 2 - 1) & ~(SQUASHFS_METADATA_SIZE - 1))) == NULL) + return NULL; + while(bytes < size) { + TRACE("squashfs_readdir: reading block 0x%x, bytes read so far %d\n", start, bytes); + if((bytes += read_block(fd, start, &start, directory_table + bytes, sBlk)) == 0) { + free(directory_table); + return NULL; + } + } + + if(!root_entries) + goto all_done; + + bytes = offset; + while(bytes < size) { + if(swap) { + squashfs_dir_header sdirh; + memcpy((void *)&sdirh, directory_table + bytes, sizeof(sdirh)); + SQUASHFS_SWAP_DIR_HEADER(&dirh, &sdirh); + } else + memcpy((void *)&dirh, directory_table + bytes, sizeof(dirh)); + + dir_count = dirh.count + 1; + TRACE("squashfs_readdir: Read directory header @ byte position 0x%x, 0x%x directory entries\n", bytes, dir_count); + bytes += sizeof(dirh); + + while(dir_count--) { + if(swap) { + squashfs_dir_entry sdire; + memcpy((void *)&sdire, directory_table + bytes, sizeof(sdire)); + SQUASHFS_SWAP_DIR_ENTRY(dire, &sdire); + } else + memcpy((void *)dire, directory_table + bytes, sizeof(dire)); + bytes += sizeof(*dire); + + memcpy((void *)dire->name, directory_table + bytes, dire->size + 1); + dire->name[dire->size + 1] = '\0'; + TRACE("squashfs_readdir: pushing directory entry %s, inode %x:%x, type 0x%x\n", dire->name, dirh.start_block, dire->offset, dire->type); + push_directory_entry(dire->name, SQUASHFS_MKINODE(dirh.start_block, dire->offset), dire->type); + bytes += dire->size + 1; + } + } + +all_done: + return directory_table; +} + + +int read_fragment_table(int fd, squashfs_super_block *sBlk, squashfs_fragment_entry **fragment_table) +{ + int i, indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk->fragments); + squashfs_fragment_index fragment_table_index[indexes]; + + TRACE("read_fragment_table: %d fragments, reading %d fragment indexes from 0x%x\n", sBlk->fragments, indexes, sBlk->fragment_table_start); + if(sBlk->fragments == 0) + return 1; + + if((*fragment_table = (squashfs_fragment_entry *) malloc(sBlk->fragments * sizeof(squashfs_fragment_entry))) == NULL) { + ERROR("Failed to allocate fragment table\n"); + return 0; + } + + if(swap) { + squashfs_fragment_index sfragment_table_index[indexes]; + + read_bytes(fd, sBlk->fragment_table_start, SQUASHFS_FRAGMENT_INDEX_BYTES(sBlk->fragments), (char *) sfragment_table_index); + SQUASHFS_SWAP_FRAGMENT_INDEXES(fragment_table_index, sfragment_table_index, indexes); + } else + read_bytes(fd, sBlk->fragment_table_start, SQUASHFS_FRAGMENT_INDEX_BYTES(sBlk->fragments), (char *) fragment_table_index); + + for(i = 0; i < indexes; i++) { + int length = read_block(fd, fragment_table_index[i], NULL, ((char *) *fragment_table) + (i * SQUASHFS_METADATA_SIZE), sBlk); + TRACE("Read fragment table block %d, from 0x%x, length %d\n", i, fragment_table_index[i], length); + } + + + if(swap) { + squashfs_fragment_entry sfragment; + for(i = 0; i < sBlk->fragments; i++) { + SQUASHFS_SWAP_FRAGMENT_ENTRY((&sfragment), (&(*fragment_table)[i])); + memcpy((char *) &(*fragment_table)[i], (char *) &sfragment, sizeof(squashfs_fragment_entry)); + } + } + + return 1; +} + + +int read_filesystem(char *root_name, int fd, squashfs_super_block *sBlk, char **cinode_table, int *inode_bytes, + char **data_cache, int *cache_bytes, int *cache_size, + char **cdirectory_table, int *directory_bytes, + char **directory_data_cache, int *directory_cache_bytes, int *directory_cache_size, + int *file_count, int *sym_count, int *dev_count, int *dir_count, int *fifo_count, int *sock_count, + squashfs_uid *uids, unsigned short *uid_count, squashfs_uid *guids, unsigned short *guid_count, + unsigned int *uncompressed_file, unsigned int *uncompressed_inode, unsigned int *uncompressed_directory, + void (push_directory_entry)(char *, squashfs_inode, int), squashfs_fragment_entry **fragment_table) +{ + unsigned char *inode_table = NULL, *directory_table; + unsigned int start = sBlk->inode_table_start, end = sBlk->directory_table_start, + root_inode_start = start + SQUASHFS_INODE_BLK(sBlk->root_inode), + root_inode_offset = SQUASHFS_INODE_OFFSET(sBlk->root_inode), root_directory_offset; + squashfs_dir_inode_header inode; + unsigned int files, root_inode_block; + + printf("Scanning existing filesystem...\n"); + + if(read_fragment_table(fd, sBlk, fragment_table) == 0) + goto error; + + if((files = scan_inode_table(fd, start, end, root_inode_start, sBlk, &inode, &inode_table, &root_inode_block, + uncompressed_file, uncompressed_directory, file_count, sym_count, dev_count, dir_count, fifo_count, sock_count)) == 0) { + ERROR("read_filesystem: inode table read failed\n"); + goto error; + } + + *uncompressed_inode = root_inode_block; + + printf("Read existing filesystem, %d inodes scanned\n", files); + + if(inode.inode_type == SQUASHFS_DIR_TYPE) { + if((directory_table = squashfs_readdir(fd, !root_name, sBlk->directory_table_start + inode.start_block, + inode.offset, inode.file_size, sBlk, push_directory_entry)) == NULL) { + ERROR("read_filesystem: Could not read root directory\n"); + goto error; + } + + if((*cinode_table = (char *) malloc(root_inode_start - start)) == NULL) { + ERROR("read_filesystem: failed to alloc space for existing filesystem inode table\n"); + goto error; + } + + read_bytes(fd, start, root_inode_start - start, *cinode_table); + + if((*cdirectory_table = (char *) malloc(inode.start_block)) == NULL) { + ERROR("read_filesystem: failed to alloc space for existing filesystem inode table\n"); + goto error; + } + + read_bytes(fd, sBlk->directory_table_start, inode.start_block, *cdirectory_table); + + *inode_bytes = root_inode_start - start; + *directory_bytes = inode.start_block; + + root_inode_offset += sizeof(inode); + root_directory_offset = inode.offset + inode.file_size; + (*dir_count) ++; + + if(((*data_cache = (char *) malloc(root_inode_offset)) == NULL) || + ((*directory_data_cache = (char *) malloc(root_directory_offset)) == NULL)) { + ERROR("read_filesystem: failed to alloc inode/directory caches\n"); + goto error; + } + + memcpy(*data_cache, inode_table + root_inode_block, root_inode_offset); + memcpy(*directory_data_cache, directory_table, root_directory_offset); + *cache_size = root_inode_offset; + *directory_cache_size = root_directory_offset; + + if(root_name) { + push_directory_entry(root_name, sBlk->root_inode, SQUASHFS_DIR_TYPE); + *cache_bytes = root_inode_offset; + *directory_cache_bytes = root_directory_offset; + } else { + *cache_bytes = root_inode_offset - sizeof(inode); + *directory_cache_bytes = inode.offset; + } + + if(!swap) + read_bytes(fd, sBlk->uid_start, sBlk->no_uids * sizeof(squashfs_uid), (char *) uids); + else { + squashfs_uid uids_copy[sBlk->no_uids]; + + read_bytes(fd, sBlk->uid_start, sBlk->no_uids * sizeof(squashfs_uid), (char *) uids_copy); + SQUASHFS_SWAP_DATA(uids, uids_copy, sBlk->no_uids, sizeof(squashfs_uid) * 8); + } + + if(!swap) + read_bytes(fd, sBlk->guid_start, sBlk->no_guids * sizeof(squashfs_uid), (char *) guids); + else { + squashfs_uid guids_copy[sBlk->no_guids]; + + read_bytes(fd, sBlk->guid_start, sBlk->no_guids * sizeof(squashfs_uid), (char *) guids_copy); + SQUASHFS_SWAP_DATA(guids, guids_copy, sBlk->no_guids, sizeof(squashfs_uid) * 8); + } + *uid_count = sBlk->no_uids; + *guid_count = sBlk->no_guids; + + free(inode_table); + free(directory_table); + return sBlk->inode_table_start; + } + +error: + return 0; +} diff --git a/hostTools/squashfs/read_fs.h b/hostTools/squashfs/read_fs.h new file mode 100644 index 0000000..fcc39c3 --- /dev/null +++ b/hostTools/squashfs/read_fs.h @@ -0,0 +1,48 @@ +/* + * macros to convert each packed bitfield structure from little endian to big + * endian and vice versa. These are needed when creating a filesystem on a + * machine with different byte ordering to the target architecture. + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * mksquashfs.h + * + */ + +/* + * macros used to swap each structure entry, taking into account + * bitfields and different bitfield placing conventions on differing architectures + */ +#if __BYTE_ORDER == __BIG_ENDIAN + /* convert from big endian to little endian */ +#define SQUASHFS_SWAP(value, p, pos, tbits) _SQUASHFS_SWAP(value, p, pos, tbits, b_pos) +#else + /* convert from little endian to big endian */ +#define SQUASHFS_SWAP(value, p, pos, tbits) _SQUASHFS_SWAP(value, p, pos, tbits, 64 - tbits - b_pos) +#endif + +#define _SQUASHFS_SWAP(value, p, pos, tbits, SHIFT) {\ + int bits;\ + int b_pos = pos % 8;\ + unsigned long long val = 0;\ + unsigned char *s = (unsigned char *)p + (pos / 8);\ + unsigned char *d = ((unsigned char *) &val) + 7;\ + for(bits = 0; bits < (tbits + b_pos); bits += 8) \ + *d-- = *s++;\ + value = (val >> (SHIFT))/* & ((1 << tbits) - 1)*/;\ +} +#define SQUASHFS_MEMSET(s, d, n) memset(s, 0, n); diff --git a/hostTools/squashfs/sort.c b/hostTools/squashfs/sort.c new file mode 100644 index 0000000..b537128 --- /dev/null +++ b/hostTools/squashfs/sort.c @@ -0,0 +1,304 @@ +/* + * Create a squashfs filesystem. This is a highly compressed read only filesystem. + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * sort.c + */ + +#define TRUE 1 +#define FALSE 0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef SQUASHFS_TRACE +#define TRACE(s, args...) printf("mksquashfs: "s, ## args) +#else +#define TRACE(s, args...) +#endif + +#define INFO(s, args...) if(!silent) printf("mksquashfs: "s, ## args) +#define ERROR(s, args...) fprintf(stderr, s, ## args) +#define EXIT_MKSQUASHFS() exit(1) +#define BAD_ERROR(s, args...) {\ + fprintf(stderr, "FATAL ERROR:" s, ##args);\ + EXIT_MKSQUASHFS();\ + } + +struct sort_info { + dev_t st_dev; + ino_t st_ino; + int priority; + struct sort_info *next; +}; + +struct sort_info *sort_info_list[65536]; + +struct priority_entry { + char *filename; + long long size; + ino_t st_ino; + dev_t st_dev; + struct priority_entry *next; +}; + +struct priority_entry *priority_list[65536]; + +struct sorted_inode_entry { + squashfs_inode inode; + ino_t st_ino; + dev_t st_dev; + struct sorted_inode_entry *next; +}; + +struct sorted_inode_entry *sorted_inode_list[65536]; + +extern int silent; +extern int excluded(char *filename, struct stat *buf); +extern squashfs_inode write_file(char *filename, long long size, int *c_size); + + +int add_to_sorted_inode_list(squashfs_inode inode, dev_t st_dev, ino_t st_ino) +{ + int hash = st_ino & 0xffff; + struct sorted_inode_entry *new_sorted_inode_entry; + + if((new_sorted_inode_entry = malloc(sizeof(struct sorted_inode_entry))) == NULL) { + ERROR("Out of memory allocating sorted inode entry\n"); + return FALSE; + } + + new_sorted_inode_entry->inode = inode; + new_sorted_inode_entry->st_ino = st_ino; + new_sorted_inode_entry->st_dev = st_dev; + new_sorted_inode_entry->next = sorted_inode_list[hash]; + sorted_inode_list[hash] = new_sorted_inode_entry; + + return TRUE; +} + + +squashfs_inode get_sorted_inode(struct stat *buf) +{ + int hash = buf->st_ino & 0xffff; + struct sorted_inode_entry *sorted_inode_entry; + + for(sorted_inode_entry = sorted_inode_list[hash]; sorted_inode_entry; sorted_inode_entry = sorted_inode_entry->next) + if(buf->st_ino == sorted_inode_entry->st_ino && buf->st_dev == sorted_inode_entry->st_dev) + break; + + if(sorted_inode_entry) + return sorted_inode_entry->inode; + else + return (squashfs_inode) 0; +} + + +int add_priority_list(char *filename, struct stat *buf, int priority) +{ + struct priority_entry *new_priority_entry; + + priority += 32768; + if((new_priority_entry = malloc(sizeof(struct priority_entry))) == NULL) { + ERROR("Out of memory allocating priority entry\n"); + return FALSE; + } + + new_priority_entry->filename = strdup(filename); + new_priority_entry->size = buf->st_size; + new_priority_entry->st_dev = buf->st_dev; + new_priority_entry->st_ino = buf->st_ino; + new_priority_entry->next = priority_list[priority]; + priority_list[priority] = new_priority_entry; + return TRUE; +} + + +int get_priority(char *filename, struct stat *buf, int priority) +{ + int hash = buf->st_ino & 0xffff; + struct sort_info *s; + + for(s = sort_info_list[hash]; s; s = s->next) + if((s->st_dev == buf->st_dev) && (s->st_ino == buf->st_ino)) { + TRACE("returning priority %d (%s)\n", s->priority, filename); + return s->priority; + } + TRACE("returning priority %d (%s)\n", priority, filename); + return priority; +} + + +#define ADD_ENTRY(buf, priority) {\ + int hash = buf.st_ino & 0xffff;\ + struct sort_info *s;\ + if((s = malloc(sizeof(struct sort_info))) == NULL) {\ + ERROR("Out of memory allocating sort list entry\n");\ + return FALSE;\ + }\ + s->st_dev = buf.st_dev;\ + s->st_ino = buf.st_ino;\ + s->priority = priority;\ + s->next = sort_info_list[hash];\ + sort_info_list[hash] = s;\ + } +int add_sort_list(char *path, int priority, int source, char *source_path[]) +{ + int i; + char buffer[4096], filename[4096]; + struct stat buf; + + TRACE("add_sort_list: filename %s, priority %d\n", path, priority); + if(strlen(path) > 1 && strcmp(path + strlen(path) - 2, "/*") == 0) + path[strlen(path) - 2] = '\0'; + + TRACE("add_sort_list: filename %s, priority %d\n", path, priority); + if(path[0] == '/' || strncmp(path, "./", 2) == 0 || strncmp(path, "../", 3) == 0) { + if(lstat(path, &buf) == -1) { + sprintf(buffer, "Cannot stat sort_list dir/file %s, ignoring", path); + perror(buffer); + return TRUE; + } + TRACE("adding filename %s, priority %d, st_dev %Lx, st_ino %Lx\n", path, priority, buf.st_dev, buf.st_ino); + ADD_ENTRY(buf, priority); + return TRUE; + } + + for(i = 0; i < source; i++) { + strcat(strcat(strcpy(filename, source_path[i]), "/"), path); + if(lstat(filename, &buf) == -1) { + if(!(errno == ENOENT || errno == ENOTDIR)) { + sprintf(buffer, "Cannot stat sort_list dir/file %s, ignoring", filename); + perror(buffer); + } + continue; + } + ADD_ENTRY(buf, priority); + } + return TRUE; +} + + +void generate_file_priorities(char *pathname, int priority, struct stat *buf) +{ + char filename[8192]; + DIR *linuxdir; + struct dirent *d_name; + + priority = get_priority(pathname, buf, priority); + + if((linuxdir = opendir(pathname)) == NULL) { + ERROR("Could not open %s, skipping...\n", pathname); + return; + } + + while((d_name = readdir(linuxdir)) != NULL) { + if(strcmp(d_name->d_name, ".") == 0 || strcmp(d_name->d_name, "..") == 0) + continue; + strcat(strcat(strcpy(filename, pathname), "/"), d_name->d_name); + + if(lstat(filename, buf) == -1) { + char buffer[8192]; + sprintf(buffer, "Cannot stat dir/file %s, ignoring", filename); + perror(buffer); + continue; + } + + if(excluded(filename, buf)) + continue; + + switch(buf->st_mode & S_IFMT) { + case S_IFREG: + add_priority_list(filename, buf, get_priority(filename, buf, priority)); + break; + case S_IFDIR: + generate_file_priorities(filename, priority, buf); + break; + } + } + + closedir(linuxdir); +} + + +int read_sort_file(char *filename, int source, char *source_path[]) +{ + FILE *fd; + char sort_filename[16385]; + int priority; + + if((fd = fopen(filename, "r")) == NULL) { + perror("Could not open sort_list file..."); + return FALSE; + } + while(fscanf(fd, "%s %d", sort_filename, &priority) != EOF) + if(priority >= -32768 && priority <= 32767) + add_sort_list(sort_filename, priority, source, source_path); + else + ERROR("Sort file %s, priority %d outside range of -32767:32768 - skipping...\n", sort_filename, priority); + fclose(fd); + return TRUE; +} + + +void sort_files_and_write(int source, char *source_path[]) +{ + struct stat buf; + int i, c_size; + struct priority_entry *entry; + squashfs_inode inode; + + for(i = 0; i < source; i++) { + if(lstat(source_path[i], &buf) == -1) { + char buffer[8192]; + sprintf(buffer, "Cannot stat dir/file %s, ignoring", source_path[i]); + perror(buffer); + continue; + } + + if(excluded(source_path[i], &buf)) + continue; + + switch(buf.st_mode & S_IFMT) { + case S_IFREG: + add_priority_list(source_path[i], &buf, get_priority(source_path[i], &buf, 0)); + break; + case S_IFDIR: + generate_file_priorities(source_path[i], 0, &buf); + break; + } + } + + for(i = 0; i < 65536; i++) + for(entry = priority_list[i]; entry; entry = entry->next) { + TRACE("%d: %s\n", i - 32768, entry->filename); + inode = write_file(entry->filename, entry->size, &c_size); + INFO("file %s, size %d bytes, inode 0x%Lx\n", entry->filename, c_size, inode); + INFO("\t%.2f%% of uncompressed file size (%Ld bytes)\n", ((float) c_size / entry->size) * 100.0, entry->size); + add_to_sorted_inode_list(inode, entry->st_dev, entry->st_ino); + } +} diff --git a/hostTools/squashfs/squashfs_fs.h b/hostTools/squashfs/squashfs_fs.h new file mode 100644 index 0000000..d810c59 --- /dev/null +++ b/hostTools/squashfs/squashfs_fs.h @@ -0,0 +1,474 @@ +#ifndef SQUASHFS_FS +#define SQUASHFS_FS +/* + * Squashfs + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * squashfs_fs.h + */ + +#define SQUASHFS_MAJOR 2 +#define SQUASHFS_MINOR 0 +#define SQUASHFS_MAGIC 0x73717368 +#define SQUASHFS_MAGIC_SWAP 0x68737173 +#define SQUASHFS_START 0 + +/* size of metadata (inode and directory) blocks */ +#define SQUASHFS_METADATA_SIZE 8192 +#define SQUASHFS_METADATA_LOG 13 + +/* default size of data blocks */ +#define SQUASHFS_FILE_SIZE 65536 +#define SQUASHFS_FILE_LOG 16 + +#define SQUASHFS_FILE_MAX_SIZE 65536 + +/* Max number of uids and gids */ +#define SQUASHFS_UIDS 256 +#define SQUASHFS_GUIDS 255 + +/* Max length of filename (not 255) */ +#define SQUASHFS_NAME_LEN 256 + +#define SQUASHFS_INVALID ((long long) 0xffffffffffff) +#define SQUASHFS_INVALID_BLK ((long long) 0xffffffff) +#define SQUASHFS_USED_BLK ((long long) 0xfffffffe) + +/* Filesystem flags */ +#define SQUASHFS_NOI 0 +#define SQUASHFS_NOD 1 +#define SQUASHFS_CHECK 2 +#define SQUASHFS_NOF 3 +#define SQUASHFS_NO_FRAG 4 +#define SQUASHFS_ALWAYS_FRAG 5 +#define SQUASHFS_DUPLICATE 6 +#define SQUASHFS_BIT(flag, bit) ((flag >> bit) & 1) +#define SQUASHFS_UNCOMPRESSED_INODES(flags) SQUASHFS_BIT(flags, SQUASHFS_NOI) +#define SQUASHFS_UNCOMPRESSED_DATA(flags) SQUASHFS_BIT(flags, SQUASHFS_NOD) +#define SQUASHFS_UNCOMPRESSED_FRAGMENTS(flags) SQUASHFS_BIT(flags, SQUASHFS_NOF) +#define SQUASHFS_NO_FRAGMENTS(flags) SQUASHFS_BIT(flags, SQUASHFS_NO_FRAG) +#define SQUASHFS_ALWAYS_FRAGMENTS(flags) SQUASHFS_BIT(flags, SQUASHFS_ALWAYS_FRAG) +#define SQUASHFS_DUPLICATES(flags) SQUASHFS_BIT(flags, SQUASHFS_DUPLICATE) +#define SQUASHFS_CHECK_DATA(flags) SQUASHFS_BIT(flags, SQUASHFS_CHECK) +#define SQUASHFS_MKFLAGS(noi, nod, check_data, nof, no_frag, always_frag, duplicate_checking) (noi | (nod << 1) | (check_data << 2) | (nof << 3) | (no_frag << 4) | (always_frag << 5) | (duplicate_checking << 6)) + +/* Max number of types and file types */ +#define SQUASHFS_DIR_TYPE 1 +#define SQUASHFS_FILE_TYPE 2 +#define SQUASHFS_SYMLINK_TYPE 3 +#define SQUASHFS_BLKDEV_TYPE 4 +#define SQUASHFS_CHRDEV_TYPE 5 +#define SQUASHFS_FIFO_TYPE 6 +#define SQUASHFS_SOCKET_TYPE 7 + +/* 1.0 filesystem type definitions */ +#define SQUASHFS_TYPES 5 +#define SQUASHFS_IPC_TYPE 0 + +/* Flag whether block is compressed or uncompressed, bit is set if block is uncompressed */ +#define SQUASHFS_COMPRESSED_BIT (1 << 15) +#define SQUASHFS_COMPRESSED_SIZE(B) (((B) & ~SQUASHFS_COMPRESSED_BIT) ? \ + (B) & ~SQUASHFS_COMPRESSED_BIT : SQUASHFS_COMPRESSED_BIT) + +#define SQUASHFS_COMPRESSED(B) (!((B) & SQUASHFS_COMPRESSED_BIT)) + +#define SQUASHFS_COMPRESSED_BIT_BLOCK (1 << 24) +#define SQUASHFS_COMPRESSED_SIZE_BLOCK(B) (((B) & ~SQUASHFS_COMPRESSED_BIT_BLOCK) ? \ + (B) & ~SQUASHFS_COMPRESSED_BIT_BLOCK : SQUASHFS_COMPRESSED_BIT_BLOCK) + +#define SQUASHFS_COMPRESSED_BLOCK(B) (!((B) & SQUASHFS_COMPRESSED_BIT_BLOCK)) + +/* + * Inode number ops. Inodes consist of a compressed block number, and an uncompressed + * offset within that block + */ +#define SQUASHFS_INODE_BLK(a) ((unsigned int) ((a) >> 16)) +#define SQUASHFS_INODE_OFFSET(a) ((unsigned int) ((a) & 0xffff)) +#define SQUASHFS_MKINODE(A, B) ((squashfs_inode)(((squashfs_inode) (A) << 16)\ + + (B))) + +/* Compute 32 bit VFS inode number from squashfs inode number */ +#define SQUASHFS_MK_VFS_INODE(a, b) ((unsigned int) (((a) << 8) + ((b) >> 2) + 1)) + +/* Translate between VFS mode and squashfs mode */ +#define SQUASHFS_MODE(a) ((a) & 0xfff) + +/* fragment and fragment table defines */ +typedef unsigned int squashfs_fragment_index; +#define SQUASHFS_FRAGMENT_BYTES(A) (A * sizeof(squashfs_fragment_entry)) +#define SQUASHFS_FRAGMENT_INDEX(A) (SQUASHFS_FRAGMENT_BYTES(A) / SQUASHFS_METADATA_SIZE) +#define SQUASHFS_FRAGMENT_INDEX_OFFSET(A) (SQUASHFS_FRAGMENT_BYTES(A) % SQUASHFS_METADATA_SIZE) +#define SQUASHFS_FRAGMENT_INDEXES(A) ((SQUASHFS_FRAGMENT_BYTES(A) + SQUASHFS_METADATA_SIZE - 1) / SQUASHFS_METADATA_SIZE) +#define SQUASHFS_FRAGMENT_INDEX_BYTES(A) (SQUASHFS_FRAGMENT_INDEXES(A) * sizeof(squashfs_fragment_index)) +#define SQUASHFS_CACHED_FRAGMENTS 3 + +/* cached data constants for filesystem */ +#define SQUASHFS_CACHED_BLKS 8 + +#define SQUASHFS_MAX_FILE_SIZE_LOG 32 +#define SQUASHFS_MAX_FILE_SIZE ((long long) 1 << (SQUASHFS_MAX_FILE_SIZE_LOG - 1)) + +#define SQUASHFS_MARKER_BYTE 0xff + +/* + * definitions for structures on disk + */ + +typedef unsigned int squashfs_block; +typedef long long squashfs_inode; + +typedef unsigned int squashfs_uid; + +typedef struct squashfs_super_block { + unsigned int s_magic; + unsigned int inodes; + unsigned int bytes_used; + unsigned int uid_start; + unsigned int guid_start; + unsigned int inode_table_start; + unsigned int directory_table_start; + unsigned int s_major:16; + unsigned int s_minor:16; + unsigned int block_size_1:16; + unsigned int block_log:16; + unsigned int flags:8; + unsigned int no_uids:8; + unsigned int no_guids:8; + unsigned int mkfs_time /* time of filesystem creation */; + squashfs_inode root_inode; + unsigned int block_size; + unsigned int fragments; + unsigned int fragment_table_start; +} __attribute__ ((packed)) squashfs_super_block; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:8; /* index into uid table */ + unsigned int guid:8; /* index into guid table */ +} __attribute__ ((packed)) squashfs_base_inode_header; + +typedef squashfs_base_inode_header squashfs_ipc_inode_header; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:8; /* index into uid table */ + unsigned int guid:8; /* index into guid table */ + unsigned short rdev; +} __attribute__ ((packed)) squashfs_dev_inode_header; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:8; /* index into uid table */ + unsigned int guid:8; /* index into guid table */ + unsigned short symlink_size; + char symlink[0]; +} __attribute__ ((packed)) squashfs_symlink_inode_header; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:8; /* index into uid table */ + unsigned int guid:8; /* index into guid table */ + unsigned int mtime; + squashfs_block start_block; + unsigned int fragment; + unsigned int offset; + unsigned int file_size:SQUASHFS_MAX_FILE_SIZE_LOG; + unsigned short block_list[0]; +} __attribute__ ((packed)) squashfs_reg_inode_header; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:8; /* index into uid table */ + unsigned int guid:8; /* index into guid table */ + unsigned int file_size:19; + unsigned int offset:13; + unsigned int mtime; + unsigned int start_block:24; +} __attribute__ ((packed)) squashfs_dir_inode_header; + +typedef union { + squashfs_base_inode_header base; + squashfs_dev_inode_header dev; + squashfs_symlink_inode_header symlink; + squashfs_reg_inode_header reg; + squashfs_dir_inode_header dir; + squashfs_ipc_inode_header ipc; +} squashfs_inode_header; + +typedef struct { + unsigned int offset:13; + unsigned int type:3; + unsigned int size:8; + char name[0]; +} __attribute__ ((packed)) squashfs_dir_entry; + +typedef struct { + unsigned int count:8; + unsigned int start_block:24; +} __attribute__ ((packed)) squashfs_dir_header; + + +typedef struct { + unsigned int start_block; + unsigned int size; +} __attribute__ ((packed)) squashfs_fragment_entry; + +extern int squashfs_uncompress_block(void *d, int dstlen, void *s, int srclen); +extern int squashfs_uncompress_init(void); +extern int squashfs_uncompress_exit(void); + +/* + * macros to convert each packed bitfield structure from little endian to big + * endian and vice versa. These are needed when creating or using a filesystem on a + * machine with different byte ordering to the target architecture. + * + */ + +#define SQUASHFS_SWAP_SUPER_BLOCK(s, d) {\ + SQUASHFS_MEMSET(s, d, sizeof(squashfs_super_block));\ + SQUASHFS_SWAP((s)->s_magic, d, 0, 32);\ + SQUASHFS_SWAP((s)->inodes, d, 32, 32);\ + SQUASHFS_SWAP((s)->bytes_used, d, 64, 32);\ + SQUASHFS_SWAP((s)->uid_start, d, 96, 32);\ + SQUASHFS_SWAP((s)->guid_start, d, 128, 32);\ + SQUASHFS_SWAP((s)->inode_table_start, d, 160, 32);\ + SQUASHFS_SWAP((s)->directory_table_start, d, 192, 32);\ + SQUASHFS_SWAP((s)->s_major, d, 224, 16);\ + SQUASHFS_SWAP((s)->s_minor, d, 240, 16);\ + SQUASHFS_SWAP((s)->block_size_1, d, 256, 16);\ + SQUASHFS_SWAP((s)->block_log, d, 272, 16);\ + SQUASHFS_SWAP((s)->flags, d, 288, 8);\ + SQUASHFS_SWAP((s)->no_uids, d, 296, 8);\ + SQUASHFS_SWAP((s)->no_guids, d, 304, 8);\ + SQUASHFS_SWAP((s)->mkfs_time, d, 312, 32);\ + SQUASHFS_SWAP((s)->root_inode, d, 344, 64);\ + SQUASHFS_SWAP((s)->block_size, d, 408, 32);\ + SQUASHFS_SWAP((s)->fragments, d, 440, 32);\ + SQUASHFS_SWAP((s)->fragment_table_start, d, 472, 32);\ +} + +#define SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, n) {\ + SQUASHFS_MEMSET(s, d, n);\ + SQUASHFS_SWAP((s)->inode_type, d, 0, 4);\ + SQUASHFS_SWAP((s)->mode, d, 4, 12);\ + SQUASHFS_SWAP((s)->uid, d, 16, 8);\ + SQUASHFS_SWAP((s)->guid, d, 24, 8);\ +} + +#define SQUASHFS_SWAP_IPC_INODE_HEADER(s, d) SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_ipc_inode_header)) + +#define SQUASHFS_SWAP_DEV_INODE_HEADER(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_dev_inode_header));\ + SQUASHFS_SWAP((s)->rdev, d, 32, 16);\ +} + +#define SQUASHFS_SWAP_SYMLINK_INODE_HEADER(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_symlink_inode_header));\ + SQUASHFS_SWAP((s)->symlink_size, d, 32, 16);\ +} + +#define SQUASHFS_SWAP_REG_INODE_HEADER(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_reg_inode_header));\ + SQUASHFS_SWAP((s)->mtime, d, 32, 32);\ + SQUASHFS_SWAP((s)->start_block, d, 64, 32);\ + SQUASHFS_SWAP((s)->fragment, d, 96, 32);\ + SQUASHFS_SWAP((s)->offset, d, 128, 32);\ + SQUASHFS_SWAP((s)->file_size, d, 160, SQUASHFS_MAX_FILE_SIZE_LOG);\ +} + +#define SQUASHFS_SWAP_DIR_INODE_HEADER(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_dir_inode_header));\ + SQUASHFS_SWAP((s)->file_size, d, 32, 19);\ + SQUASHFS_SWAP((s)->offset, d, 51, 13);\ + SQUASHFS_SWAP((s)->mtime, d, 64, 32);\ + SQUASHFS_SWAP((s)->start_block, d, 96, 24);\ +} + +#define SQUASHFS_SWAP_DIR_HEADER(s, d) {\ + SQUASHFS_MEMSET(s, d, sizeof(squashfs_dir_header));\ + SQUASHFS_SWAP((s)->count, d, 0, 8);\ + SQUASHFS_SWAP((s)->start_block, d, 8, 24);\ +} + +#define SQUASHFS_SWAP_DIR_ENTRY(s, d) {\ + SQUASHFS_MEMSET(s, d, sizeof(squashfs_dir_entry));\ + SQUASHFS_SWAP((s)->offset, d, 0, 13);\ + SQUASHFS_SWAP((s)->type, d, 13, 3);\ + SQUASHFS_SWAP((s)->size, d, 16, 8);\ +} + +#define SQUASHFS_SWAP_FRAGMENT_ENTRY(s, d) {\ + SQUASHFS_MEMSET(s, d, sizeof(squashfs_fragment_entry));\ + SQUASHFS_SWAP((s)->start_block, d, 0, 32);\ + SQUASHFS_SWAP((s)->size, d, 32, 32);\ +} + +#define SQUASHFS_SWAP_SHORTS(s, d, n) {\ + int entry;\ + int bit_position;\ + SQUASHFS_MEMSET(s, d, n * 2);\ + for(entry = 0, bit_position = 0; entry < n; entry++, bit_position += 16)\ + SQUASHFS_SWAP(s[entry], d, bit_position, 16);\ +} + +#define SQUASHFS_SWAP_INTS(s, d, n) {\ + int entry;\ + int bit_position;\ + SQUASHFS_MEMSET(s, d, n * 4);\ + for(entry = 0, bit_position = 0; entry < n; entry++, bit_position += 32)\ + SQUASHFS_SWAP(s[entry], d, bit_position, 32);\ +} + +#define SQUASHFS_SWAP_DATA(s, d, n, bits) {\ + int entry;\ + int bit_position;\ + SQUASHFS_MEMSET(s, d, n * bits / 8);\ + for(entry = 0, bit_position = 0; entry < n; entry++, bit_position += bits)\ + SQUASHFS_SWAP(s[entry], d, bit_position, bits);\ +} + +#define SQUASHFS_SWAP_FRAGMENT_INDEXES(s, d, n) SQUASHFS_SWAP_INTS(s, d, n) + +#ifdef SQUASHFS_1_0_COMPATIBILITY +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:4; /* index into uid table */ + unsigned int guid:4; /* index into guid table */ +} __attribute__ ((packed)) squashfs_base_inode_header_1; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:4; /* index into uid table */ + unsigned int guid:4; /* index into guid table */ + unsigned int type:4; + unsigned int offset:4; +} __attribute__ ((packed)) squashfs_ipc_inode_header_1; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:4; /* index into uid table */ + unsigned int guid:4; /* index into guid table */ + unsigned short rdev; +} __attribute__ ((packed)) squashfs_dev_inode_header_1; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:4; /* index into uid table */ + unsigned int guid:4; /* index into guid table */ + unsigned short symlink_size; + char symlink[0]; +} __attribute__ ((packed)) squashfs_symlink_inode_header_1; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:4; /* index into uid table */ + unsigned int guid:4; /* index into guid table */ + unsigned int mtime; + squashfs_block start_block; + unsigned int file_size:SQUASHFS_MAX_FILE_SIZE_LOG; + unsigned short block_list[0]; +} __attribute__ ((packed)) squashfs_reg_inode_header_1; + +typedef struct { + unsigned int inode_type:4; + unsigned int mode:12; /* protection */ + unsigned int uid:4; /* index into uid table */ + unsigned int guid:4; /* index into guid table */ + unsigned int file_size:19; + unsigned int offset:13; + unsigned int mtime; + unsigned int start_block:24; +} __attribute__ ((packed)) squashfs_dir_inode_header_1; + +#define SQUASHFS_SWAP_BASE_INODE_HEADER_1(s, d, n) {\ + SQUASHFS_MEMSET(s, d, n);\ + SQUASHFS_SWAP((s)->inode_type, d, 0, 4);\ + SQUASHFS_SWAP((s)->mode, d, 4, 12);\ + SQUASHFS_SWAP((s)->uid, d, 16, 4);\ + SQUASHFS_SWAP((s)->guid, d, 20, 4);\ +} + +#define SQUASHFS_SWAP_IPC_INODE_HEADER_1(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER_1(s, d, sizeof(squashfs_ipc_inode_header_1));\ + SQUASHFS_SWAP((s)->type, d, 24, 4);\ + SQUASHFS_SWAP((s)->offset, d, 28, 4);\ +} + +#define SQUASHFS_SWAP_DEV_INODE_HEADER_1(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER_1(s, d, sizeof(squashfs_dev_inode_header_1));\ + SQUASHFS_SWAP((s)->rdev, d, 24, 16);\ +} + +#define SQUASHFS_SWAP_SYMLINK_INODE_HEADER_1(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_symlink_inode_header_1));\ + SQUASHFS_SWAP((s)->symlink_size, d, 24, 16);\ +} + +#define SQUASHFS_SWAP_REG_INODE_HEADER_1(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_reg_inode_header_1));\ + SQUASHFS_SWAP((s)->mtime, d, 24, 32);\ + SQUASHFS_SWAP((s)->start_block, d, 56, 32);\ + SQUASHFS_SWAP((s)->file_size, d, 88, SQUASHFS_MAX_FILE_SIZE_LOG);\ +} + +#define SQUASHFS_SWAP_DIR_INODE_HEADER_1(s, d) {\ + SQUASHFS_SWAP_BASE_INODE_HEADER(s, d, sizeof(squashfs_dir_inode_header_1));\ + SQUASHFS_SWAP((s)->file_size, d, 24, 19);\ + SQUASHFS_SWAP((s)->offset, d, 43, 13);\ + SQUASHFS_SWAP((s)->mtime, d, 56, 32);\ + SQUASHFS_SWAP((s)->start_block, d, 88, 24);\ +} +#endif + +#ifdef __KERNEL__ +/* + * macros used to swap each structure entry, taking into account + * bitfields and different bitfield placing conventions on differing architectures + */ +#include +#ifdef __BIG_ENDIAN + /* convert from little endian to big endian */ +#define SQUASHFS_SWAP(value, p, pos, tbits) _SQUASHFS_SWAP(value, p, pos, tbits, b_pos) +#else + /* convert from big endian to little endian */ +#define SQUASHFS_SWAP(value, p, pos, tbits) _SQUASHFS_SWAP(value, p, pos, tbits, 64 - tbits - b_pos) +#endif + +#define _SQUASHFS_SWAP(value, p, pos, tbits, SHIFT) {\ + int bits;\ + int b_pos = pos % 8;\ + unsigned long long val = 0;\ + unsigned char *s = (unsigned char *)p + (pos / 8);\ + unsigned char *d = ((unsigned char *) &val) + 7;\ + for(bits = 0; bits < (tbits + b_pos); bits += 8) \ + *d-- = *s++;\ + value = (val >> (SHIFT))/* & ((1 << tbits) - 1)*/;\ +} +#define SQUASHFS_MEMSET(s, d, n) memset(s, 0, n); +#endif +#endif diff --git a/hostTools/squashfs/squashfs_fs_i.h b/hostTools/squashfs/squashfs_fs_i.h new file mode 100644 index 0000000..cbbec93 --- /dev/null +++ b/hostTools/squashfs/squashfs_fs_i.h @@ -0,0 +1,34 @@ +#ifndef SQUASHFS_FS_I +#define SQUASHFS_FS_I +/* + * Squashfs + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * squashfs_fs_i.h + */ + +typedef struct squashfs_inode_info { + unsigned int start_block; + unsigned int block_list_start; + unsigned int offset; + unsigned int fragment_start_block; + unsigned int fragment_size; + unsigned int fragment_offset; + struct inode vfs_inode; + } squashfs_inode_info; +#endif diff --git a/hostTools/squashfs/squashfs_fs_sb.h b/hostTools/squashfs/squashfs_fs_sb.h new file mode 100644 index 0000000..6ce5e22 --- /dev/null +++ b/hostTools/squashfs/squashfs_fs_sb.h @@ -0,0 +1,65 @@ +#ifndef SQUASHFS_FS_SB +#define SQUASHFS_FS_SB +/* + * Squashfs + * + * Copyright (c) 2002, 2003, 2004 Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * squashfs_fs_sb.h + */ + +#include + +typedef struct { + unsigned int block; + int length; + unsigned int next_index; + char *data; + } squashfs_cache; + +struct squashfs_fragment_cache { + unsigned int block; + int length; + unsigned int locked; + char *data; + }; + +typedef struct squashfs_sb_info { + squashfs_super_block sBlk; + int devblksize; + int devblksize_log2; + int swap; + squashfs_cache *block_cache; + struct squashfs_fragment_cache *fragment; + int next_cache; + int next_fragment; + squashfs_uid *uid; + squashfs_uid *guid; + squashfs_fragment_index *fragment_index; + unsigned int read_size; + char *read_data; + char *read_page; + struct semaphore read_page_mutex; + struct semaphore block_cache_mutex; + struct semaphore fragment_mutex; + wait_queue_head_t waitq; + wait_queue_head_t fragment_wait_queue; + struct inode *(*iget)(struct super_block *s, squashfs_inode inode); + unsigned int (*read_blocklist)(struct inode *inode, int index, int readahead_blks, + char *block_list, char **block_p, unsigned int *bsize); + } squashfs_sb_info; +#endif -- cgit v1.2.3