From a5f64e9bbf2c34b0b07f58d8d01ce97e8e0cad11 Mon Sep 17 00:00:00 2001 From: Matthias Blankertz Date: Fri, 17 Apr 2015 18:31:01 +0200 Subject: [PATCH] treexplore: Add dump option --- .gitignore | 3 +- Makefile | 2 +- TreFile.cc | 326 ++++++++++++++++++++++++++++++++++++++++++++++++++ TreFile.hh | 39 ++++++ treexplore.cc | 296 ++++++++++----------------------------------- 5 files changed, 431 insertions(+), 235 deletions(-) create mode 100644 TreFile.cc create mode 100644 TreFile.hh diff --git a/.gitignore b/.gitignore index 0fc5bfb..9b93e24 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ tmp/ *~ iffexplore font2png - +pntr2png +treexplore diff --git a/Makefile b/Makefile index c7b273f..e5ebddb 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ LDOPTS= IFFEXPLORE_CXXSRCS=iffexplore.cc IFFEXPLORE_OBJS=$(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.o)) -TREEXPLORE_CXXSRCS=treexplore.cc +TREEXPLORE_CXXSRCS=treexplore.cc TreFile.cc TREEXPLORE_OBJS=$(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.o)) FONT2PNG_CXXSRCS=font2png.cc diff --git a/TreFile.cc b/TreFile.cc new file mode 100644 index 0000000..55450cc --- /dev/null +++ b/TreFile.cc @@ -0,0 +1,326 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common.hh" + +#include "TreFile.hh" + +struct TreHeader { // little endian + char magic[4]; // == "XTRE" + uint32_t unknown; + uint32_t table1Ofs; + uint32_t table2Ofs; + uint32_t table3Ofs; + uint32_t dataOfs; +} __attribute__((__packed__)); + +struct Table1Entry { // little endian + uint32_t crc; + uint32_t table3Ptr; +} __attribute__((__packed__)); + +struct Table3Entry { // little endian + unsigned dataPtr:32; + // unsigned unknown1:8; + unsigned length:24; + unsigned unknown2:8; +} __attribute__((__packed__)); + + +TreFile::TreFile(std::string const& filename) + : file_(nullptr), start_(0), length_(0) +{ + file_ = fopen(filename.c_str(), "rb"); + if (!file_) + throw POSIXException{errno, "Could not open "s + filename}; + + { + struct stat statBuf; + + if (fstat(fileno(file_), &statBuf) != 0) { + fclose(file_); + throw POSIXException(errno, "Could not stat"); + } + + length_ = statBuf.st_size; + } + try { + construct_(); + } catch(...) { + fclose(file_); + throw; + } +} + +TreFile::TreFile(FILE* file, off_t start, off_t length) + : file_(file), start_(start), length_(length) +{ + construct_(); +} + +TreFile::~TreFile() +{ + if (file_) { + fclose(file_); + file_ = nullptr; + } +} + +void TreFile::dumpName(std::string path, std::string const& name) +{ + auto it = table2_.find(name); + if (it == table2_.end()) + throw Exception{name + " not found"}; + + if (path.back() != '/') + path.push_back('/'); + path.append(name); + + dumpIdx_(path, it->second); +} + +void TreFile::dumpCRC(std::string path, uint32_t crc) +{ + char crcStr[9]; + snprintf(crcStr, 9, "%.8X", crc); + + auto it = table1_.find(crc); + if (it == table1_.end()) + throw Exception{"CRC "s + crcStr + " not found"s}; + + if (path.back() != '/') + path.push_back('/'); + path.append(crcStr); + + dumpIdx_(path, it->second); +} + +void TreFile::dumpAll(std::string path) +{ + if (path.back() != '/') + path.push_back('/'); + + for (auto ent : table1_) { + char crcStr[9]; + snprintf(crcStr, 9, "%.8X", ent.first); + std::string name = path; + name.append(crcStr); + + dumpIdx_(name, ent.second); + } + + for (auto ent : table2_) { + std::string name = path; + name.append(ent.first); + + dumpIdx_(name, ent.second); + } +} + +void TreFile::printStructure() +{ + printf("Table 1: %lu entries\n", table1_.size()); + printf("Table 2: %lu entries\n", table2_.size()); + printf("Table 3: %lu entries\n", table3_.size()); + + printf("Files by CRC:\n"); + for(auto const& ent : table1_) { + printf("\t%.8x -> (ofs %.8x, len %.8x)\n", ent.first, + std::get<0>(table3_[ent.second]), + std::get<1>(table3_[ent.second])); + } + + printf("Files by Name:\n"); + for(auto const& ent : table2_) { + printf("\t%s -> (ofs %.8x, len %.8x)\n", ent.first.c_str(), + std::get<0>(table3_[ent.second]), + std::get<1>(table3_[ent.second])); + } +} + +void TreFile::construct_() +{ + if (length_ < sizeof(TreHeader)) + throw FormatException{"Size < header size"}; + + if (fseeko(file_, start_, SEEK_SET) != 0) { + throw POSIXException(errno, "Could not seek"); + } + + TreHeader header; + if (fread(&header, sizeof(TreHeader), 1, file_) != 1) + throw POSIXException{errno, "Could not read header"}; + + if (memcmp(header.magic, "XTRE", 4) != 0) + throw FormatException{"Wrong magic, not a TRE?"}; + + if ((header.table1Ofs > length_) || + (header.table2Ofs > length_) || + (header.table3Ofs > length_) || + (header.dataOfs > length_)) + throw FormatException{"Table offset exceeds length"}; + + if ((header.table1Ofs > header.table2Ofs) || + (header.table2Ofs > header.table3Ofs) || + (header.table3Ofs > header.dataOfs)) + throw FormatException{"Table offsets not in ascending order"}; + + const size_t numTable1Entries = (header.table2Ofs-header.table1Ofs)/sizeof(Table1Entry); + const size_t numTable3Entries = (header.dataOfs-header.table3Ofs)/sizeof(Table3Entry); + + printf("Table 1: start %u, length %u (%lu entries)\n", + header.table1Ofs, header.table2Ofs-header.table1Ofs, + numTable1Entries); + printf("Table 2: start %u, length %u\n", + header.table2Ofs, header.table3Ofs-header.table2Ofs); + printf("Table 3: start %u, length %u (%lu entries)\n", + header.table3Ofs, header.dataOfs-header.table3Ofs, + numTable3Entries); + printf("Data: start %u, length %lu\n", + header.dataOfs, length_ - header.dataOfs); + + // Read Table 3 + if (fseeko(file_, start_+header.table3Ofs, SEEK_SET) != 0) { + throw POSIXException{errno, "Could not seek"}; + } + + table3_.reserve(numTable3Entries); + for (size_t i = 0;i < numTable3Entries;++i) { + Table3Entry entry; + if (fread(&entry, sizeof(Table3Entry), 1, file_) != 1) + throw POSIXException{errno, "Could not read"}; + + if ((entry.dataPtr > length_) || + (entry.dataPtr < header.dataOfs)) + throw FormatException{"Data pointer out of range"}; + + if ((entry.dataPtr + entry.length) > length_) { + fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", entry.dataPtr, entry.length); + //throw FormatException{"Data length exceeds file length"}; + } + + table3_.push_back(std::make_tuple(static_cast(entry.dataPtr), + static_cast(entry.length))); + } + + // Read Table 2 + if (fseeko(file_, start_+header.table2Ofs, SEEK_SET) != 0) { + throw POSIXException{errno, "Could not seek"}; + } + + std::map table2Pos; + + size_t pos = header.table2Ofs; + while((pos+5) < header.table3Ofs) { + uint8_t nameLen; + + if (fread(&nameLen, 1, 1, file_) != 1) + throw POSIXException{errno, "Could not read"}; + + if (pos+nameLen+5 > header.table3Ofs) + throw FormatException{"Table 2 entry exceeds table "s + std::to_string(nameLen)}; + + std::vector nameData(nameLen); + if (fread(nameData.data(), nameLen, 1, file_) != 1) + throw POSIXException{errno, "Could not read"}; + + uint32_t table3Ptr; + if (fread(&table3Ptr, 4, 1, file_) != 1) + throw POSIXException{errno, "Could not read"}; + + if ((table3Ptr < header.table3Ofs) || + (table3Ptr >= header.dataOfs)) { + throw FormatException{"Table3 pointer out of range"}; + } + + std::string nameStr{nameData.data(), nameLen}; + size_t table3Index = (table3Ptr - header.table3Ofs)/sizeof(Table3Entry); + + auto ins = table2_.emplace(nameStr, table3Index); + if (!ins.second) { + if (ins.first->second == table3Index) + printf("Duplicate entry for name %s\n", nameStr.c_str()); + else + printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(), + ins.first->second, table3Index); + } + // Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's + table2Pos.emplace(pos, table3Ptr); + + pos += 1+nameLen+4; + } + + // Read Table 1 + if (fseeko(file_, start_+header.table1Ofs, SEEK_SET) != 0) { + throw POSIXException{errno, "Could not seek"}; + } + + for (size_t i = 0;i < numTable1Entries;++i) { + Table1Entry entry; + if (fread(&entry, sizeof(Table1Entry), 1, file_) != 1) + throw POSIXException{errno, "Could not read"}; + + if ((entry.crc == 0x00000000) && + (entry.table3Ptr == 0xffffffff)) + continue; // Unused entry + + if ((entry.table3Ptr < header.table2Ofs) || + (entry.table3Ptr >= header.dataOfs)) { + throw FormatException{"Table3 pointer out of range"}; + } + + if (entry.table3Ptr < header.table3Ofs) { + auto it = table2Pos.find(entry.table3Ptr); + if (it == table2Pos.end()) { + fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n", + entry.crc, entry.table3Ptr); + continue; + } + printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n", + entry.table3Ptr, it->second); + entry.table3Ptr = it->second; + } + + size_t table3Index = (entry.table3Ptr - header.table3Ofs)/sizeof(Table3Entry); + auto ins = table1_.emplace(static_cast(entry.crc), table3Index); + if (!ins.second) { + if (ins.first->second == table3Index) + printf("Duplicate entry for CRC %.8x\n", entry.crc); + else + printf("Collision for CRC %.8x: prev %lu, new %lu\n", entry.crc, ins.first->second, table3Index); + } + } +} + +void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) +{ + uint32_t ofs, len; + std::tie(ofs, len) = table3_[table3Idx]; + + if (ofs%2 != 0) { + ++ofs; + len; + } + + std::vector buf(len); + + if (fseeko(file_, start_+ofs, SEEK_SET) != 0) + throw POSIXException{errno, "Could not seek"}; + + if (fread(buf.data(), len, 1, file_) != 1) + throw POSIXException{errno, "Could not read data"}; + + FILEUPtr outFile{fopen(name.c_str(), "wb")}; + if (!outFile) + throw POSIXException{errno, "Could not open " + name}; + + if (fwrite(buf.data(), len, 1, outFile.get()) != 1) + throw POSIXException{errno, "Could not write"}; +} diff --git a/TreFile.hh b/TreFile.hh new file mode 100644 index 0000000..80643b1 --- /dev/null +++ b/TreFile.hh @@ -0,0 +1,39 @@ +#ifndef WC3RE_TREFILE_HH__ +#define WC3RE_TREFILE_HH__ + +#include +#include +#include +#include +#include + +class TreFile { +public: + TreFile(std::string const& filename); + TreFile(FILE* file, off_t start, off_t length); + + ~TreFile(); + + TreFile(TreFile const& copy) = delete; + TreFile& operator=(TreFile const& copy) = delete; + + void dumpName(std::string path, std::string const& name); + void dumpCRC(std::string path, uint32_t crc); + void dumpAll(std::string path); + + void printStructure(); + +private: + void construct_(); + + void dumpIdx_(std::string const& name, size_t table3Idx); + + std::map table1_; + std::map table2_; + std::vector > table3_; + FILE* file_; + off_t start_; + size_t length_; +}; + +#endif diff --git a/treexplore.cc b/treexplore.cc index 5021487..a05cbfc 100644 --- a/treexplore.cc +++ b/treexplore.cc @@ -3,8 +3,7 @@ #include #include #include -#include -#include +#include #include #include @@ -12,244 +11,75 @@ #include #include "common.hh" +#include "TreFile.hh" -struct TreHeader { // little endian - char magic[4]; // == "XTRE" - uint32_t unknown; - uint32_t table1Ofs; - uint32_t table2Ofs; - uint32_t table3Ofs; - uint32_t dataOfs; -} __attribute__((__packed__)); - -struct Table1Entry { // little endian - uint32_t crc; - uint32_t table3Ptr; -} __attribute__((__packed__)); - -struct Table2EntryHead { - uint8_t len; -} __attribute__((__packed__)); - -struct Table2EntryTail { // little endian - uint32_t table3Ptr; -} __attribute__((__packed__)); - -struct Table3Entry { // little endian - unsigned dataPtr:32; - // unsigned unknown1:8; - unsigned length:24; - unsigned unknown2:8; -} __attribute__((__packed__)); - -class TreFile { -public: - TreFile(std::string const& filename) { - FILEUPtr file{fopen(filename.c_str(), "rb")}; - if (!file) - throw POSIXException{errno, "Could not open "s + filename}; - - struct stat statBuf; - - if (fstat(fileno(file.get()), &statBuf) != 0) - throw POSIXException(errno, "Could not stat"); - - construct_(file.get(), 0, statBuf.st_size); - } - - TreFile(FILE* file, off_t start, off_t length) { - construct_(file, start, length); - } - - void printStructure() { - printf("Table 1: %lu entries\n", table1_.size()); - printf("Table 2: %lu entries\n", table2_.size()); - printf("Table 3: %lu entries\n", table3_.size()); - - printf("Files by CRC:\n"); - for(auto const& ent : table1_) { - printf("\t%.8x -> (ofs %.8x, len %.8x)\n", ent.first, - std::get<0>(table3_[ent.second]), - std::get<1>(table3_[ent.second])); - } - - printf("Files by Name:\n"); - for(auto const& ent : table2_) { - printf("\t%s -> (ofs %.8x, len %.8x)\n", ent.first.c_str(), - std::get<0>(table3_[ent.second]), - std::get<1>(table3_[ent.second])); - } - } - -private: - void construct_(FILE* file, off_t start, size_t length) { - if (length < sizeof(TreHeader)) - throw FormatException{"Size < header size"}; - - if (fseeko(file, start, SEEK_SET) != 0) { - throw POSIXException(errno, "Could not seek"); - } - - TreHeader header; - if (fread(&header, sizeof(TreHeader), 1, file) != 1) - throw POSIXException{errno, "Could not read header"}; - - if (memcmp(header.magic, "XTRE", 4) != 0) - throw FormatException{"Wrong magic, not a TRE?"}; - - if ((header.table1Ofs > length) || - (header.table2Ofs > length) || - (header.table3Ofs > length) || - (header.dataOfs > length)) - throw FormatException{"Table offset exceeds length"}; - - if ((header.table1Ofs > header.table2Ofs) || - (header.table2Ofs > header.table3Ofs) || - (header.table3Ofs > header.dataOfs)) - throw FormatException{"Table offsets not in ascending order"}; - - const size_t numTable1Entries = (header.table2Ofs-header.table1Ofs)/sizeof(Table1Entry); - const size_t numTable3Entries = (header.dataOfs-header.table3Ofs)/sizeof(Table3Entry); - - printf("Table 1: start %u, length %u (%lu entries)\n", - header.table1Ofs, header.table2Ofs-header.table1Ofs, - numTable1Entries); - printf("Table 2: start %u, length %u\n", - header.table2Ofs, header.table3Ofs-header.table2Ofs); - printf("Table 3: start %u, length %u (%lu entries)\n", - header.table3Ofs, header.dataOfs-header.table3Ofs, - numTable3Entries); - printf("Data: start %u, length %lu\n", - header.dataOfs, length - header.dataOfs); - - // Read Table 3 - if (fseeko(file, start+header.table3Ofs, SEEK_SET) != 0) { - throw POSIXException{errno, "Could not seek"}; - } - - table3_.reserve(numTable3Entries); - for (size_t i = 0;i < numTable3Entries;++i) { - Table3Entry entry; - if (fread(&entry, sizeof(Table3Entry), 1, file) != 1) - throw POSIXException{errno, "Could not read"}; - - if ((entry.dataPtr > length) || - (entry.dataPtr < header.dataOfs)) - throw FormatException{"Data pointer out of range"}; - - if ((entry.dataPtr + entry.length) > length) { - fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", entry.dataPtr, entry.length); - //throw FormatException{"Data length exceeds file length"}; - } - - table3_.push_back(std::make_tuple(static_cast(entry.dataPtr), - static_cast(entry.length))); - } - - // Read Table 2 - if (fseeko(file, start+header.table2Ofs, SEEK_SET) != 0) { - throw POSIXException{errno, "Could not seek"}; - } - - std::map table2Pos; - - size_t pos = header.table2Ofs; - while(pos < header.table3Ofs) { - uint8_t nameLen; - - if (fread(&nameLen, 1, 1, file) != 1) - throw POSIXException{errno, "Could not read"}; - - if (pos+nameLen+4 > header.table3Ofs) - throw FormatException{"Table 2 entry exceeds table "s + std::to_string(nameLen)}; - - std::vector nameData(nameLen); - if (fread(nameData.data(), nameLen, 1, file) != 1) - throw POSIXException{errno, "Could not read"}; - - uint32_t table3Ptr; - if (fread(&table3Ptr, 4, 1, file) != 1) - throw POSIXException{errno, "Could not read"}; - - if ((table3Ptr < header.table3Ofs) || - (table3Ptr >= header.dataOfs)) { - throw FormatException{"Table3 pointer out of range"}; - } - - std::string nameStr{nameData.data(), nameLen}; - size_t table3Index = (table3Ptr - header.table3Ofs)/sizeof(Table3Entry); - - auto ins = table2_.emplace(nameStr, table3Index); - if (!ins.second) { - if (ins.first->second == table3Index) - printf("Duplicate entry for name %s\n", nameStr.c_str()); - else - printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(), - ins.first->second, table3Index); - } - // Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's - table2Pos.emplace(pos, table3Ptr); - - pos += 1+nameLen+4; - } - - // Read Table 1 - if (fseeko(file, start+header.table1Ofs, SEEK_SET) != 0) { - throw POSIXException{errno, "Could not seek"}; - } - - for (size_t i = 0;i < numTable1Entries;++i) { - Table1Entry entry; - if (fread(&entry, sizeof(Table1Entry), 1, file) != 1) - throw POSIXException{errno, "Could not read"}; - - if ((entry.crc == 0x00000000) && - (entry.table3Ptr == 0xffffffff)) - continue; // Unused entry - - if ((entry.table3Ptr < header.table2Ofs) || - (entry.table3Ptr >= header.dataOfs)) { - throw FormatException{"Table3 pointer out of range"}; - } - - if (entry.table3Ptr < header.table3Ofs) { - auto it = table2Pos.find(entry.table3Ptr); - if (it == table2Pos.end()) { - fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n", - entry.crc, entry.table3Ptr); - continue; - } - printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n", - entry.table3Ptr, it->second); - entry.table3Ptr = it->second; - } - - size_t table3Index = (entry.table3Ptr - header.table3Ofs)/sizeof(Table3Entry); - auto ins = table1_.emplace(static_cast(entry.crc), table3Index); - if (!ins.second) { - if (ins.first->second == table3Index) - printf("Duplicate entry for CRC %.8x\n", entry.crc); - else - printf("Collision for CRC %.8x: prev %lu, new %lu\n", entry.crc, ins.first->second, table3Index); - } - } - } - - std::map table1_; - std::map table2_; - std::vector > table3_; -}; +void usage(char *argv0) { + fprintf(stderr, "Usage: %s [-sh] [-d dest] tre-file [filenames/crcs...]\n", argv0); + fprintf(stderr, "\t-s\tPrint the tre-file's structure\n"); + fprintf(stderr, "\t-d dest\tDump files to dest/\n"); + fprintf(stderr, "\t\tif dilenames/crcs are supplied, dump those object, else dump all\n"); + fprintf(stderr, "\t-h\tPrint this help\n"); +} int main(int argc, char *argv[]) { - if (argc != 2) { - fprintf(stderr, "Usage: %s tre-file\n", argv[0]); - return 1; + bool printStructure = false, dumpFiles = false; + std::string dumpPath, treFile; + std::vector fileSpecs; + + { + int opt; + while ((opt = getopt(argc, argv, "hsd:")) != -1) { + switch (opt) { + case 'h': + usage(argv[0]); + return 0; + case 's': + printStructure = true; + break; + case 'd': + dumpPath = optarg; + dumpFiles = true; + break; + default: + usage(argv[0]); + return 1; + } + } + + if (optind >= argc) { + usage(argv[0]); + return 1; + } + treFile = argv[optind]; + + for (int i = optind+1;i < argc;++i) + fileSpecs.push_back(argv[i]); } try { - TreFile file{argv[1]}; + TreFile file{treFile}; - file.printStructure(); + if (printStructure) + file.printStructure(); + + if (dumpFiles) { + if (fileSpecs.empty()) + file.dumpAll(dumpPath); + else { + for (auto spec : fileSpecs) { + try { + unsigned long fileCRC = std::stoul(spec, nullptr, 16); + if (fileCRC <= std::numeric_limits::max()) { + file.dumpCRC(dumpPath, fileCRC); + continue; + } + } catch (std::invalid_argument &ex) { + } catch (std::out_of_range &ex) { + } + file.dumpName(dumpPath, spec); + } + } + } } catch (POSIXException &ex) { fprintf(stderr, "%s\n", ex.toString().c_str()); return 2;