From ae51cd24c4996fd1a6dabc37b671af8c8ae9522f Mon Sep 17 00:00:00 2001 From: Matthias Blankertz Date: Fri, 24 Apr 2015 22:13:32 +0200 Subject: [PATCH] MmapFile helper; MveDecoder WIP Added an MmapFile helper and changed iffexplore and treexplore to use it. WIP: MveDecoder Misc. changes/fixes --- IffFile.cc | 4 +- IffFile.hh | 24 +--- Makefile | 51 +++---- MveDecoder.cc | 294 +++++++++++++++++++++++++++++++++++++++ MveDecoder.hh | 33 +++++ TreFile.cc | 378 ++++++++++++++++++-------------------------------- TreFile.hh | 72 +++++++--- iffexplore.cc | 41 ++---- mvedecode.cc | 89 ++++++++++++ treexplore.cc | 8 +- util.cc | 87 ++++++++++++ util.hh | 32 +++++ 12 files changed, 761 insertions(+), 352 deletions(-) create mode 100644 MveDecoder.cc create mode 100644 MveDecoder.hh create mode 100644 mvedecode.cc create mode 100644 util.cc create mode 100644 util.hh diff --git a/IffFile.cc b/IffFile.cc index eb5af64..21b5172 100644 --- a/IffFile.cc +++ b/IffFile.cc @@ -31,7 +31,7 @@ static void _printStructure(IffFile::Object const& obj, unsigned level) printf("%s Length %lu (0x%.lx)", obj.getType().c_str(), obj.getSize(), obj.getSize()); if (obj.isForm()) { - auto form = dynamic_cast(obj); + auto& form = dynamic_cast(obj); printf(", Subtype %s\n", form.getSubtype().c_str()); for(auto it = form.childrenBegin();it != form.childrenEnd();++it) _printStructure(*it, level+1); @@ -44,7 +44,7 @@ static void _printStructure(IffFile::Object const& obj, unsigned level) } } -void IffFile::printStructure(unsigned level) +void IffFile::printStructure(unsigned level) const { _printStructure(*root_, level); } diff --git a/IffFile.hh b/IffFile.hh index 46c092e..8fa207e 100644 --- a/IffFile.hh +++ b/IffFile.hh @@ -16,17 +16,11 @@ public: class Object { public: Object(std::string type, char const* base, size_t length); - Object(Object const& copy) - : base_(copy.base_), length_(copy.length_), type_(copy.type_) { - } - + Object(Object const& copy) = delete; + virtual ~Object() { } - virtual Object* copy() const { - return new Object(*this); - } - std::string const& getType() const { return type_; } @@ -58,16 +52,6 @@ public: class Form final : public Object { public: Form(std::string type, char const* base, size_t length); - Form(Form const& copy) - : Object(copy), subtype_(copy.subtype_) { - for(auto& ent : copy.children_) { - children_.push_back(std::unique_ptr(ent->copy())); - } - } - - Form* copy() const override { - return new Form(*this); - } ~Form() {} @@ -183,11 +167,11 @@ public: std::string subtype_; }; - Object const& getRoot() { + Object const& getRoot() const { return *root_; } - void printStructure(unsigned level = 0); + void printStructure(unsigned level = 0) const; private: static std::unique_ptr parseObject(char const* base, size_t length); diff --git a/Makefile b/Makefile index dd040c8..04bc257 100644 --- a/Makefile +++ b/Makefile @@ -2,21 +2,21 @@ CXX=g++ CXXOPTS=-Og -ggdb -Wall -Wextra -pedantic -std=c++14 -flto LDOPTS= -IFFEXPLORE_CXXSRCS=iffexplore.cc IffFile.cc -IFFEXPLORE_OBJS=$(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.o)) +iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc +iffexplore_LIBS ::= -TREEXPLORE_CXXSRCS=treexplore.cc TreFile.cc IffFile.cc -TREEXPLORE_OBJS=$(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.o)) +treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc +treexplore_LIBS ::= -FONT2PNG_CXXSRCS=font2png.cc -FONT2PNG_OBJS=$(addprefix objs/,$(FONT2PNG_CXXSRCS:.cc=.o)) -FONT2PNG_LIBS=-lpng +font2png_CXXSRCS ::= font2png.cc +font2png_LIBS ::= -lpng -PNTR2PNG_CXXSRCS=pntr2png.cc -PNTR2PNG_OBJS=$(addprefix objs/,$(PNTR2PNG_CXXSRCS:.cc=.o)) -PNTR2PNG_LIBS=-lpng +mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc +mvedecode_LIBS ::= -all: iffexplore font2png pntr2png treexplore +progs ::= iffexplore font2png treexplore mvedecode + +all: $(progs) objs/%.o: %.cc $(CXX) $(CXXOPTS) -c -MMD -MP -o $@ $< @@ -25,28 +25,17 @@ objs/%.o: %.cc %.pb.cc %.pb.h: %.proto protoc --cpp_out=. $< -iffexplore: $(IFFEXPLORE_OBJS) - $(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(IFFEXPLORE_OBJS) - -treexplore: $(TREEXPLORE_OBJS) - $(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(TREEXPLORE_OBJS) - -font2png: $(FONT2PNG_OBJS) - $(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(FONT2PNG_OBJS) $(FONT2PNG_LIBS) - -pntr2png: $(PNTR2PNG_OBJS) - $(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(PNTR2PNG_OBJS) $(PNTR2PNG_LIBS) - +$(progs): %: + $(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $^ $($(@)_LIBS) clean: - rm -f iffexplore $(IFFEXPLORE_OBJS) $(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.P)) - rm -f treexplore $(TREEXPLORE_OBJS) $(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.P)) - rm -f font2png $(FONT2PNG_OBJS) $(addprefix objs/,$(FONT2PNG_CXXSRCS:.cc=.P)) - rm -f pntr2png $(PNTR2PNG_OBJS) $(addprefix objs/,$(PNTR2PNG_CXXSRCS:.cc=.P)) + rm -f $(progs)\ + $(foreach prog,$(progs),$(addprefix objs/,$($(prog)_CXXSRCS:.cc=.o)))\ + $(foreach prog,$(progs),$(addprefix objs/,$($(prog)_CXXSRCS:.cc=.P))) .PHONY: clean all --include $(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.P)) --include $(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.P)) --include $(addprefix objs/,$(FONT2PNG_CXXSRCS:.cc=.P)) --include $(addprefix objs/,$(PNTR2PNG_CXXSRCS:.cc=.P)) +-include $(foreach prog,$(progs),$(addprefix objs/,$($(prog)_CXXSRCS:.cc=.P))) + +.SECONDEXPANSION: +$(progs): %: $$(addprefix objs/,$$($$@_CXXSRCS:.cc=.o)) diff --git a/MveDecoder.cc b/MveDecoder.cc new file mode 100644 index 0000000..77638c7 --- /dev/null +++ b/MveDecoder.cc @@ -0,0 +1,294 @@ +#include + +#include "common.hh" +#include "IffFile.hh" +#include "MveDecoder.hh" + +struct PCChunk { + uint32_t unknown[2]; + uint32_t PALTcount; +} __attribute__((__packed__)); + +MveDecoder::MveDecoder(char const* base, size_t length) + : iff_(base, length) +{ + //iff_.printStructure(); + + auto& root = iff_.getRoot(); + if (!root.isForm()) + throw FormatException{"Root node not FORM"}; + + auto& rootForm = dynamic_cast(root); + if (rootForm.getSubtype() != "MOVE") + throw FormatException{"Root form not MOVE"}; + + auto it = rootForm.childrenBegin(); + + // Parse _PC_ chunk + auto& PC = *it++; + if ((PC.getType() != "_PC_") || + (PC.getSize() != 12)) + throw FormatException{"_PC_ chunk missing or wrong size"}; + + PCChunk const* PCc = reinterpret_cast(PC.begin()); +#ifndef NDEBUG + printf("_PC_: %u PALTs\n", PCc->PALTcount); +#endif + + // Parse SOND chunk + auto& SOND = *it++; + if ((SOND.getType() != "SOND") || + (SOND.getSize() != 4)) + throw FormatException{"SOND chunk missing or wrong size"}; + + uint32_t const* SONDc = reinterpret_cast(SOND.begin()); + if (*SONDc != 3) + throw FormatException{"Unexpected SOND value"}; + + // Parse data + int curSHOT = -1; + unsigned curPALT = 0; + std::vector PALTs; + + for (;it != rootForm.childrenEnd();++it) { + if (it->getType() == "PALT") { + if (curPALT > PCc->PALTcount) + throw FormatException{"Number of PALT chunks exceeds amount specified in _PC_"}; + if (it->getSize() != 768) + throw FormatException{"Unexpected PALT size"}; + palts_.emplace_back(); + + std::transform(it->begin(), it->end(), palts_.back().begin(), + [](const char& in) -> uint8_t {return (in << 2) | ((in >> 6)&0x3);}); + ++curPALT; + } else if (it->getType() == "SHOT") { + uint32_t const* SHOTc = reinterpret_cast(it->begin()); + if (*SHOTc > PCc->PALTcount) + throw FormatException{"SHOT refers to PALT outside amount specified in _PC_"}; + ++curSHOT; + Shot shot{palts_.at(*SHOTc), {}, {}}; + shots_.push_back(shot); + } else if (it->getType() == "VGA ") { + if (curSHOT < 0) + throw FormatException{"VGA outside SHOT"}; + shots_.back().VGAs.push_back(parseVGA(*it, shots_.back().VGAs.empty()?nullptr:&shots_.back().VGAs.back())); + } else if (it->getType() == "AUDI") { + if (curSHOT < 0) + throw FormatException{"AUDI outside SHOT"}; + if (it->getSize() != 2940) + throw FormatException{"Unexpected AUDI size"}; + shots_.back().AUDIs.push_back(&*it); + } else if ((it->getType() == "INDX") || + (it->getType() == "BRCH")) { + // Index/branches NYI + } else + throw FormatException{"Encountered unexpected chunk: " + it->getType()}; + } + + unsigned i = 0; + for(auto& shot : shots_) { +#ifndef NDEBUG + printf("Shot %u: Palette %ld, %lu video frames, %lu audio blocks\n", + i++, std::find(palts_.begin(), palts_.end(), shot.palt)-palts_.begin(), + shot.VGAs.size(), shot.AUDIs.size()); +#endif + if (shot.VGAs.size() != shot.AUDIs.size()) + throw FormatException{"Video/audio block count mismatch"}; + } + +} + +char *binify(uint8_t a) { + static char buf[9]; + buf[8] = '\0'; + for(unsigned i = 0;i < 8;++i) { + if(a&0x80) + buf[i] = '1'; + else + buf[i] = '0'; + a <<= 1; + } + + return buf; +} + +std::vector parseHuff_(char const* data, size_t len) +{ + uint8_t numHuffVals; + memcpy(&numHuffVals, data, 1); + if (numHuffVals != 22) + throw FormatException{"Unexpected huffman tree size"}; + + std::array huffTree; + memcpy(huffTree.data(), data+1, 44); + +#ifdef HUFFDEBUG_ + printf("Coded data length: %d\nTree:\n", segOfs[1]-(segOfs[0]+45)); + printf(" 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f\n"); + printf("0x00: "); + for(unsigned i = 0;i < huffTree.size();++i) { + printf("%.2hhx", huffTree[i]); + if ((i+1)%16 == 0) + printf("\n0x%.2hhx: ", (i+1)&0xf0); + else + printf(" "); + } + printf("\n"); +#endif + + int byteValid = 0; + uint8_t byteBuf = 0; + unsigned bytePos = 45; + std::vector commands; + unsigned huffIdx = huffTree.size(); + while (huffIdx != 22) { + // Read next bit + unsigned bit; + if (byteValid) { + bit = byteBuf&0x1; + byteBuf >>= 1; + --byteValid; + } else { + if (bytePos >= len) + throw FormatException{"Huffman stream overrun"}; + memcpy(&byteBuf, data+bytePos, 1); + //printf("Byte: %.2hhx (%s)\n", byteBuf, binify(byteBuf)); + ++bytePos; + bit = byteBuf&0x1; + byteBuf >>= 1; + byteValid = 7; + } + + huffIdx = huffTree.at(huffIdx-(bit?1:23)); + + //printf("step: %d %.2x\n", bit, huffIdx); + + if (huffIdx < 22) { + commands.push_back(huffIdx); + //printf("out: %.2hhx\n", commands.back()); + huffIdx = huffTree.size(); + } + } + + if (commands.empty()) + throw FormatException{"No commands decoded"}; + +#ifdef HUFFDEBUG_ + for(auto ent : commands) + printf("%.2hhx ", ent); + printf("\n"); +#endif + + return commands; +} + +std::vector parsePixels_(char const* data, size_t len) +{ + std::vector ret; + + if (*data == 0x02) { + size_t pos = 1; + while (pos < len) { + uint8_t b = *(data+pos++); + if (!((b&0xe0)==0xe0)) { + unsigned size = 0, replSize = 0; + unsigned replOfs = 0; + if (!(b&0x80)) { + //printf("Code: Repl1 %.2hhx\n", b); + if (pos >= len) + throw FormatException{"Pixel data overrun"}; + + uint8_t ofs = *(data+pos++); + size = b&0x3; + if (pos+size >= len) + throw FormatException{"Pixel data overrun"}; + + replSize = ((b&0x1c)>>2) + 3; + replOfs = ret.size()-(((b&0x60)<<3)+ofs+1)+size; + } else if (!(b&0x40)) { + //printf("Code: Repl3 %.2hhx\n", b); + if (pos+1 >= len) + throw FormatException{"Pixel data overrun"}; + + uint8_t b1 = *(data+pos++); + uint8_t b2 = *(data+pos++); + + size = (b1&0xc0)>>6; + if (pos+size >= len) + throw FormatException{"Pixel data overrun"}; + + replSize = (b&0x3f)+4; + replOfs = ret.size()-(((b1&0x3f)<<8)+b2+1)+size; + } else if (!(b&0x20)) { + //printf("Code: Repl2 %.2hhx\n", b); + if (pos+2 >= len) + throw FormatException{"Pixel data overrun"}; + + uint8_t b1 = *(data+pos++); + uint8_t b2 = *(data+pos++); + uint8_t b3 = *(data+pos++); + + size = b&0x3; + if (pos+size >= len) + throw FormatException{"Pixel data overrun"}; + + replSize = b3+5+((b&0xc)<<6); + replOfs = ret.size()-(((b&0x10)<<12)+1+(b1<<8)+b2)+size; + } + for (unsigned i = 0;i < size;++i) + ret.push_back(*(data+pos++)); + + for (unsigned i = 0;i < replSize;++i) + ret.push_back(ret[replOfs+i]); + } else { + //printf("Code: Copy %.2hhx\n", b); + unsigned size = (b&0x1f)*4+4; + if (size > 0x70) + break; + if (pos+size > len) + throw FormatException{"Pixel data overrun"}; + for (unsigned i = 0;i < size;++i) + ret.push_back(*(data+pos++)); + } + + } + } else + std::copy(data+1, data+len, std::back_inserter(ret)); + + return ret; +} + +MveDecoder::Frame MveDecoder::parseVGA(IffFile::Object const& vga, Frame const* prev) +{ + // Parse header + if (vga.getSize() < 8) + throw FormatException{"VGA chunk smaller than VGA header"}; + + std::array segOfs; + memcpy(segOfs.data(), vga.begin(), 8); + for(unsigned i = 0;i < segOfs.size();++i) { + if (segOfs[i] >= vga.getSize()) + throw FormatException{"Segment offset exceeds chunk"}; + } + + // Parse segment 1 + if ((segOfs[1] - segOfs[0]) < 45) + throw FormatException{"Segment 1 too small"}; + + auto commands = parseHuff_(vga.begin()+segOfs[0], segOfs[1]-segOfs[0]); + + std::vector pixels; + if (segOfs[3] > 0) + pixels = parsePixels_(vga.begin()+segOfs[3], vga.getSize()-segOfs[3]); + + if (pixels.size() == 52800) { + FILEUPtr outFile{fopen("pixel.data", "wb")}; + if (!outFile) + throw POSIXException{errno, "Could not open pixel.data"}; + + if (fwrite(pixels.data(), pixels.size(), 1, outFile.get()) != 1) + throw POSIXException{errno, "Could not write"}; + + // exit(0); + } +} diff --git a/MveDecoder.hh b/MveDecoder.hh new file mode 100644 index 0000000..4109925 --- /dev/null +++ b/MveDecoder.hh @@ -0,0 +1,33 @@ +#ifndef WC3RE_MVEDECODER_HH__ +#define WC3RE_MVEDECODER_HH__ + +#include +#include + +#include "IffFile.hh" + +class MveDecoder { +public: + MveDecoder(char const* base, size_t length); + +private: + IffFile iff_; + + using Palette = std::array; + + struct Frame { + }; + + struct Shot { + Palette const& palt; + std::vector AUDIs; + std::vector VGAs; + }; + + std::vector shots_; + std::vector palts_; + + static Frame parseVGA(IffFile::Object const& vga, Frame const* prev); +}; + +#endif diff --git a/TreFile.cc b/TreFile.cc index fb696b0..9b27b11 100644 --- a/TreFile.cc +++ b/TreFile.cc @@ -1,13 +1,6 @@ #include -#include -#include #include -#include -#include -#include -#include - #include "common.hh" #include "TreFile.hh" @@ -26,51 +19,130 @@ struct Table1Entry { // little endian uint32_t table3Ptr; } __attribute__((__packed__)); -struct Table3Entry { // little endian - unsigned dataPtr:32; - // unsigned unknown1:8; - unsigned length:24; - unsigned unknown2:8; -} __attribute__((__packed__)); - - -TreFile::TreFile(std::string const& filename) - : file_(nullptr), start_(0), length_(0) +TreFile::TreFile(char const* base, size_t length) + : base_(base), length_(length) { - file_ = fopen(filename.c_str(), "rb"); - if (!file_) - throw POSIXException{errno, "Could not open "s + filename}; + if (length_ < sizeof(TreHeader)) + throw FormatException{"Size < header size"}; - { - struct stat statBuf; + TreHeader const* header = reinterpret_cast(base); + + if (memcmp(header->magic, "XTRE", 4) != 0) + throw FormatException{"Wrong magic, not a TRE?"}; + + if ((header->table1Ofs > length_) || + (header->table2Ofs > length_) || + (header->table3Ofs > length_) || + (header->dataOfs > length_)) + throw FormatException{"Table offset exceeds length"}; + + if ((header->table1Ofs > header->table2Ofs) || + (header->table2Ofs > header->table3Ofs) || + (header->table3Ofs > header->dataOfs)) + throw FormatException{"Table offsets not in ascending order"}; + + const size_t numTable1Entries = (header->table2Ofs-header->table1Ofs)/sizeof(Table1Entry); + table3Size_ = (header->dataOfs-header->table3Ofs)/sizeof(Table3Entry); + +#ifndef NDEBUG + printf("Table 1: start %u, length %u (%lu entries)\n", + header->table1Ofs, header->table2Ofs-header->table1Ofs, + numTable1Entries); + printf("Table 2: start %u, length %u\n", + header->table2Ofs, header->table3Ofs-header->table2Ofs); + printf("Table 3: start %u, length %u (%lu entries)\n", + header->table3Ofs, header->dataOfs-header->table3Ofs, + table3Size_); + printf("Data: start %u, length %lu\n", + header->dataOfs, length_ - header->dataOfs); +#endif + + // Check Table 3 + table3_ = reinterpret_cast(base+header->table3Ofs); + for (size_t i = 0;i < table3Size_;++i) { + if ((table3_[i].dataPtr > length_) || + (table3_[i].dataPtr < header->dataOfs)) + throw FormatException{"Data pointer out of range"}; + + if ((table3_[i].dataPtr + table3_[i].length) > length_) { + fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", table3_[i].dataPtr, table3_[i].length); + } + } + + // Read Table 2 + std::map table2Pos; - if (fstat(fileno(file_), &statBuf) != 0) { - fclose(file_); - throw POSIXException(errno, "Could not stat"); + size_t pos = header->table2Ofs; + while((pos+5) < header->table3Ofs) { + uint8_t const* nameLen = reinterpret_cast(base+pos); + + if (pos+*nameLen+5 > header->table3Ofs) + throw FormatException{"Table 2 entry exceeds table "s + std::to_string(*nameLen)}; + + std::string nameStr(base+pos+1, *nameLen); + + uint32_t const* table3Ptr = reinterpret_cast(base+pos+*nameLen+1); + + if ((*table3Ptr < header->table3Ofs) || + (*table3Ptr >= header->dataOfs)) { + throw FormatException{"Table3 pointer out of range"}; } - length_ = statBuf.st_size; + const size_t table3Index = (*table3Ptr - header->table3Ofs)/sizeof(Table3Entry); + + auto ins = table2_.emplace(nameStr, table3Index); + if (!ins.second) { + if (ins.first->second == table3Index) + printf("Duplicate entry for name %s\n", nameStr.c_str()); + else + printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(), + ins.first->second, table3Index); + } + // Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's + table2Pos.emplace(pos, *table3Ptr); + + pos += 1+*nameLen+4; } - try { - construct_(); - } catch(...) { - fclose(file_); - throw; + + // Read Table 1 + Table1Entry const* table1 = reinterpret_cast(base+header->table1Ofs); + for (size_t i = 0;i < numTable1Entries;++i) { + if ((table1[i].crc == 0x00000000) && + (table1[i].table3Ptr == 0xffffffff)) + continue; // Unused entry + + if ((table1[i].table3Ptr < header->table2Ofs) || + (table1[i].table3Ptr >= header->dataOfs)) { + throw FormatException{"Table3 pointer out of range"}; + } + + uint32_t table3Ptr = table1[i].table3Ptr; + if (table3Ptr < header->table3Ofs) { + auto it = table2Pos.find(table3Ptr); + if (it == table2Pos.end()) { + fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n", + table1[i].crc, table3Ptr); + continue; + } + printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n", + table3Ptr, it->second); + table3Ptr = it->second; + } + + const size_t table3Index = (table3Ptr - header->table3Ofs)/sizeof(Table3Entry); + auto ins = table1_.emplace(static_cast(table1[i].crc), table3Index); + if (!ins.second) { + if (ins.first->second == table3Index) + printf("Duplicate entry for CRC %.8x\n", table1[i].crc); + else + printf("Collision for CRC %.8x: prev %lu, new %lu\n", table1[i].crc, ins.first->second, table3Index); + } } } -TreFile::TreFile(FILE* file, off_t start, off_t length) - : file_(file), start_(start), length_(length) -{ - construct_(); -} TreFile::~TreFile() { - if (file_) { - fclose(file_); - file_ = nullptr; - } } std::vector TreFile::getNames() const @@ -93,12 +165,12 @@ std::vector TreFile::getCRCs() const return ret; } -TreFile::File TreFile::openName(std::string const& name) const +TreFile::Object TreFile::openName(std::string const& name) const { return openIdx_(findName_(name)); } -TreFile::File TreFile::openCRC(uint32_t crc) const +TreFile::Object TreFile::openCRC(uint32_t crc) const { return openIdx_(findCRC_(crc)); } @@ -113,7 +185,7 @@ TreFile::Stat TreFile::statCRC(uint32_t crc) const return statIdx_(findCRC_(crc)); } -void TreFile::dumpName(std::string path, std::string const& name) +void TreFile::dumpName(std::string path, std::string const& name) const { auto idx = findName_(name); @@ -124,7 +196,7 @@ void TreFile::dumpName(std::string path, std::string const& name) dumpIdx_(path, idx); } -void TreFile::dumpCRC(std::string path, uint32_t crc) +void TreFile::dumpCRC(std::string path, uint32_t crc) const { auto idx = findCRC_(crc); char crcStr[9]; @@ -137,7 +209,7 @@ void TreFile::dumpCRC(std::string path, uint32_t crc) dumpIdx_(path, idx); } -void TreFile::dumpAll(std::string path) +void TreFile::dumpAll(std::string path) const { if (path.back() != '/') path.push_back('/'); @@ -163,177 +235,22 @@ void TreFile::printStructure() { printf("Table 1: %lu entries\n", table1_.size()); printf("Table 2: %lu entries\n", table2_.size()); - printf("Table 3: %lu entries\n", table3_.size()); + printf("Table 3: %lu entries\n", table3Size_); printf("Files by CRC:\n"); for(auto const& ent : table1_) { printf("\t%.8x -> (ofs %.8x, len %.8x, flags %.2hhx)\n", ent.first, - std::get<0>(table3_[ent.second]), - std::get<1>(table3_[ent.second]), - std::get<2>(table3_[ent.second])); + table3_[ent.second].dataPtr, + table3_[ent.second].length, + table3_[ent.second].flags); } printf("Files by Name:\n"); for(auto const& ent : table2_) { printf("\t%s -> (ofs %.8x, len %.8x, flags %.2hhx)\n", ent.first.c_str(), - std::get<0>(table3_[ent.second]), - std::get<1>(table3_[ent.second]), - std::get<2>(table3_[ent.second])); - } -} - -void TreFile::construct_() -{ - if (length_ < sizeof(TreHeader)) - throw FormatException{"Size < header size"}; - - if (fseeko(file_, start_, SEEK_SET) != 0) { - throw POSIXException(errno, "Could not seek"); - } - - TreHeader header; - if (fread(&header, sizeof(TreHeader), 1, file_) != 1) - throw POSIXException{errno, "Could not read header"}; - - if (memcmp(header.magic, "XTRE", 4) != 0) - throw FormatException{"Wrong magic, not a TRE?"}; - - if ((header.table1Ofs > length_) || - (header.table2Ofs > length_) || - (header.table3Ofs > length_) || - (header.dataOfs > length_)) - throw FormatException{"Table offset exceeds length"}; - - if ((header.table1Ofs > header.table2Ofs) || - (header.table2Ofs > header.table3Ofs) || - (header.table3Ofs > header.dataOfs)) - throw FormatException{"Table offsets not in ascending order"}; - - const size_t numTable1Entries = (header.table2Ofs-header.table1Ofs)/sizeof(Table1Entry); - const size_t numTable3Entries = (header.dataOfs-header.table3Ofs)/sizeof(Table3Entry); - - printf("Table 1: start %u, length %u (%lu entries)\n", - header.table1Ofs, header.table2Ofs-header.table1Ofs, - numTable1Entries); - printf("Table 2: start %u, length %u\n", - header.table2Ofs, header.table3Ofs-header.table2Ofs); - printf("Table 3: start %u, length %u (%lu entries)\n", - header.table3Ofs, header.dataOfs-header.table3Ofs, - numTable3Entries); - printf("Data: start %u, length %lu\n", - header.dataOfs, length_ - header.dataOfs); - - // Read Table 3 - if (fseeko(file_, start_+header.table3Ofs, SEEK_SET) != 0) { - throw POSIXException{errno, "Could not seek"}; - } - - table3_.reserve(numTable3Entries); - for (size_t i = 0;i < numTable3Entries;++i) { - Table3Entry entry; - if (fread(&entry, sizeof(Table3Entry), 1, file_) != 1) - throw POSIXException{errno, "Could not read"}; - - if ((entry.dataPtr > length_) || - (entry.dataPtr < header.dataOfs)) - throw FormatException{"Data pointer out of range"}; - - if ((entry.dataPtr + entry.length) > length_) { - fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", entry.dataPtr, entry.length); - //throw FormatException{"Data length exceeds file length"}; - } - - table3_.push_back(std::make_tuple(static_cast(entry.dataPtr), - static_cast(entry.length), - static_cast(entry.unknown2))); - } - - // Read Table 2 - if (fseeko(file_, start_+header.table2Ofs, SEEK_SET) != 0) { - throw POSIXException{errno, "Could not seek"}; - } - - std::map table2Pos; - - size_t pos = header.table2Ofs; - while((pos+5) < header.table3Ofs) { - uint8_t nameLen; - - if (fread(&nameLen, 1, 1, file_) != 1) - throw POSIXException{errno, "Could not read"}; - - if (pos+nameLen+5 > header.table3Ofs) - throw FormatException{"Table 2 entry exceeds table "s + std::to_string(nameLen)}; - - std::vector nameData(nameLen); - if (fread(nameData.data(), nameLen, 1, file_) != 1) - throw POSIXException{errno, "Could not read"}; - - uint32_t table3Ptr; - if (fread(&table3Ptr, 4, 1, file_) != 1) - throw POSIXException{errno, "Could not read"}; - - if ((table3Ptr < header.table3Ofs) || - (table3Ptr >= header.dataOfs)) { - throw FormatException{"Table3 pointer out of range"}; - } - - std::string nameStr{nameData.data(), nameLen}; - size_t table3Index = (table3Ptr - header.table3Ofs)/sizeof(Table3Entry); - - auto ins = table2_.emplace(nameStr, table3Index); - if (!ins.second) { - if (ins.first->second == table3Index) - printf("Duplicate entry for name %s\n", nameStr.c_str()); - else - printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(), - ins.first->second, table3Index); - } - // Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's - table2Pos.emplace(pos, table3Ptr); - - pos += 1+nameLen+4; - } - - // Read Table 1 - if (fseeko(file_, start_+header.table1Ofs, SEEK_SET) != 0) { - throw POSIXException{errno, "Could not seek"}; - } - - for (size_t i = 0;i < numTable1Entries;++i) { - Table1Entry entry; - if (fread(&entry, sizeof(Table1Entry), 1, file_) != 1) - throw POSIXException{errno, "Could not read"}; - - if ((entry.crc == 0x00000000) && - (entry.table3Ptr == 0xffffffff)) - continue; // Unused entry - - if ((entry.table3Ptr < header.table2Ofs) || - (entry.table3Ptr >= header.dataOfs)) { - throw FormatException{"Table3 pointer out of range"}; - } - - if (entry.table3Ptr < header.table3Ofs) { - auto it = table2Pos.find(entry.table3Ptr); - if (it == table2Pos.end()) { - fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n", - entry.crc, entry.table3Ptr); - continue; - } - printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n", - entry.table3Ptr, it->second); - entry.table3Ptr = it->second; - } - - size_t table3Index = (entry.table3Ptr - header.table3Ofs)/sizeof(Table3Entry); - auto ins = table1_.emplace(static_cast(entry.crc), table3Index); - if (!ins.second) { - if (ins.first->second == table3Index) - printf("Duplicate entry for CRC %.8x\n", entry.crc); - else - printf("Collision for CRC %.8x: prev %lu, new %lu\n", entry.crc, ins.first->second, table3Index); - } + table3_[ent.second].dataPtr, + table3_[ent.second].length, + table3_[ent.second].flags); } } @@ -358,56 +275,37 @@ size_t TreFile::findCRC_(uint32_t crc) const return it->second; } -void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) +void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const { - uint32_t ofs, len; - std::tie(ofs, len, std::ignore) = table3_[table3Idx]; - - // if (ofs%2 != 0) { - // ++ofs; - // len; - // } - - std::vector buf(len); - - if (fseeko(file_, start_+ofs, SEEK_SET) != 0) - throw POSIXException{errno, "Could not seek"}; - - if (fread(buf.data(), len, 1, file_) != 1) - throw POSIXException{errno, "Could not read data"}; + if(table3_[table3Idx].flags&0x80) + throw Exception{"Compressed TRE objects NYI"}; FILEUPtr outFile{fopen(name.c_str(), "wb")}; if (!outFile) throw POSIXException{errno, "Could not open " + name}; - if (fwrite(buf.data(), len, 1, outFile.get()) != 1) + if (fwrite(base_+table3_[table3Idx].dataPtr, table3_[table3Idx].length, 1, outFile.get()) != 1) throw POSIXException{errno, "Could not write"}; } -TreFile::File TreFile::openIdx_(size_t table3Idx) const +TreFile::Object TreFile::openIdx_(size_t table3Idx) const { - uint32_t ofs, len; - uint8_t flags; - std::tie(ofs, len, flags) = table3_[table3Idx]; - - if(flags&0x80) + if(table3_[table3Idx].flags&0x80) throw Exception{"Compressed TRE objects NYI"}; + + if ((table3_[table3Idx].dataPtr + table3_[table3Idx].length) > length_) + throw FormatException{"length exceeds file size"}; - std::vector buf(len); - - if (fseeko(file_, start_+ofs, SEEK_SET) != 0) - throw POSIXException{errno, "Could not seek"}; - - if (fread(buf.data(), len, 1, file_) != 1) - throw POSIXException{errno, "Could not read data"}; - - return File{buf}; + return Object(base_+table3_[table3Idx].dataPtr, + table3_[table3Idx].length); } TreFile::Stat TreFile::statIdx_(size_t table3Idx) const { Stat ret; - std::tie(std::ignore, ret.size, ret.flags) = table3_[table3Idx]; + ret.size = table3_[table3Idx].length; + ret.flags = table3_[table3Idx].flags; + return ret; } diff --git a/TreFile.hh b/TreFile.hh index 295a187..9b2ee58 100644 --- a/TreFile.hh +++ b/TreFile.hh @@ -9,45 +9,65 @@ class TreFile { public: - TreFile(std::string const& filename); - TreFile(FILE* file, off_t start, off_t length); - + TreFile(char const* base, size_t length); + ~TreFile(); - TreFile(TreFile const& copy) = delete; - TreFile& operator=(TreFile const& copy) = delete; - std::vector getNames() const; std::vector getCRCs() const; - void dumpName(std::string path, std::string const& name); - void dumpCRC(std::string path, uint32_t crc); - void dumpAll(std::string path); + void dumpName(std::string path, std::string const& name) const; + void dumpCRC(std::string path, uint32_t crc) const; + void dumpAll(std::string path) const; void printStructure(); - class File { + class Object { public: + Object() : base_(nullptr), length_(0) { + } + + Object(Object const& copy) = delete; + Object(Object && move) + : base_(move.base_), length_(move.length_) { + move.base_ = nullptr; + move.length_ = 0; + } + + Object& operator=(Object const& copy) = delete; + Object& operator=(Object && move) { + base_ = move.base_; + length_ = move.length_; + move.base_ = nullptr; + move.length_ = 0; + return *this; + } + char const* data() const { - return data_.data(); + return base_; } size_t size() const { - return data_.size(); + return length_; } + operator bool() const { + return base_; + } + private: - std::vector data_; + char const* base_; + size_t length_; - File(std::vector data) - : data_(std::move(data)) { + Object(char const* base, size_t length) + : base_(base), length_(length) { } friend class TreFile; }; - File openName(std::string const& name) const; - File openCRC(uint32_t crc) const; + Object openName(std::string const& name) const; + Object openCRC(uint32_t crc) const; struct Stat { uint32_t size; @@ -63,15 +83,23 @@ private: size_t findName_(std::string const& name) const; size_t findCRC_(uint32_t crc) const; - void dumpIdx_(std::string const& name, size_t table3Idx); - File openIdx_(size_t table3Idx) const; + void dumpIdx_(std::string const& name, size_t table3Idx) const; + Object openIdx_(size_t table3Idx) const; Stat statIdx_(size_t table3Idx) const; std::map table1_; std::map table2_; - std::vector > table3_; - mutable FILE* file_; - off_t start_; + + struct Table3Entry { // little endian + unsigned dataPtr:32; + unsigned length:24; + unsigned flags:8; + } __attribute__((__packed__)); + + Table3Entry const* table3_; + size_t table3Size_; + + char const* base_; size_t length_; }; diff --git a/iffexplore.cc b/iffexplore.cc index e3c3011..65b4e9c 100644 --- a/iffexplore.cc +++ b/iffexplore.cc @@ -4,13 +4,10 @@ #include #include -#include -#include -#include #include -#include #include "common.hh" +#include "util.hh" #include "IffFile.hh" void blobDump(IffFile::Object const& obj, std::string const& filename) @@ -31,7 +28,7 @@ void iffDumper(IffFile::Object const& obj, bool dumpBlobs, std::string dumpPath, printf("%s Length %lu (0x%.lx)", obj.getType().c_str(), obj.getSize(), obj.getSize()); if (obj.isForm()) { - auto form = dynamic_cast(obj); + auto& form = dynamic_cast(obj); printf(", Subtype %s\n", form.getSubtype().c_str()); for(auto it = form.childrenBegin();it != form.childrenEnd();++it) iffDumper(*it, dumpBlobs, dumpPath, blobCount, level+1); @@ -89,36 +86,14 @@ int main(int argc, char *argv[]) } try { - FILEUPtr iffFD{fopen(iffFile.c_str(), "rb")}; - if (!iffFD) { - throw POSIXException{errno, "Could not open "s + iffFile}; - } - - struct stat statBuf; - - if (fstat(fileno(iffFD.get()), &statBuf) != 0) { - throw POSIXException(errno, "Could not stat"); - } - - char *mmapBase = static_cast(mmap(nullptr, statBuf.st_size, PROT_READ, MAP_SHARED, fileno(iffFD.get()), 0)); - if (!mmapBase) - throw POSIXException(errno, "mmap failed"); - - try { - IffFile iff{mmapBase, static_cast(statBuf.st_size)}; + MmapFile mmap{iffFile}; + + IffFile iff{mmap.data(), mmap.size()}; - if (printStructure) { - unsigned blobCount = 0; - iffDumper(iff.getRoot(), dumpBlobs, dumpPath, blobCount); - } - } catch(...) { - munmap(mmapBase, statBuf.st_size); - throw; + if (printStructure) { + unsigned blobCount = 0; + iffDumper(iff.getRoot(), dumpBlobs, dumpPath, blobCount); } - - if (munmap(mmapBase, statBuf.st_size) != 0) - fprintf(stderr, "Warning: munmap failed: %s\n", strerror(errno)); - } catch (POSIXException &ex) { fflush(stdout); fprintf(stderr, "%s\n", ex.toString().c_str()); diff --git a/mvedecode.cc b/mvedecode.cc new file mode 100644 index 0000000..d7c5869 --- /dev/null +++ b/mvedecode.cc @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "common.hh" +#include "util.hh" +#include "TreFile.hh" +#include "IffFile.hh" +#include "MveDecoder.hh" + +void usage(char *argv0) { + fprintf(stderr, "Usage: %s [-h] (tre-file name/crc)/iff-file\n", argv0); + fprintf(stderr, "\tAttempt to decode the movie stored in iff-file, or in the\n\tiff-object \"name\"/\"crc\" contained in tre-file\n"); + fprintf(stderr, "\t-h\tPrint this help\n"); +} + +int main(int argc, char *argv[]) { + std::string inFile, objectSpec; + bool useTre = false; + { + int opt; + while ((opt = getopt(argc, argv, "h")) != -1) { + switch (opt) { + case 'h': + usage(argv[0]); + return 0; + default: + usage(argv[0]); + return 1; + } + } + + if (optind >= argc) { + usage(argv[0]); + return 1; + } + + inFile = argv[optind++]; + + if (optind < argc) { + useTre = true; + objectSpec = argv[optind]; + } + } + + try { + MmapFile mmap{inFile}; + + std::unique_ptr tre; + TreFile::Object object; + std::unique_ptr mve; + if (useTre) { + tre = std::make_unique(mmap.data(), mmap.size()); + + // Try to parse objectSpec as CRC + try { + unsigned long CRC = std::stoul(objectSpec, nullptr, 16); + if (CRC <= std::numeric_limits::max()) { + object = tre->openCRC(CRC); + } + } catch (std::invalid_argument &ex) { + } catch (std::out_of_range &ex) { + } + + + if (!object) // Wasn't a CRC, try as name + object = tre->openName(objectSpec); + + mve = std::make_unique(object.data(), object.size()); + } else + mve = std::make_unique(mmap.data(), mmap.size()); + + } catch (POSIXException &ex) { + fflush(stdout); + fprintf(stderr, "%s\n", ex.toString().c_str()); + return 2; + } catch (Exception &ex) { + fflush(stdout); + fprintf(stderr, "%s\n", ex.toString().c_str()); + return 3; + } + + return 0; +} diff --git a/treexplore.cc b/treexplore.cc index cbf69ed..7557905 100644 --- a/treexplore.cc +++ b/treexplore.cc @@ -5,12 +5,10 @@ #include #include -#include -#include -#include #include #include "common.hh" +#include "util.hh" #include "TreFile.hh" #include "IffFile.hh" @@ -62,7 +60,9 @@ int main(int argc, char *argv[]) { } try { - TreFile file{treFile}; + MmapFile mmap{treFile}; + + TreFile file{mmap.data(), mmap.size()}; if (printStructure) file.printStructure(); diff --git a/util.cc b/util.cc new file mode 100644 index 0000000..b73c902 --- /dev/null +++ b/util.cc @@ -0,0 +1,87 @@ +#include + +#include +#include +#include +#include + +#include "common.hh" +#include "util.hh" + +MmapFile::MmapFile(std::string fileName) + : name_(std::move(fileName)), fd_(fopen(name_.c_str(), "rb")), size_(0), base_(nullptr) +{ + if (!fd_) + throw POSIXException{errno, "Could not open "s + name_}; + + struct stat statBuf; + + if (fstat(fileno(fd_.get()), &statBuf) != 0) + throw POSIXException(errno, "Could not stat"); + + + if (statBuf.st_size <= 0) + throw Exception{"File size is zero or less"}; + + size_ = static_cast(statBuf.st_size); + + base_ = mmap(nullptr, size_, PROT_READ, MAP_SHARED, fileno(fd_.get()), 0); + if (!base_) + throw POSIXException(errno, "mmap failed"); +} + +MmapFile::MmapFile(MmapFile && move): + name_(std::move(move.name_)), fd_(std::move(move.fd_)), size_(move.size_), + base_(move.base_) +{ + move.base_ = nullptr; +} + +MmapFile::~MmapFile() +{ + if (base_) { + if (munmap(base_, size_) != 0) + fprintf(stderr, "Warning: munmap failed: %s\n", strerror(errno)); + base_ = nullptr; + } +} + +MmapFile& MmapFile::operator=(MmapFile && move) +{ + fd_ = std::move(move.fd_); + if (base_) + if (munmap(base_, size_) != 0) + fprintf(stderr, "Warning: munmap failed: %s\n", strerror(errno)); + base_ = move.base_; + move.base_ = nullptr; + size_ = move.size_; + name_ = std::move(move.name_); + + return *this; +} + +std::string const& MmapFile::name() const +{ + if (!*this) + throw std::logic_error("name() called on invalid MmapFile"); + return name_; +} + +size_t MmapFile::size() const +{ + if (!*this) + throw std::logic_error("size() called on invalid MmapFile"); + return size_; +} + +char const* MmapFile::data() const +{ + if (!*this) + throw std::logic_error("data() called on invalid MmapFile"); + return static_cast(base_); +} + +MmapFile::operator bool() const +{ + return base_; +} diff --git a/util.hh b/util.hh new file mode 100644 index 0000000..972b2d9 --- /dev/null +++ b/util.hh @@ -0,0 +1,32 @@ +#ifndef WC3RE_UTIL_HH__ +#define WC3RE_UTIL_HH__ + +#include +#include + +#include "common.hh" + +class MmapFile { +public: + MmapFile(std::string fileName); + MmapFile(MmapFile const& copy) = delete; + MmapFile(MmapFile && move); + + ~MmapFile(); + + MmapFile& operator=(MmapFile const& copy) = delete; + MmapFile& operator=(MmapFile && move); + + std::string const& name() const; + size_t size() const; + char const* data() const; + + operator bool() const; +private: + std::string name_; + FILEUPtr fd_; + size_t size_; + void *base_; +}; + +#endif