matthias/wc3re
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

MmapFile helper; MveDecoder WIP

Browse Source 
Added an MmapFile helper and changed iffexplore and treexplore to use it.
WIP: MveDecoder
Misc. changes/fixes
This commit is contained in:
Matthias Blankertz
2015-04-24 22:13:32 +02:00
parent 89688c5fba
commit ae51cd24c4
12 changed files with 761 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
IffFile.cc
View File

@@ -31,7 +31,7 @@ static void _printStructure(IffFile::Object const& obj, unsigned level)
printf("%s Length %lu (0x%.lx)", obj.getType().c_str(), obj.getSize(), obj.getSize());
if (obj.isForm()) {
auto form = dynamic_cast<IffFile::Form const&>(obj);
auto& form = dynamic_cast<IffFile::Form const&>(obj);
printf(", Subtype %s\n", form.getSubtype().c_str());
for(auto it = form.childrenBegin();it != form.childrenEnd();++it)
_printStructure(*it, level+1);
@@ -44,7 +44,7 @@ static void _printStructure(IffFile::Object const& obj, unsigned level)
}
}
void IffFile::printStructure(unsigned level)
void IffFile::printStructure(unsigned level) const
{
_printStructure(*root_, level);
}

22
IffFile.hh
View File

@@ -16,17 +16,11 @@ public:
class Object {
public:
Object(std::string type, char const* base, size_t length);
Object(Object const& copy)
: base_(copy.base_), length_(copy.length_), type_(copy.type_) {
}
Object(Object const& copy) = delete;
virtual ~Object() {
}
virtual Object* copy() const {
return new Object(*this);
}
std::string const& getType() const {
return type_;
}
@@ -58,16 +52,6 @@ public:
class Form final : public Object {
public:
Form(std::string type, char const* base, size_t length);
Form(Form const& copy)
: Object(copy), subtype_(copy.subtype_) {
for(auto& ent : copy.children_) {
children_.push_back(std::unique_ptr<Object>(ent->copy()));
}
}
Form* copy() const override {
return new Form(*this);
}
~Form() {}
@@ -183,11 +167,11 @@ public:
std::string subtype_;
};
Object const& getRoot() {
Object const& getRoot() const {
return *root_;
}
void printStructure(unsigned level = 0);
void printStructure(unsigned level = 0) const;
private:
static std::unique_ptr<Object> parseObject(char const* base, size_t length);

51
Makefile
View File

@@ -2,21 +2,21 @@ CXX=g++
CXXOPTS=-Og -ggdb -Wall -Wextra -pedantic -std=c++14 -flto
LDOPTS=
IFFEXPLORE_CXXSRCS=iffexplore.cc IffFile.cc
IFFEXPLORE_OBJS=$(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.o))
iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc
iffexplore_LIBS ::=
TREEXPLORE_CXXSRCS=treexplore.cc TreFile.cc IffFile.cc
TREEXPLORE_OBJS=$(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.o))
treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc
treexplore_LIBS ::=
FONT2PNG_CXXSRCS=font2png.cc
FONT2PNG_OBJS=$(addprefix objs/,$(FONT2PNG_CXXSRCS:.cc=.o))
FONT2PNG_LIBS=-lpng
font2png_CXXSRCS ::= font2png.cc
font2png_LIBS ::= -lpng
PNTR2PNG_CXXSRCS=pntr2png.cc
PNTR2PNG_OBJS=$(addprefix objs/,$(PNTR2PNG_CXXSRCS:.cc=.o))
PNTR2PNG_LIBS=-lpng
mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc
mvedecode_LIBS ::=
all: iffexplore font2png pntr2png treexplore
progs ::= iffexplore font2png treexplore mvedecode
all: $(progs)
objs/%.o: %.cc
$(CXX) $(CXXOPTS) -c -MMD -MP -o $@ $<
@@ -25,28 +25,17 @@ objs/%.o: %.cc
%.pb.cc %.pb.h: %.proto
protoc --cpp_out=. $<
iffexplore: $(IFFEXPLORE_OBJS)
$(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(IFFEXPLORE_OBJS)
treexplore: $(TREEXPLORE_OBJS)
$(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(TREEXPLORE_OBJS)
font2png: $(FONT2PNG_OBJS)
$(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(FONT2PNG_OBJS) $(FONT2PNG_LIBS)
pntr2png: $(PNTR2PNG_OBJS)
$(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $(PNTR2PNG_OBJS) $(PNTR2PNG_LIBS)
$(progs): %:
$(CXX) $(CXXOPTS) $(LDOPTS) -o $@ $^ $($(@)_LIBS)
clean:
rm -f iffexplore $(IFFEXPLORE_OBJS) $(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.P))
rm -f treexplore $(TREEXPLORE_OBJS) $(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.P))
rm -f font2png $(FONT2PNG_OBJS) $(addprefix objs/,$(FONT2PNG_CXXSRCS:.cc=.P))
rm -f pntr2png $(PNTR2PNG_OBJS) $(addprefix objs/,$(PNTR2PNG_CXXSRCS:.cc=.P))
rm -f $(progs)\
$(foreach prog,$(progs),$(addprefix objs/,$($(prog)_CXXSRCS:.cc=.o)))\
$(foreach prog,$(progs),$(addprefix objs/,$($(prog)_CXXSRCS:.cc=.P)))
.PHONY: clean all
-include $(addprefix objs/,$(IFFEXPLORE_CXXSRCS:.cc=.P))
-include $(addprefix objs/,$(TREEXPLORE_CXXSRCS:.cc=.P))
-include $(addprefix objs/,$(FONT2PNG_CXXSRCS:.cc=.P))
-include $(addprefix objs/,$(PNTR2PNG_CXXSRCS:.cc=.P))
-include $(foreach prog,$(progs),$(addprefix objs/,$($(prog)_CXXSRCS:.cc=.P)))
.SECONDEXPANSION:
$(progs): %: $$(addprefix objs/,$$($$@_CXXSRCS:.cc=.o))

294
MveDecoder.cc Normal file
View File

@@ -0,0 +1,294 @@
#include <algorithm>
#include "common.hh"
#include "IffFile.hh"
#include "MveDecoder.hh"
struct PCChunk {
uint32_t unknown[2];
uint32_t PALTcount;
} __attribute__((__packed__));
MveDecoder::MveDecoder(char const* base, size_t length)
: iff_(base, length)
{
//iff_.printStructure();
auto& root = iff_.getRoot();
if (!root.isForm())
throw FormatException{"Root node not FORM"};
auto& rootForm = dynamic_cast<IffFile::Form const&>(root);
if (rootForm.getSubtype() != "MOVE")
throw FormatException{"Root form not MOVE"};
auto it = rootForm.childrenBegin();
// Parse _PC_ chunk
auto& PC = *it++;
if ((PC.getType() != "_PC_") ||
(PC.getSize() != 12))
throw FormatException{"_PC_ chunk missing or wrong size"};
PCChunk const* PCc = reinterpret_cast<PCChunk const*>(PC.begin());
#ifndef NDEBUG
printf("_PC_: %u PALTs\n", PCc->PALTcount);
#endif
// Parse SOND chunk
auto& SOND = *it++;
if ((SOND.getType() != "SOND") ||
(SOND.getSize() != 4))
throw FormatException{"SOND chunk missing or wrong size"};
uint32_t const* SONDc = reinterpret_cast<uint32_t const*>(SOND.begin());
if (*SONDc != 3)
throw FormatException{"Unexpected SOND value"};
// Parse data
int curSHOT = -1;
unsigned curPALT = 0;
std::vector<IffFile::Object const*> PALTs;
for (;it != rootForm.childrenEnd();++it) {
if (it->getType() == "PALT") {
if (curPALT > PCc->PALTcount)
throw FormatException{"Number of PALT chunks exceeds amount specified in _PC_"};
if (it->getSize() != 768)
throw FormatException{"Unexpected PALT size"};
palts_.emplace_back();
std::transform(it->begin(), it->end(), palts_.back().begin(),
[](const char& in) -> uint8_t {return (in << 2) | ((in >> 6)&0x3);});
++curPALT;
} else if (it->getType() == "SHOT") {
uint32_t const* SHOTc = reinterpret_cast<uint32_t const*>(it->begin());
if (*SHOTc > PCc->PALTcount)
throw FormatException{"SHOT refers to PALT outside amount specified in _PC_"};
++curSHOT;
Shot shot{palts_.at(*SHOTc), {}, {}};
shots_.push_back(shot);
} else if (it->getType() == "VGA ") {
if (curSHOT < 0)
throw FormatException{"VGA outside SHOT"};
shots_.back().VGAs.push_back(parseVGA(*it, shots_.back().VGAs.empty()?nullptr:&shots_.back().VGAs.back()));
} else if (it->getType() == "AUDI") {
if (curSHOT < 0)
throw FormatException{"AUDI outside SHOT"};
if (it->getSize() != 2940)
throw FormatException{"Unexpected AUDI size"};
shots_.back().AUDIs.push_back(&*it);
} else if ((it->getType() == "INDX") ||
(it->getType() == "BRCH")) {
// Index/branches NYI
} else
throw FormatException{"Encountered unexpected chunk: " + it->getType()};
}
unsigned i = 0;
for(auto& shot : shots_) {
#ifndef NDEBUG
printf("Shot %u: Palette %ld, %lu video frames, %lu audio blocks\n",
i++, std::find(palts_.begin(), palts_.end(), shot.palt)-palts_.begin(),
shot.VGAs.size(), shot.AUDIs.size());
#endif
if (shot.VGAs.size() != shot.AUDIs.size())
throw FormatException{"Video/audio block count mismatch"};
}
}
char *binify(uint8_t a) {
static char buf[9];
buf[8] = '\0';
for(unsigned i = 0;i < 8;++i) {
if(a&0x80)
buf[i] = '1';
else
buf[i] = '0';
a <<= 1;
}
return buf;
}
std::vector<char> parseHuff_(char const* data, size_t len)
{
uint8_t numHuffVals;
memcpy(&numHuffVals, data, 1);
if (numHuffVals != 22)
throw FormatException{"Unexpected huffman tree size"};
std::array<uint8_t, 44> huffTree;
memcpy(huffTree.data(), data+1, 44);
#ifdef HUFFDEBUG_
printf("Coded data length: %d\nTree:\n", segOfs[1]-(segOfs[0]+45));
printf(" 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f\n");
printf("0x00: ");
for(unsigned i = 0;i < huffTree.size();++i) {
printf("%.2hhx", huffTree[i]);
if ((i+1)%16 == 0)
printf("\n0x%.2hhx: ", (i+1)&0xf0);
else
printf(" ");
}
printf("\n");
#endif
int byteValid = 0;
uint8_t byteBuf = 0;
unsigned bytePos = 45;
std::vector<char> commands;
unsigned huffIdx = huffTree.size();
while (huffIdx != 22) {
// Read next bit
unsigned bit;
if (byteValid) {
bit = byteBuf&0x1;
byteBuf >>= 1;
--byteValid;
} else {
if (bytePos >= len)
throw FormatException{"Huffman stream overrun"};
memcpy(&byteBuf, data+bytePos, 1);
//printf("Byte: %.2hhx (%s)\n", byteBuf, binify(byteBuf));
++bytePos;
bit = byteBuf&0x1;
byteBuf >>= 1;
byteValid = 7;
}
huffIdx = huffTree.at(huffIdx-(bit?1:23));
//printf("step: %d %.2x\n", bit, huffIdx);
if (huffIdx < 22) {
commands.push_back(huffIdx);
//printf("out: %.2hhx\n", commands.back());
huffIdx = huffTree.size();
}
}
if (commands.empty())
throw FormatException{"No commands decoded"};
#ifdef HUFFDEBUG_
for(auto ent : commands)
printf("%.2hhx ", ent);
printf("\n");
#endif
return commands;
}
std::vector<uint8_t> parsePixels_(char const* data, size_t len)
{
std::vector<uint8_t> ret;
if (*data == 0x02) {
size_t pos = 1;
while (pos < len) {
uint8_t b = *(data+pos++);
if (!((b&0xe0)==0xe0)) {
unsigned size = 0, replSize = 0;
unsigned replOfs = 0;
if (!(b&0x80)) {
//printf("Code: Repl1 %.2hhx\n", b);
if (pos >= len)
throw FormatException{"Pixel data overrun"};
uint8_t ofs = *(data+pos++);
size = b&0x3;
if (pos+size >= len)
throw FormatException{"Pixel data overrun"};
replSize = ((b&0x1c)>>2) + 3;
replOfs = ret.size()-(((b&0x60)<<3)+ofs+1)+size;
} else if (!(b&0x40)) {
//printf("Code: Repl3 %.2hhx\n", b);
if (pos+1 >= len)
throw FormatException{"Pixel data overrun"};
uint8_t b1 = *(data+pos++);
uint8_t b2 = *(data+pos++);
size = (b1&0xc0)>>6;
if (pos+size >= len)
throw FormatException{"Pixel data overrun"};
replSize = (b&0x3f)+4;
replOfs = ret.size()-(((b1&0x3f)<<8)+b2+1)+size;
} else if (!(b&0x20)) {
//printf("Code: Repl2 %.2hhx\n", b);
if (pos+2 >= len)
throw FormatException{"Pixel data overrun"};
uint8_t b1 = *(data+pos++);
uint8_t b2 = *(data+pos++);
uint8_t b3 = *(data+pos++);
size = b&0x3;
if (pos+size >= len)
throw FormatException{"Pixel data overrun"};
replSize = b3+5+((b&0xc)<<6);
replOfs = ret.size()-(((b&0x10)<<12)+1+(b1<<8)+b2)+size;
}
for (unsigned i = 0;i < size;++i)
ret.push_back(*(data+pos++));
for (unsigned i = 0;i < replSize;++i)
ret.push_back(ret[replOfs+i]);
} else {
//printf("Code: Copy %.2hhx\n", b);
unsigned size = (b&0x1f)*4+4;
if (size > 0x70)
break;
if (pos+size > len)
throw FormatException{"Pixel data overrun"};
for (unsigned i = 0;i < size;++i)
ret.push_back(*(data+pos++));
}
}
} else
std::copy(data+1, data+len, std::back_inserter(ret));
return ret;
}
MveDecoder::Frame MveDecoder::parseVGA(IffFile::Object const& vga, Frame const* prev)
{
// Parse header
if (vga.getSize() < 8)
throw FormatException{"VGA chunk smaller than VGA header"};
std::array<uint16_t, 4> segOfs;
memcpy(segOfs.data(), vga.begin(), 8);
for(unsigned i = 0;i < segOfs.size();++i) {
if (segOfs[i] >= vga.getSize())
throw FormatException{"Segment offset exceeds chunk"};
}
// Parse segment 1
if ((segOfs[1] - segOfs[0]) < 45)
throw FormatException{"Segment 1 too small"};
auto commands = parseHuff_(vga.begin()+segOfs[0], segOfs[1]-segOfs[0]);
std::vector<uint8_t> pixels;
if (segOfs[3] > 0)
pixels = parsePixels_(vga.begin()+segOfs[3], vga.getSize()-segOfs[3]);
if (pixels.size() == 52800) {
FILEUPtr outFile{fopen("pixel.data", "wb")};
if (!outFile)
throw POSIXException{errno, "Could not open pixel.data"};
if (fwrite(pixels.data(), pixels.size(), 1, outFile.get()) != 1)
throw POSIXException{errno, "Could not write"};
// exit(0);
}
}

33
MveDecoder.hh Normal file
View File

@@ -0,0 +1,33 @@
#ifndef WC3RE_MVEDECODER_HH__
#define WC3RE_MVEDECODER_HH__
#include <vector>
#include <array>
#include "IffFile.hh"
class MveDecoder {
public:
MveDecoder(char const* base, size_t length);
private:
IffFile iff_;
using Palette = std::array<uint8_t, 768>;
struct Frame {
};
struct Shot {
Palette const& palt;
std::vector<IffFile::Object const*> AUDIs;
std::vector<Frame> VGAs;
};
std::vector<Shot> shots_;
std::vector<Palette> palts_;
static Frame parseVGA(IffFile::Object const& vga, Frame const* prev);
};
#endif

376
TreFile.cc
View File

@@ -1,13 +1,6 @@
#include <cstdio>
#include <cstring>
#include <cerrno>
#include <cstdint>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common.hh"
#include "TreFile.hh"
@@ -26,51 +19,130 @@ struct Table1Entry { // little endian
uint32_t table3Ptr;
} __attribute__((__packed__));
struct Table3Entry { // little endian
unsigned dataPtr:32;
// unsigned unknown1:8;
unsigned length:24;
unsigned unknown2:8;
} __attribute__((__packed__));
TreFile::TreFile(std::string const& filename)
: file_(nullptr), start_(0), length_(0)
TreFile::TreFile(char const* base, size_t length)
: base_(base), length_(length)
{
file_ = fopen(filename.c_str(), "rb");
if (!file_)
throw POSIXException{errno, "Could not open "s + filename};
if (length_ < sizeof(TreHeader))
throw FormatException{"Size < header size"};
{
struct stat statBuf;
TreHeader const* header = reinterpret_cast<TreHeader const*>(base);
if (fstat(fileno(file_), &statBuf) != 0) {
fclose(file_);
throw POSIXException(errno, "Could not stat");
if (memcmp(header->magic, "XTRE", 4) != 0)
throw FormatException{"Wrong magic, not a TRE?"};
if ((header->table1Ofs > length_) ||
(header->table2Ofs > length_) ||
(header->table3Ofs > length_) ||
(header->dataOfs > length_))
throw FormatException{"Table offset exceeds length"};
if ((header->table1Ofs > header->table2Ofs) ||
(header->table2Ofs > header->table3Ofs) ||
(header->table3Ofs > header->dataOfs))
throw FormatException{"Table offsets not in ascending order"};
const size_t numTable1Entries = (header->table2Ofs-header->table1Ofs)/sizeof(Table1Entry);
table3Size_ = (header->dataOfs-header->table3Ofs)/sizeof(Table3Entry);
#ifndef NDEBUG
printf("Table 1: start %u, length %u (%lu entries)\n",
header->table1Ofs, header->table2Ofs-header->table1Ofs,
numTable1Entries);
printf("Table 2: start %u, length %u\n",
header->table2Ofs, header->table3Ofs-header->table2Ofs);
printf("Table 3: start %u, length %u (%lu entries)\n",
header->table3Ofs, header->dataOfs-header->table3Ofs,
table3Size_);
printf("Data: start %u, length %lu\n",
header->dataOfs, length_ - header->dataOfs);
#endif
// Check Table 3
table3_ = reinterpret_cast<Table3Entry const*>(base+header->table3Ofs);
for (size_t i = 0;i < table3Size_;++i) {
if ((table3_[i].dataPtr > length_) ||
(table3_[i].dataPtr < header->dataOfs))
throw FormatException{"Data pointer out of range"};
if ((table3_[i].dataPtr + table3_[i].length) > length_) {
fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", table3_[i].dataPtr, table3_[i].length);
}
}
length_ = statBuf.st_size;
// Read Table 2
std::map<size_t, uint32_t> table2Pos;
size_t pos = header->table2Ofs;
while((pos+5) < header->table3Ofs) {
uint8_t const* nameLen = reinterpret_cast<uint8_t const*>(base+pos);
if (pos+*nameLen+5 > header->table3Ofs)
throw FormatException{"Table 2 entry exceeds table "s + std::to_string(*nameLen)};
std::string nameStr(base+pos+1, *nameLen);
uint32_t const* table3Ptr = reinterpret_cast<uint32_t const*>(base+pos+*nameLen+1);
if ((*table3Ptr < header->table3Ofs) ||
(*table3Ptr >= header->dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
const size_t table3Index = (*table3Ptr - header->table3Ofs)/sizeof(Table3Entry);
auto ins = table2_.emplace(nameStr, table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for name %s\n", nameStr.c_str());
else
printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(),
ins.first->second, table3Index);
}
// Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's
table2Pos.emplace(pos, *table3Ptr);
pos += 1+*nameLen+4;
}
// Read Table 1
Table1Entry const* table1 = reinterpret_cast<Table1Entry const*>(base+header->table1Ofs);
for (size_t i = 0;i < numTable1Entries;++i) {
if ((table1[i].crc == 0x00000000) &&
(table1[i].table3Ptr == 0xffffffff))
continue; // Unused entry
if ((table1[i].table3Ptr < header->table2Ofs) ||
(table1[i].table3Ptr >= header->dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
uint32_t table3Ptr = table1[i].table3Ptr;
if (table3Ptr < header->table3Ofs) {
auto it = table2Pos.find(table3Ptr);
if (it == table2Pos.end()) {
fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n",
table1[i].crc, table3Ptr);
continue;
}
printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n",
table3Ptr, it->second);
table3Ptr = it->second;
}
const size_t table3Index = (table3Ptr - header->table3Ofs)/sizeof(Table3Entry);
auto ins = table1_.emplace(static_cast<uint32_t>(table1[i].crc), table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for CRC %.8x\n", table1[i].crc);
else
printf("Collision for CRC %.8x: prev %lu, new %lu\n", table1[i].crc, ins.first->second, table3Index);
}
try {
construct_();
} catch(...) {
fclose(file_);
throw;
}
}
TreFile::TreFile(FILE* file, off_t start, off_t length)
: file_(file), start_(start), length_(length)
{
construct_();
}
TreFile::~TreFile()
{
if (file_) {
fclose(file_);
file_ = nullptr;
}
}
std::vector<std::string> TreFile::getNames() const
@@ -93,12 +165,12 @@ std::vector<uint32_t> TreFile::getCRCs() const
return ret;
}
TreFile::File TreFile::openName(std::string const& name) const
TreFile::Object TreFile::openName(std::string const& name) const
{
return openIdx_(findName_(name));
}
TreFile::File TreFile::openCRC(uint32_t crc) const
TreFile::Object TreFile::openCRC(uint32_t crc) const
{
return openIdx_(findCRC_(crc));
}
@@ -113,7 +185,7 @@ TreFile::Stat TreFile::statCRC(uint32_t crc) const
return statIdx_(findCRC_(crc));
}
void TreFile::dumpName(std::string path, std::string const& name)
void TreFile::dumpName(std::string path, std::string const& name) const
{
auto idx = findName_(name);
@@ -124,7 +196,7 @@ void TreFile::dumpName(std::string path, std::string const& name)
dumpIdx_(path, idx);
}
void TreFile::dumpCRC(std::string path, uint32_t crc)
void TreFile::dumpCRC(std::string path, uint32_t crc) const
{
auto idx = findCRC_(crc);
char crcStr[9];
@@ -137,7 +209,7 @@ void TreFile::dumpCRC(std::string path, uint32_t crc)
dumpIdx_(path, idx);
}
void TreFile::dumpAll(std::string path)
void TreFile::dumpAll(std::string path) const
{
if (path.back() != '/')
path.push_back('/');
@@ -163,177 +235,22 @@ void TreFile::printStructure()
{
printf("Table 1: %lu entries\n", table1_.size());
printf("Table 2: %lu entries\n", table2_.size());
printf("Table 3: %lu entries\n", table3_.size());
printf("Table 3: %lu entries\n", table3Size_);
printf("Files by CRC:\n");
for(auto const& ent : table1_) {
printf("\t%.8x -> (ofs %.8x, len %.8x, flags %.2hhx)\n", ent.first,
std::get<0>(table3_[ent.second]),
std::get<1>(table3_[ent.second]),
std::get<2>(table3_[ent.second]));
table3_[ent.second].dataPtr,
table3_[ent.second].length,
table3_[ent.second].flags);
}
printf("Files by Name:\n");
for(auto const& ent : table2_) {
printf("\t%s -> (ofs %.8x, len %.8x, flags %.2hhx)\n", ent.first.c_str(),
std::get<0>(table3_[ent.second]),
std::get<1>(table3_[ent.second]),
std::get<2>(table3_[ent.second]));
}
}
void TreFile::construct_()
{
if (length_ < sizeof(TreHeader))
throw FormatException{"Size < header size"};
if (fseeko(file_, start_, SEEK_SET) != 0) {
throw POSIXException(errno, "Could not seek");
}
TreHeader header;
if (fread(&header, sizeof(TreHeader), 1, file_) != 1)
throw POSIXException{errno, "Could not read header"};
if (memcmp(header.magic, "XTRE", 4) != 0)
throw FormatException{"Wrong magic, not a TRE?"};
if ((header.table1Ofs > length_) ||
(header.table2Ofs > length_) ||
(header.table3Ofs > length_) ||
(header.dataOfs > length_))
throw FormatException{"Table offset exceeds length"};
if ((header.table1Ofs > header.table2Ofs) ||
(header.table2Ofs > header.table3Ofs) ||
(header.table3Ofs > header.dataOfs))
throw FormatException{"Table offsets not in ascending order"};
const size_t numTable1Entries = (header.table2Ofs-header.table1Ofs)/sizeof(Table1Entry);
const size_t numTable3Entries = (header.dataOfs-header.table3Ofs)/sizeof(Table3Entry);
printf("Table 1: start %u, length %u (%lu entries)\n",
header.table1Ofs, header.table2Ofs-header.table1Ofs,
numTable1Entries);
printf("Table 2: start %u, length %u\n",
header.table2Ofs, header.table3Ofs-header.table2Ofs);
printf("Table 3: start %u, length %u (%lu entries)\n",
header.table3Ofs, header.dataOfs-header.table3Ofs,
numTable3Entries);
printf("Data: start %u, length %lu\n",
header.dataOfs, length_ - header.dataOfs);
// Read Table 3
if (fseeko(file_, start_+header.table3Ofs, SEEK_SET) != 0) {
throw POSIXException{errno, "Could not seek"};
}
table3_.reserve(numTable3Entries);
for (size_t i = 0;i < numTable3Entries;++i) {
Table3Entry entry;
if (fread(&entry, sizeof(Table3Entry), 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if ((entry.dataPtr > length_) ||
(entry.dataPtr < header.dataOfs))
throw FormatException{"Data pointer out of range"};
if ((entry.dataPtr + entry.length) > length_) {
fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", entry.dataPtr, entry.length);
//throw FormatException{"Data length exceeds file length"};
}
table3_.push_back(std::make_tuple(static_cast<uint32_t>(entry.dataPtr),
static_cast<uint32_t>(entry.length),
static_cast<uint8_t>(entry.unknown2)));
}
// Read Table 2
if (fseeko(file_, start_+header.table2Ofs, SEEK_SET) != 0) {
throw POSIXException{errno, "Could not seek"};
}
std::map<size_t, uint32_t> table2Pos;
size_t pos = header.table2Ofs;
while((pos+5) < header.table3Ofs) {
uint8_t nameLen;
if (fread(&nameLen, 1, 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if (pos+nameLen+5 > header.table3Ofs)
throw FormatException{"Table 2 entry exceeds table "s + std::to_string(nameLen)};
std::vector<char> nameData(nameLen);
if (fread(nameData.data(), nameLen, 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
uint32_t table3Ptr;
if (fread(&table3Ptr, 4, 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if ((table3Ptr < header.table3Ofs) ||
(table3Ptr >= header.dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
std::string nameStr{nameData.data(), nameLen};
size_t table3Index = (table3Ptr - header.table3Ofs)/sizeof(Table3Entry);
auto ins = table2_.emplace(nameStr, table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for name %s\n", nameStr.c_str());
else
printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(),
ins.first->second, table3Index);
}
// Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's
table2Pos.emplace(pos, table3Ptr);
pos += 1+nameLen+4;
}
// Read Table 1
if (fseeko(file_, start_+header.table1Ofs, SEEK_SET) != 0) {
throw POSIXException{errno, "Could not seek"};
}
for (size_t i = 0;i < numTable1Entries;++i) {
Table1Entry entry;
if (fread(&entry, sizeof(Table1Entry), 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if ((entry.crc == 0x00000000) &&
(entry.table3Ptr == 0xffffffff))
continue; // Unused entry
if ((entry.table3Ptr < header.table2Ofs) ||
(entry.table3Ptr >= header.dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
if (entry.table3Ptr < header.table3Ofs) {
auto it = table2Pos.find(entry.table3Ptr);
if (it == table2Pos.end()) {
fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n",
entry.crc, entry.table3Ptr);
continue;
}
printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n",
entry.table3Ptr, it->second);
entry.table3Ptr = it->second;
}
size_t table3Index = (entry.table3Ptr - header.table3Ofs)/sizeof(Table3Entry);
auto ins = table1_.emplace(static_cast<uint32_t>(entry.crc), table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for CRC %.8x\n", entry.crc);
else
printf("Collision for CRC %.8x: prev %lu, new %lu\n", entry.crc, ins.first->second, table3Index);
}
table3_[ent.second].dataPtr,
table3_[ent.second].length,
table3_[ent.second].flags);
}
}
@@ -358,56 +275,37 @@ size_t TreFile::findCRC_(uint32_t crc) const
return it->second;
}
void TreFile::dumpIdx_(std::string const& name, size_t table3Idx)
void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const
{
uint32_t ofs, len;
std::tie(ofs, len, std::ignore) = table3_[table3Idx];
// if (ofs%2 != 0) {
// ++ofs;
// len;
// }
std::vector<char> buf(len);
if (fseeko(file_, start_+ofs, SEEK_SET) != 0)
throw POSIXException{errno, "Could not seek"};
if (fread(buf.data(), len, 1, file_) != 1)
throw POSIXException{errno, "Could not read data"};
if(table3_[table3Idx].flags&0x80)
throw Exception{"Compressed TRE objects NYI"};
FILEUPtr outFile{fopen(name.c_str(), "wb")};
if (!outFile)
throw POSIXException{errno, "Could not open " + name};
if (fwrite(buf.data(), len, 1, outFile.get()) != 1)
if (fwrite(base_+table3_[table3Idx].dataPtr, table3_[table3Idx].length, 1, outFile.get()) != 1)
throw POSIXException{errno, "Could not write"};
}
TreFile::File TreFile::openIdx_(size_t table3Idx) const
TreFile::Object TreFile::openIdx_(size_t table3Idx) const
{
uint32_t ofs, len;
uint8_t flags;
std::tie(ofs, len, flags) = table3_[table3Idx];
if(flags&0x80)
if(table3_[table3Idx].flags&0x80)
throw Exception{"Compressed TRE objects NYI"};
std::vector<char> buf(len);
if ((table3_[table3Idx].dataPtr + table3_[table3Idx].length) > length_)
throw FormatException{"length exceeds file size"};
if (fseeko(file_, start_+ofs, SEEK_SET) != 0)
throw POSIXException{errno, "Could not seek"};
if (fread(buf.data(), len, 1, file_) != 1)
throw POSIXException{errno, "Could not read data"};
return File{buf};
return Object(base_+table3_[table3Idx].dataPtr,
table3_[table3Idx].length);
}
TreFile::Stat TreFile::statIdx_(size_t table3Idx) const
{
Stat ret;
std::tie(std::ignore, ret.size, ret.flags) = table3_[table3Idx];
ret.size = table3_[table3Idx].length;
ret.flags = table3_[table3Idx].flags;
return ret;
}

70
TreFile.hh
View File

@@ -9,45 +9,65 @@
class TreFile {
public:
TreFile(std::string const& filename);
TreFile(FILE* file, off_t start, off_t length);
TreFile(char const* base, size_t length);
~TreFile();
TreFile(TreFile const& copy) = delete;
TreFile& operator=(TreFile const& copy) = delete;
std::vector<std::string> getNames() const;
std::vector<uint32_t> getCRCs() const;
void dumpName(std::string path, std::string const& name);
void dumpCRC(std::string path, uint32_t crc);
void dumpAll(std::string path);
void dumpName(std::string path, std::string const& name) const;
void dumpCRC(std::string path, uint32_t crc) const;
void dumpAll(std::string path) const;
void printStructure();
class File {
class Object {
public:
Object() : base_(nullptr), length_(0) {
}
Object(Object const& copy) = delete;
Object(Object && move)
: base_(move.base_), length_(move.length_) {
move.base_ = nullptr;
move.length_ = 0;
}
Object& operator=(Object const& copy) = delete;
Object& operator=(Object && move) {
base_ = move.base_;
length_ = move.length_;
move.base_ = nullptr;
move.length_ = 0;
return *this;
}
char const* data() const {
return data_.data();
return base_;
}
size_t size() const {
return data_.size();
return length_;
}
operator bool() const {
return base_;
}
private:
std::vector<char> data_;
char const* base_;
size_t length_;
File(std::vector<char> data)
: data_(std::move(data)) {
Object(char const* base, size_t length)
: base_(base), length_(length) {
}
friend class TreFile;
};
File openName(std::string const& name) const;
File openCRC(uint32_t crc) const;
Object openName(std::string const& name) const;
Object openCRC(uint32_t crc) const;
struct Stat {
uint32_t size;
@@ -63,15 +83,23 @@ private:
size_t findName_(std::string const& name) const;
size_t findCRC_(uint32_t crc) const;
void dumpIdx_(std::string const& name, size_t table3Idx);
File openIdx_(size_t table3Idx) const;
void dumpIdx_(std::string const& name, size_t table3Idx) const;
Object openIdx_(size_t table3Idx) const;
Stat statIdx_(size_t table3Idx) const;
std::map<uint32_t, size_t> table1_;
std::map<std::string, size_t> table2_;
std::vector<std::tuple<uint32_t, uint32_t, uint8_t> > table3_;
mutable FILE* file_;
off_t start_;
struct Table3Entry { // little endian
unsigned dataPtr:32;
unsigned length:24;
unsigned flags:8;
} __attribute__((__packed__));
Table3Entry const* table3_;
size_t table3Size_;
char const* base_;
size_t length_;
};

33
iffexplore.cc
View File

@@ -4,13 +4,10 @@
#include <cstdint>
#include <vector>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/mman.h>
#include "common.hh"
#include "util.hh"
#include "IffFile.hh"
void blobDump(IffFile::Object const& obj, std::string const& filename)
@@ -31,7 +28,7 @@ void iffDumper(IffFile::Object const& obj, bool dumpBlobs, std::string dumpPath,
printf("%s Length %lu (0x%.lx)", obj.getType().c_str(), obj.getSize(), obj.getSize());
if (obj.isForm()) {
auto form = dynamic_cast<IffFile::Form const&>(obj);
auto& form = dynamic_cast<IffFile::Form const&>(obj);
printf(", Subtype %s\n", form.getSubtype().c_str());
for(auto it = form.childrenBegin();it != form.childrenEnd();++it)
iffDumper(*it, dumpBlobs, dumpPath, blobCount, level+1);
@@ -89,36 +86,14 @@ int main(int argc, char *argv[])
}
try {
FILEUPtr iffFD{fopen(iffFile.c_str(), "rb")};
if (!iffFD) {
throw POSIXException{errno, "Could not open "s + iffFile};
}
MmapFile mmap{iffFile};
struct stat statBuf;
if (fstat(fileno(iffFD.get()), &statBuf) != 0) {
throw POSIXException(errno, "Could not stat");
}
char *mmapBase = static_cast<char*>(mmap(nullptr, statBuf.st_size, PROT_READ, MAP_SHARED, fileno(iffFD.get()), 0));
if (!mmapBase)
throw POSIXException(errno, "mmap failed");
try {
IffFile iff{mmapBase, static_cast<size_t>(statBuf.st_size)};
IffFile iff{mmap.data(), mmap.size()};
if (printStructure) {
unsigned blobCount = 0;
iffDumper(iff.getRoot(), dumpBlobs, dumpPath, blobCount);
}
} catch(...) {
munmap(mmapBase, statBuf.st_size);
throw;
}
if (munmap(mmapBase, statBuf.st_size) != 0)
fprintf(stderr, "Warning: munmap failed: %s\n", strerror(errno));
} catch (POSIXException &ex) {
fflush(stdout);
fprintf(stderr, "%s\n", ex.toString().c_str());

89
mvedecode.cc Normal file
View File

@@ -0,0 +1,89 @@
#include <cstdio>
#include <cstring>
#include <cerrno>
#include <cstdint>
#include <memory>
#include <limits>
#include <unistd.h>
#include "common.hh"
#include "util.hh"
#include "TreFile.hh"
#include "IffFile.hh"
#include "MveDecoder.hh"
void usage(char *argv0) {
fprintf(stderr, "Usage: %s [-h] (tre-file name/crc)/iff-file\n", argv0);
fprintf(stderr, "\tAttempt to decode the movie stored in iff-file, or in the\n\tiff-object \"name\"/\"crc\" contained in tre-file\n");
fprintf(stderr, "\t-h\tPrint this help\n");
}
int main(int argc, char *argv[]) {
std::string inFile, objectSpec;
bool useTre = false;
{
int opt;
while ((opt = getopt(argc, argv, "h")) != -1) {
switch (opt) {
case 'h':
usage(argv[0]);
return 0;
default:
usage(argv[0]);
return 1;
}
}
if (optind >= argc) {
usage(argv[0]);
return 1;
}
inFile = argv[optind++];
if (optind < argc) {
useTre = true;
objectSpec = argv[optind];
}
}
try {
MmapFile mmap{inFile};
std::unique_ptr<TreFile> tre;
TreFile::Object object;
std::unique_ptr<MveDecoder> mve;
if (useTre) {
tre = std::make_unique<TreFile>(mmap.data(), mmap.size());
// Try to parse objectSpec as CRC
try {
unsigned long CRC = std::stoul(objectSpec, nullptr, 16);
if (CRC <= std::numeric_limits<uint32_t>::max()) {
object = tre->openCRC(CRC);
}
} catch (std::invalid_argument &ex) {
} catch (std::out_of_range &ex) {
}
if (!object) // Wasn't a CRC, try as name
object = tre->openName(objectSpec);
mve = std::make_unique<MveDecoder>(object.data(), object.size());
} else
mve = std::make_unique<MveDecoder>(mmap.data(), mmap.size());
} catch (POSIXException &ex) {
fflush(stdout);
fprintf(stderr, "%s\n", ex.toString().c_str());
return 2;
} catch (Exception &ex) {
fflush(stdout);
fprintf(stderr, "%s\n", ex.toString().c_str());
return 3;
}
return 0;
}

8
treexplore.cc
View File

@@ -5,12 +5,10 @@
#include <vector>
#include <limits>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common.hh"
#include "util.hh"
#include "TreFile.hh"
#include "IffFile.hh"
@@ -62,7 +60,9 @@ int main(int argc, char *argv[]) {
}
try {
TreFile file{treFile};
MmapFile mmap{treFile};
TreFile file{mmap.data(), mmap.size()};
if (printStructure)
file.printStructure();

87
util.cc Normal file
View File

@@ -0,0 +1,87 @@
#include <stdexcept>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common.hh"
#include "util.hh"
MmapFile::MmapFile(std::string fileName)
: name_(std::move(fileName)), fd_(fopen(name_.c_str(), "rb")), size_(0), base_(nullptr)
{
if (!fd_)
throw POSIXException{errno, "Could not open "s + name_};
struct stat statBuf;
if (fstat(fileno(fd_.get()), &statBuf) != 0)
throw POSIXException(errno, "Could not stat");
if (statBuf.st_size <= 0)
throw Exception{"File size is zero or less"};
size_ = static_cast<size_t>(statBuf.st_size);
base_ = mmap(nullptr, size_, PROT_READ, MAP_SHARED, fileno(fd_.get()), 0);
if (!base_)
throw POSIXException(errno, "mmap failed");
}
MmapFile::MmapFile(MmapFile && move):
name_(std::move(move.name_)), fd_(std::move(move.fd_)), size_(move.size_),
base_(move.base_)
{
move.base_ = nullptr;
}
MmapFile::~MmapFile()
{
if (base_) {
if (munmap(base_, size_) != 0)
fprintf(stderr, "Warning: munmap failed: %s\n", strerror(errno));
base_ = nullptr;
}
}
MmapFile& MmapFile::operator=(MmapFile && move)
{
fd_ = std::move(move.fd_);
if (base_)
if (munmap(base_, size_) != 0)
fprintf(stderr, "Warning: munmap failed: %s\n", strerror(errno));
base_ = move.base_;
move.base_ = nullptr;
size_ = move.size_;
name_ = std::move(move.name_);
return *this;
}
std::string const& MmapFile::name() const
{
if (!*this)
throw std::logic_error("name() called on invalid MmapFile");
return name_;
}
size_t MmapFile::size() const
{
if (!*this)
throw std::logic_error("size() called on invalid MmapFile");
return size_;
}
char const* MmapFile::data() const
{
if (!*this)
throw std::logic_error("data() called on invalid MmapFile");
return static_cast<char const*>(base_);
}
MmapFile::operator bool() const
{
return base_;
}

32
util.hh Normal file
View File

@@ -0,0 +1,32 @@
#ifndef WC3RE_UTIL_HH__
#define WC3RE_UTIL_HH__
#include <cstdint>
#include <string>
#include "common.hh"
class MmapFile {
public:
MmapFile(std::string fileName);
MmapFile(MmapFile const& copy) = delete;
MmapFile(MmapFile && move);
~MmapFile();
MmapFile& operator=(MmapFile const& copy) = delete;
MmapFile& operator=(MmapFile && move);
std::string const& name() const;
size_t size() const;
char const* data() const;
operator bool() const;
private:
std::string name_;
FILEUPtr fd_;
size_t size_;
void *base_;
};
#endif