matthias/wc3re
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

MmapFile helper; MveDecoder WIP

Browse Source 
Added an MmapFile helper and changed iffexplore and treexplore to use it.
WIP: MveDecoder
Misc. changes/fixes
This commit is contained in:
Matthias Blankertz
2015-04-24 22:13:32 +02:00
parent 89688c5fba
commit ae51cd24c4
12 changed files with 761 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

378
TreFile.cc
View File

@@ -1,13 +1,6 @@
#include <cstdio>
#include <cstring>
#include <cerrno>
#include <cstdint>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common.hh"
#include "TreFile.hh"
@@ -26,51 +19,130 @@ struct Table1Entry { // little endian
uint32_t table3Ptr;
} __attribute__((__packed__));
struct Table3Entry { // little endian
unsigned dataPtr:32;
// unsigned unknown1:8;
unsigned length:24;
unsigned unknown2:8;
} __attribute__((__packed__));
TreFile::TreFile(std::string const& filename)
: file_(nullptr), start_(0), length_(0)
TreFile::TreFile(char const* base, size_t length)
: base_(base), length_(length)
{
file_ = fopen(filename.c_str(), "rb");
if (!file_)
throw POSIXException{errno, "Could not open "s + filename};
if (length_ < sizeof(TreHeader))
throw FormatException{"Size < header size"};
{
struct stat statBuf;
TreHeader const* header = reinterpret_cast<TreHeader const*>(base);
if (memcmp(header->magic, "XTRE", 4) != 0)
throw FormatException{"Wrong magic, not a TRE?"};
if ((header->table1Ofs > length_) ||
(header->table2Ofs > length_) ||
(header->table3Ofs > length_) ||
(header->dataOfs > length_))
throw FormatException{"Table offset exceeds length"};
if ((header->table1Ofs > header->table2Ofs) ||
(header->table2Ofs > header->table3Ofs) ||
(header->table3Ofs > header->dataOfs))
throw FormatException{"Table offsets not in ascending order"};
const size_t numTable1Entries = (header->table2Ofs-header->table1Ofs)/sizeof(Table1Entry);
table3Size_ = (header->dataOfs-header->table3Ofs)/sizeof(Table3Entry);
#ifndef NDEBUG
printf("Table 1: start %u, length %u (%lu entries)\n",
header->table1Ofs, header->table2Ofs-header->table1Ofs,
numTable1Entries);
printf("Table 2: start %u, length %u\n",
header->table2Ofs, header->table3Ofs-header->table2Ofs);
printf("Table 3: start %u, length %u (%lu entries)\n",
header->table3Ofs, header->dataOfs-header->table3Ofs,
table3Size_);
printf("Data: start %u, length %lu\n",
header->dataOfs, length_ - header->dataOfs);
#endif
// Check Table 3
table3_ = reinterpret_cast<Table3Entry const*>(base+header->table3Ofs);
for (size_t i = 0;i < table3Size_;++i) {
if ((table3_[i].dataPtr > length_) ||
(table3_[i].dataPtr < header->dataOfs))
throw FormatException{"Data pointer out of range"};
if ((table3_[i].dataPtr + table3_[i].length) > length_) {
fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", table3_[i].dataPtr, table3_[i].length);
}
}
// Read Table 2
std::map<size_t, uint32_t> table2Pos;
if (fstat(fileno(file_), &statBuf) != 0) {
fclose(file_);
throw POSIXException(errno, "Could not stat");
size_t pos = header->table2Ofs;
while((pos+5) < header->table3Ofs) {
uint8_t const* nameLen = reinterpret_cast<uint8_t const*>(base+pos);
if (pos+*nameLen+5 > header->table3Ofs)
throw FormatException{"Table 2 entry exceeds table "s + std::to_string(*nameLen)};
std::string nameStr(base+pos+1, *nameLen);
uint32_t const* table3Ptr = reinterpret_cast<uint32_t const*>(base+pos+*nameLen+1);
if ((*table3Ptr < header->table3Ofs) ||
(*table3Ptr >= header->dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
length_ = statBuf.st_size;
const size_t table3Index = (*table3Ptr - header->table3Ofs)/sizeof(Table3Entry);
auto ins = table2_.emplace(nameStr, table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for name %s\n", nameStr.c_str());
else
printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(),
ins.first->second, table3Index);
}
// Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's
table2Pos.emplace(pos, *table3Ptr);
pos += 1+*nameLen+4;
}
try {
construct_();
} catch(...) {
fclose(file_);
throw;
// Read Table 1
Table1Entry const* table1 = reinterpret_cast<Table1Entry const*>(base+header->table1Ofs);
for (size_t i = 0;i < numTable1Entries;++i) {
if ((table1[i].crc == 0x00000000) &&
(table1[i].table3Ptr == 0xffffffff))
continue; // Unused entry
if ((table1[i].table3Ptr < header->table2Ofs) ||
(table1[i].table3Ptr >= header->dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
uint32_t table3Ptr = table1[i].table3Ptr;
if (table3Ptr < header->table3Ofs) {
auto it = table2Pos.find(table3Ptr);
if (it == table2Pos.end()) {
fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n",
table1[i].crc, table3Ptr);
continue;
}
printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n",
table3Ptr, it->second);
table3Ptr = it->second;
}
const size_t table3Index = (table3Ptr - header->table3Ofs)/sizeof(Table3Entry);
auto ins = table1_.emplace(static_cast<uint32_t>(table1[i].crc), table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for CRC %.8x\n", table1[i].crc);
else
printf("Collision for CRC %.8x: prev %lu, new %lu\n", table1[i].crc, ins.first->second, table3Index);
}
}
}
TreFile::TreFile(FILE* file, off_t start, off_t length)
: file_(file), start_(start), length_(length)
{
construct_();
}
TreFile::~TreFile()
{
if (file_) {
fclose(file_);
file_ = nullptr;
}
}
std::vector<std::string> TreFile::getNames() const
@@ -93,12 +165,12 @@ std::vector<uint32_t> TreFile::getCRCs() const
return ret;
}
TreFile::File TreFile::openName(std::string const& name) const
TreFile::Object TreFile::openName(std::string const& name) const
{
return openIdx_(findName_(name));
}
TreFile::File TreFile::openCRC(uint32_t crc) const
TreFile::Object TreFile::openCRC(uint32_t crc) const
{
return openIdx_(findCRC_(crc));
}
@@ -113,7 +185,7 @@ TreFile::Stat TreFile::statCRC(uint32_t crc) const
return statIdx_(findCRC_(crc));
}
void TreFile::dumpName(std::string path, std::string const& name)
void TreFile::dumpName(std::string path, std::string const& name) const
{
auto idx = findName_(name);
@@ -124,7 +196,7 @@ void TreFile::dumpName(std::string path, std::string const& name)
dumpIdx_(path, idx);
}
void TreFile::dumpCRC(std::string path, uint32_t crc)
void TreFile::dumpCRC(std::string path, uint32_t crc) const
{
auto idx = findCRC_(crc);
char crcStr[9];
@@ -137,7 +209,7 @@ void TreFile::dumpCRC(std::string path, uint32_t crc)
dumpIdx_(path, idx);
}
void TreFile::dumpAll(std::string path)
void TreFile::dumpAll(std::string path) const
{
if (path.back() != '/')
path.push_back('/');
@@ -163,177 +235,22 @@ void TreFile::printStructure()
{
printf("Table 1: %lu entries\n", table1_.size());
printf("Table 2: %lu entries\n", table2_.size());
printf("Table 3: %lu entries\n", table3_.size());
printf("Table 3: %lu entries\n", table3Size_);
printf("Files by CRC:\n");
for(auto const& ent : table1_) {
printf("\t%.8x -> (ofs %.8x, len %.8x, flags %.2hhx)\n", ent.first,
std::get<0>(table3_[ent.second]),
std::get<1>(table3_[ent.second]),
std::get<2>(table3_[ent.second]));
table3_[ent.second].dataPtr,
table3_[ent.second].length,
table3_[ent.second].flags);
}
printf("Files by Name:\n");
for(auto const& ent : table2_) {
printf("\t%s -> (ofs %.8x, len %.8x, flags %.2hhx)\n", ent.first.c_str(),
std::get<0>(table3_[ent.second]),
std::get<1>(table3_[ent.second]),
std::get<2>(table3_[ent.second]));
}
}
void TreFile::construct_()
{
if (length_ < sizeof(TreHeader))
throw FormatException{"Size < header size"};
if (fseeko(file_, start_, SEEK_SET) != 0) {
throw POSIXException(errno, "Could not seek");
}
TreHeader header;
if (fread(&header, sizeof(TreHeader), 1, file_) != 1)
throw POSIXException{errno, "Could not read header"};
if (memcmp(header.magic, "XTRE", 4) != 0)
throw FormatException{"Wrong magic, not a TRE?"};
if ((header.table1Ofs > length_) ||
(header.table2Ofs > length_) ||
(header.table3Ofs > length_) ||
(header.dataOfs > length_))
throw FormatException{"Table offset exceeds length"};
if ((header.table1Ofs > header.table2Ofs) ||
(header.table2Ofs > header.table3Ofs) ||
(header.table3Ofs > header.dataOfs))
throw FormatException{"Table offsets not in ascending order"};
const size_t numTable1Entries = (header.table2Ofs-header.table1Ofs)/sizeof(Table1Entry);
const size_t numTable3Entries = (header.dataOfs-header.table3Ofs)/sizeof(Table3Entry);
printf("Table 1: start %u, length %u (%lu entries)\n",
header.table1Ofs, header.table2Ofs-header.table1Ofs,
numTable1Entries);
printf("Table 2: start %u, length %u\n",
header.table2Ofs, header.table3Ofs-header.table2Ofs);
printf("Table 3: start %u, length %u (%lu entries)\n",
header.table3Ofs, header.dataOfs-header.table3Ofs,
numTable3Entries);
printf("Data: start %u, length %lu\n",
header.dataOfs, length_ - header.dataOfs);
// Read Table 3
if (fseeko(file_, start_+header.table3Ofs, SEEK_SET) != 0) {
throw POSIXException{errno, "Could not seek"};
}
table3_.reserve(numTable3Entries);
for (size_t i = 0;i < numTable3Entries;++i) {
Table3Entry entry;
if (fread(&entry, sizeof(Table3Entry), 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if ((entry.dataPtr > length_) ||
(entry.dataPtr < header.dataOfs))
throw FormatException{"Data pointer out of range"};
if ((entry.dataPtr + entry.length) > length_) {
fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", entry.dataPtr, entry.length);
//throw FormatException{"Data length exceeds file length"};
}
table3_.push_back(std::make_tuple(static_cast<uint32_t>(entry.dataPtr),
static_cast<uint32_t>(entry.length),
static_cast<uint8_t>(entry.unknown2)));
}
// Read Table 2
if (fseeko(file_, start_+header.table2Ofs, SEEK_SET) != 0) {
throw POSIXException{errno, "Could not seek"};
}
std::map<size_t, uint32_t> table2Pos;
size_t pos = header.table2Ofs;
while((pos+5) < header.table3Ofs) {
uint8_t nameLen;
if (fread(&nameLen, 1, 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if (pos+nameLen+5 > header.table3Ofs)
throw FormatException{"Table 2 entry exceeds table "s + std::to_string(nameLen)};
std::vector<char> nameData(nameLen);
if (fread(nameData.data(), nameLen, 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
uint32_t table3Ptr;
if (fread(&table3Ptr, 4, 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if ((table3Ptr < header.table3Ofs) ||
(table3Ptr >= header.dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
std::string nameStr{nameData.data(), nameLen};
size_t table3Index = (table3Ptr - header.table3Ofs)/sizeof(Table3Entry);
auto ins = table2_.emplace(nameStr, table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for name %s\n", nameStr.c_str());
else
printf("Collision for name %s: prev %lu, new %lu\n", nameStr.c_str(),
ins.first->second, table3Index);
}
// Store an position-indexed copy of Table2 to resolve Table1->Table2 ptr's
table2Pos.emplace(pos, table3Ptr);
pos += 1+nameLen+4;
}
// Read Table 1
if (fseeko(file_, start_+header.table1Ofs, SEEK_SET) != 0) {
throw POSIXException{errno, "Could not seek"};
}
for (size_t i = 0;i < numTable1Entries;++i) {
Table1Entry entry;
if (fread(&entry, sizeof(Table1Entry), 1, file_) != 1)
throw POSIXException{errno, "Could not read"};
if ((entry.crc == 0x00000000) &&
(entry.table3Ptr == 0xffffffff))
continue; // Unused entry
if ((entry.table3Ptr < header.table2Ofs) ||
(entry.table3Ptr >= header.dataOfs)) {
throw FormatException{"Table3 pointer out of range"};
}
if (entry.table3Ptr < header.table3Ofs) {
auto it = table2Pos.find(entry.table3Ptr);
if (it == table2Pos.end()) {
fprintf(stderr, "Table1->Table2 pointer: Table 2 entry not found: crc %.8x ptr %.8x\n",
entry.crc, entry.table3Ptr);
continue;
}
printf("Table1->Table2 pointer resolved: Table 2 %.8x -> Table 3 %.8x\n",
entry.table3Ptr, it->second);
entry.table3Ptr = it->second;
}
size_t table3Index = (entry.table3Ptr - header.table3Ofs)/sizeof(Table3Entry);
auto ins = table1_.emplace(static_cast<uint32_t>(entry.crc), table3Index);
if (!ins.second) {
if (ins.first->second == table3Index)
printf("Duplicate entry for CRC %.8x\n", entry.crc);
else
printf("Collision for CRC %.8x: prev %lu, new %lu\n", entry.crc, ins.first->second, table3Index);
}
table3_[ent.second].dataPtr,
table3_[ent.second].length,
table3_[ent.second].flags);
}
}
@@ -358,56 +275,37 @@ size_t TreFile::findCRC_(uint32_t crc) const
return it->second;
}
void TreFile::dumpIdx_(std::string const& name, size_t table3Idx)
void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const
{
uint32_t ofs, len;
std::tie(ofs, len, std::ignore) = table3_[table3Idx];
// if (ofs%2 != 0) {
// ++ofs;
// len;
// }
std::vector<char> buf(len);
if (fseeko(file_, start_+ofs, SEEK_SET) != 0)
throw POSIXException{errno, "Could not seek"};
if (fread(buf.data(), len, 1, file_) != 1)
throw POSIXException{errno, "Could not read data"};
if(table3_[table3Idx].flags&0x80)
throw Exception{"Compressed TRE objects NYI"};
FILEUPtr outFile{fopen(name.c_str(), "wb")};
if (!outFile)
throw POSIXException{errno, "Could not open " + name};
if (fwrite(buf.data(), len, 1, outFile.get()) != 1)
if (fwrite(base_+table3_[table3Idx].dataPtr, table3_[table3Idx].length, 1, outFile.get()) != 1)
throw POSIXException{errno, "Could not write"};
}
TreFile::File TreFile::openIdx_(size_t table3Idx) const
TreFile::Object TreFile::openIdx_(size_t table3Idx) const
{
uint32_t ofs, len;
uint8_t flags;
std::tie(ofs, len, flags) = table3_[table3Idx];
if(flags&0x80)
if(table3_[table3Idx].flags&0x80)
throw Exception{"Compressed TRE objects NYI"};
if ((table3_[table3Idx].dataPtr + table3_[table3Idx].length) > length_)
throw FormatException{"length exceeds file size"};
std::vector<char> buf(len);
if (fseeko(file_, start_+ofs, SEEK_SET) != 0)
throw POSIXException{errno, "Could not seek"};
if (fread(buf.data(), len, 1, file_) != 1)
throw POSIXException{errno, "Could not read data"};
return File{buf};
return Object(base_+table3_[table3Idx].dataPtr,
table3_[table3Idx].length);
}
TreFile::Stat TreFile::statIdx_(size_t table3Idx) const
{
Stat ret;
std::tie(std::ignore, ret.size, ret.flags) = table3_[table3Idx];
ret.size = table3_[table3Idx].length;
ret.flags = table3_[table3Idx].flags;
return ret;
}