matthias/wc3re
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Generalized LZ-style decompression; extend MveDecoder to decode in-flight MVEs

Browse Source
This commit is contained in:
Matthias Blankertz
2015-04-29 14:43:28 +02:00
parent 962bfed0cb
commit d6bbdabaa6
10 changed files with 291 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
IffFile.cc
View File

@@ -10,15 +10,13 @@ struct ChunkHeader {
};
IffFile::IffFile(char const* base, size_t length)
: base_(base), length_(length), root_(nullptr)
: base_(base), length_(length)
{
root_ = parseObject(base, length);
#ifndef NDEBUG
if (length > (root_->getSize()+8)) {
printf("%lu excess bytes parsing IFF\n", length-(root_->getSize()+8));
size_t pos = 0;
while (pos < length) {
roots_.push_back(parseObject(base+pos, length-pos));
pos += roots_.back()->getSize() + 8;
}
#endif
}
IffFile::~IffFile()
@@ -47,7 +45,8 @@ static void _printStructure(IffFile::Object const& obj, unsigned level)
void IffFile::printStructure(unsigned level) const
{
_printStructure(*root_, level);
for (auto& root : *this)
_printStructure(root, level);
}

200
IffFile.hh
View File

@@ -13,6 +13,102 @@ public:
~IffFile();
class Object;
class Form;
class ObjectIterator : public std::iterator<std::random_access_iterator_tag, const Object> {
public:
Object const& operator*() const {
return **implIt_;
}
ObjectIterator& operator++() {
++implIt_;
return *this;
}
ObjectIterator& operator--() {
--implIt_;
return *this;
}
bool operator!=(ObjectIterator const& other) const {
return (implIt_ != other.implIt_);
}
bool operator>(ObjectIterator const& other) const {
return (implIt_ > other.implIt_);
}
bool operator>=(ObjectIterator const& other) const {
return (implIt_ >= other.implIt_);
}
bool operator<(ObjectIterator const& other) const {
return (implIt_ < other.implIt_);
}
bool operator<=(ObjectIterator const& other) const {
return (implIt_ <= other.implIt_);
}
Object const* operator->() const {
return implIt_->get();
}
ObjectIterator operator++(int) {
ObjectIterator ret = *this;
++implIt_;
return ret;
}
ObjectIterator operator--(int) {
ObjectIterator ret = *this;
--implIt_;
return ret;
}
ObjectIterator& operator+=(ptrdiff_t n) {
implIt_ += n;
return *this;
}
ObjectIterator operator+(ptrdiff_t n) const {
ObjectIterator ret = *this;
ret += n;
return ret;
}
ObjectIterator& operator-=(ptrdiff_t n) {
implIt_ -= n;
return *this;
}
ObjectIterator operator-(ptrdiff_t n) const {
ObjectIterator ret = *this;
ret -= n;
return ret;
}
ptrdiff_t operator-(ObjectIterator const& other) const {
return implIt_ - other.implIt_;
}
Object const& operator[](ptrdiff_t n) const {
return *(implIt_[n]);
}
private:
ObjectIterator(std::vector<std::unique_ptr<Object> >::const_iterator implIt)
: implIt_(std::move(implIt)) {
}
friend class Form;
friend class IffFile;
std::vector<std::unique_ptr<Object> >::const_iterator implIt_;
};
class Object {
public:
Object(std::string type, char const* base, size_t length);
@@ -63,98 +159,6 @@ public:
return children_.size();
}
class ObjectIterator : public std::iterator<std::random_access_iterator_tag, const Object> {
public:
Object const& operator*() const {
return **implIt_;
}
ObjectIterator& operator++() {
++implIt_;
return *this;
}
ObjectIterator& operator--() {
--implIt_;
return *this;
}
bool operator!=(ObjectIterator const& other) const {
return (implIt_ != other.implIt_);
}
bool operator>(ObjectIterator const& other) const {
return (implIt_ > other.implIt_);
}
bool operator>=(ObjectIterator const& other) const {
return (implIt_ >= other.implIt_);
}
bool operator<(ObjectIterator const& other) const {
return (implIt_ < other.implIt_);
}
bool operator<=(ObjectIterator const& other) const {
return (implIt_ <= other.implIt_);
}
Object const* operator->() const {
return implIt_->get();
}
ObjectIterator operator++(int) {
ObjectIterator ret = *this;
++implIt_;
return ret;
}
ObjectIterator operator--(int) {
ObjectIterator ret = *this;
--implIt_;
return ret;
}
ObjectIterator& operator+=(ptrdiff_t n) {
implIt_ += n;
return *this;
}
ObjectIterator operator+(ptrdiff_t n) const {
ObjectIterator ret = *this;
ret += n;
return ret;
}
ObjectIterator& operator-=(ptrdiff_t n) {
implIt_ -= n;
return *this;
}
ObjectIterator operator-(ptrdiff_t n) const {
ObjectIterator ret = *this;
ret -= n;
return ret;
}
ptrdiff_t operator-(ObjectIterator const& other) const {
return implIt_ - other.implIt_;
}
Object const& operator[](ptrdiff_t n) const {
return *(implIt_[n]);
}
private:
ObjectIterator(std::vector<std::unique_ptr<Object> >::const_iterator implIt)
: implIt_(std::move(implIt)) {
}
friend class Form;
std::vector<std::unique_ptr<Object> >::const_iterator implIt_;
};
ObjectIterator childrenBegin() const {
return ObjectIterator(children_.cbegin());
}
@@ -168,9 +172,17 @@ public:
};
Object const& getRoot() const {
return *root_;
return *roots_[0];
}
ObjectIterator begin() const {
return ObjectIterator(roots_.cbegin());
}
ObjectIterator end() const {
return ObjectIterator(roots_.cend());
}
void printStructure(unsigned level = 0) const;
private:
@@ -178,7 +190,7 @@ private:
char const* base_;
const size_t length_;
std::unique_ptr<Object> root_;
std::vector<std::unique_ptr<Object> > roots_;
};
#endif

4
Makefile
View File

@@ -5,13 +5,13 @@ LDOPTS=-Wl,--sort-common,--as-needed
iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc exceptions.cc
iffexplore_LIBS ::=
treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc
treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc decompress.cc
treexplore_LIBS ::=
font2png_CXXSRCS ::= font2png.cc
font2png_LIBS ::= -lpng
mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc
mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc decompress.cc
mvedecode_LIBS ::= -lSDL2
progs ::= iffexplore treexplore mvedecode

156
MveDecoder.cc
View File

@@ -4,6 +4,7 @@
#include "common.hh"
#include "util.hh"
#include "IffFile.hh"
#include "decompress.hh"
#include "MveDecoder.hh"
MveDecoder::MveDecoder(char const* base, size_t length)
@@ -32,26 +33,31 @@ MveDecoder::MveDecoder(char const* base, size_t length)
printf("_PC_: %u PALTs\n", PC_PALTcount);
#endif
// Parse SOND chunk
auto& SOND = *it++;
if ((SOND.getType() != "SOND") ||
(SOND.getSize() != 4))
throw FormatException{"SOND chunk missing or wrong size"};
// // Parse SOND chunk
// auto& SOND = *it++;
// if ((SOND.getType() != "SOND") ||
// (SOND.getSize() != 4))
// throw FormatException{"SOND chunk missing or wrong size"};
const uint32_t SONDc = readU32LE(SOND.begin());
if (SONDc != 3)
throw FormatException{"Unexpected SOND value"};
// const uint32_t SONDc = readU32LE(SOND.begin());
// if (SONDc != 3)
// throw FormatException{"Unexpected SOND value"};
// Parse data
unsigned curPALT = 0, curBRCH = 0;
bool haveIndex = false;
std::vector<IffFile::Object const*> PALTs;
// Store branch offsets, resolve to shots later
std::vector<uint32_t> branches;
Shot *curSHOT = nullptr;
for (;it != rootForm.childrenEnd();++it) {
if (it->getType() == "PALT") {
if (it->getType() == "SIZE") {
if (it->getSize() != 8)
throw FormatException{"Unexpected SIZE size"};
width_ = readU32LE(it->begin());
height_ = readU32LE(it->begin()+4);
} else if (it->getType() == "PALT") {
if (curPALT > PC_PALTcount)
throw FormatException{"Number of PALT chunks exceeds amount specified in _PC_"};
if (it->getSize() != 768)
@@ -99,11 +105,13 @@ MveDecoder::MveDecoder(char const* base, size_t length)
curBRCH = (idxIt-branches.begin());
} else if (it->getType() == "TEXT") {
// Subtitle NYI
} else if (it->getType() == "SOND") {
// ignore
} else
throw FormatException{"Encountered unexpected chunk: " + it->getType()};
}
#ifdef VIDDEBUG_
#ifdef VIDDEBUG_
unsigned i = 0;
#endif
for (auto& brch : branches_) {
@@ -113,26 +121,15 @@ MveDecoder::MveDecoder(char const* base, size_t length)
i++, std::find(palts_.begin(), palts_.end(), shot.palt)-palts_.begin(),
shot.VGAs.size(), shot.AUDIs.size());
#endif
if (shot.VGAs.size() != shot.AUDIs.size())
throw FormatException{"Video/audio block count mismatch"};
if ((shot.AUDIs.size() != 0) &&
(shot.VGAs.size() != shot.AUDIs.size()))
throw FormatException{"Video/audio block count mismatch: V " +
std::to_string(shot.VGAs.size()) + ", A " +
std::to_string(shot.AUDIs.size())};
}
}
}
char *binify(uint8_t a) {
static char buf[9];
buf[8] = '\0';
for(unsigned i = 0;i < 8;++i) {
if(a&0x80)
buf[i] = '1';
else
buf[i] = '0';
a <<= 1;
}
return buf;
}
std::vector<char> parseHuff_(char const* data, size_t len)
{
uint8_t numHuffVals;
@@ -201,97 +198,13 @@ std::vector<char> parseHuff_(char const* data, size_t len)
std::vector<uint8_t> parsePixels_(char const* data, size_t len)
{
std::vector<uint8_t> ret;
if (*data == 0x02) {
size_t pos = 1;
while (pos < len) {
uint8_t b = *(data+pos++);
if (!((b&0xe0)==0xe0)) {
unsigned size = 0, replSize = 0;
unsigned replOfs = 0;
if (!(b&0x80)) {
#ifdef PIXELDEBUG_
printf("Code: Repl1 %.2hhx\n", b);
#endif
if (pos >= len)
throw FormatException{"Pixel data overrun"};
uint8_t ofs = *(data+pos++);
size = b&0x3;
replSize = ((b&0x1c)>>2) + 3;
replOfs = ret.size()-(((b&0x60)<<3)+ofs+1)+size;
} else if (!(b&0x40)) {
#ifdef PIXELDEBUG_
printf("Code: Repl2 %.2hhx\n", b);
#endif
if (pos+1 >= len)
throw FormatException{"Pixel data overrun"};
uint8_t b1 = *(data+pos++);
uint8_t b2 = *(data+pos++);
size = (b1&0xc0)>>6;
replSize = (b&0x3f)+4;
replOfs = ret.size()-(((b1&0x3f)<<8)+b2+1)+size;
} else if (!(b&0x20)) {
#ifdef PIXELDEBUG_
printf("Code: Repl3 %.2hhx\n", b);
#endif
if (pos+2 >= len)
throw FormatException{"Pixel data overrun"};
uint8_t b1 = *(data+pos++);
uint8_t b2 = *(data+pos++);
uint8_t b3 = *(data+pos++);
size = b&0x3;
replSize = b3+5+((b&0xc)<<6);
replOfs = ret.size()-(((b&0x10)<<12)+1+(b1<<8)+b2)+size;
}
if (pos+size >= len)
throw FormatException{"Pixel data overrun"};
std::copy(data+pos, data+pos+size,
std::back_inserter(ret));
pos += size;
if (replOfs >= ret.size())
throw FormatException{"Replication offset exceeds buffer"};
for (unsigned i = 0;i < replSize;++i)
ret.push_back(ret[replOfs+i]);
} else {
#ifdef PIXELDEBUG_
printf("Code: Copy %.2hhx\n", b);
#endif
unsigned size = (b&0x1f)*4+4;
if (size > 0x70) {
if (pos+(b&0x3) > len)
throw FormatException{"Pixel data overrun"};
std::copy(data+pos, data+pos+(b&0x3),
std::back_inserter(ret));
pos += (b&0x3);
#ifndef NDEBUG
if (pos < len)
printf("%lu unparsed bytes in pixel data\n", len-pos);
#endif
break;
}
if (pos+size > len)
throw FormatException{"Pixel data overrun"};
std::copy(data+pos, data+pos+size,
std::back_inserter(ret));
pos += size;
}
}
} else
return decompressLZ(data+1, len-1);
} else {
std::vector<uint8_t> ret;
std::copy(data+1, data+len, std::back_inserter(ret));
return ret;
return ret;
}
}
MveDecoder::Frame MveDecoder::parseVGA(IffFile::Object const& vga, Frame const* prev) const
@@ -583,9 +496,13 @@ void MveDecoder::play(unsigned branch) const
}
AudioCBData audioCBData;
audioCBData.cur = &*curShot;
if (curShot->AUDIs.size() > 0) {
audioCBData.cur = &*curShot;
audioCBData.aCur = curShot->AUDIs.cbegin();
} else {
audioCBData.cur = nullptr;
}
audioCBData.next = nullptr;
audioCBData.aCur = curShot->AUDIs.cbegin();
audioCBData.pos = 0;
SDL_AudioSpec want;
@@ -666,7 +583,7 @@ void MveDecoder::play(unsigned branch) const
SDL_Surface *vidSurfRGB = SDL_ConvertSurface(vidSurf, screen->format, 0);
if (!vidSurfRGB) {
printf("Could not get video surface: %s\n", SDL_GetError());
printf("Could not convert video surface: %s\n", SDL_GetError());
SDL_FreeSurface(vidSurf);
SDL_CloseAudioDevice(audioDev);
SDL_FreePalette(palette);
@@ -709,6 +626,7 @@ void MveDecoder::play(unsigned branch) const
if (SDL_SetPaletteColors(palette, colors.data(), 0, 256) != 0) {
printf("Could not set palette: %s\n", SDL_GetError());
SDL_CloseAudioDevice(audioDev);
SDL_FreePalette(palette);
SDL_DestroyWindow(window);
SDL_Quit();

29
TreFile.cc
View File

@@ -4,6 +4,7 @@
#include "common.hh"
#include "util.hh"
#include "decompress.hh"
#include "TreFile.hh"
struct TreHeader { // little endian
@@ -293,22 +294,14 @@ size_t TreFile::findCRC_(uint32_t crc) const
void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const
{
uint32_t dataPtr, length;
uint8_t flags;
std::tie(dataPtr, length, flags) = table3_[table3Idx];
auto obj = openIdx_(table3Idx);
if(flags&0x80)
throw Exception{"Compressed TRE objects NYI"};
if ((dataPtr + length) > length_)
throw FormatException{"length exceeds file size"};
FILEUPtr outFile{fopen(name.c_str(), "wb")};
if (!outFile)
throw POSIXException{errno, "Could not open " + name};
if (fwrite(base_+dataPtr, length, 1, outFile.get()) != 1)
throw POSIXException{errno, "Could not write"};
if (fwrite(obj.data(), obj.size(), 1, outFile.get()) != 1)
throw POSIXException{errno, "Could not write"};
}
TreFile::Object TreFile::openIdx_(size_t table3Idx) const
@@ -316,15 +309,19 @@ TreFile::Object TreFile::openIdx_(size_t table3Idx) const
uint32_t dataPtr, length;
uint8_t flags;
std::tie(dataPtr, length, flags) = table3_[table3Idx];
if(flags&0x80)
throw Exception{"Compressed TRE objects NYI"};
if ((dataPtr + length) > length_)
throw FormatException{"length exceeds file size"};
return Object(base_+dataPtr,
length);
if (flags&0x80) {
if (flags&0x40) {
return Object(decompressLZ(base_+dataPtr, length));
} else
throw Exception{"Compression type 0 NYI"};
} else {
return Object(base_+dataPtr,
length);
}
}
TreFile::Stat TreFile::statIdx_(size_t table3Idx) const

27
TreFile.hh
View File

@@ -6,6 +6,7 @@
#include <cstdio>
#include <vector>
#include <tuple>
#include <experimental/optional>
class TreFile {
public:
@@ -29,22 +30,28 @@ public:
Object(Object const& copy) = delete;
Object(Object && move)
: base_(move.base_), length_(move.length_) {
: data_(std::move(move.data_)), base_(move.base_), length_(move.length_) {
move.data_ = std::experimental::nullopt;
move.base_ = nullptr;
move.length_ = 0;
}
Object& operator=(Object const& copy) = delete;
Object& operator=(Object && move) {
data_ = std::move(move.data_);
base_ = move.base_;
length_ = move.length_;
move.data_ = std::experimental::nullopt;
move.base_ = nullptr;
move.length_ = 0;
return *this;
}
char const* data() const {
return base_;
if (data_)
return (char const*)(data_->data());
else
return base_;
}
size_t size() const {
@@ -52,20 +59,32 @@ public:
}
operator bool() const {
return base_;
return data_ || base_;
}
private:
// Object either owns data (when decompressed) ...
std::experimental::optional<std::vector<uint8_t> > data_;
// .. or points to area in underlying data (when not compressed)
char const* base_;
size_t length_;
Object(char const* base, size_t length)
: base_(base), length_(length) {
: data_(), base_(base), length_(length) {
}
Object(std::vector<uint8_t> data)
: data_(std::move(data)), base_(nullptr), length_(data_->size()) {
}
friend class TreFile;
};
/* Open an object of the TRE file by name or by CRC
* The returned object should be destroyed when no longer needed as it may
* hold a memory allocation for its data. But it must not be used after the
* TreFile object is destroyed.
*/
Object openName(std::string const& name) const;
Object openCRC(uint32_t crc) const;

82
decompress.cc Normal file
View File

@@ -0,0 +1,82 @@
#include "common.hh"
#include "decompress.hh"
std::vector<uint8_t> decompressLZ(char const* data, size_t len)
{
std::vector<uint8_t> ret;
size_t pos = 0;
while (pos < len) {
uint8_t b = *(data+pos++);
if (!((b&0xe0)==0xe0)) {
unsigned size = 0, replSize = 0;
unsigned replOfs = 0;
if (!(b&0x80)) {
if (pos >= len)
throw FormatException{"Compressed stream overrun"};
uint8_t ofs = *(data+pos++);
size = b&0x3;
replSize = ((b&0x1c)>>2) + 3;
replOfs = ((b&0x60)<<3)+ofs+1;
} else if (!(b&0x40)) {
if (pos+1 >= len)
throw FormatException{"Compressed stream overrun"};
uint8_t b1 = *(data+pos++);
uint8_t b2 = *(data+pos++);
size = (b1&0xc0)>>6;
replSize = (b&0x3f)+4;
replOfs = ((b1&0x3f)<<8)+b2+1;
} else if (!(b&0x20)) {
if (pos+2 >= len)
throw FormatException{"Compressed stream overrun"};
uint8_t b1 = *(data+pos++);
uint8_t b2 = *(data+pos++);
uint8_t b3 = *(data+pos++);
size = b&0x3;
replSize = b3+5+((b&0xc)<<6);
replOfs = ((b&0x10)<<12)+1+(b1<<8)+b2;
}
if (pos+size >= len)
throw FormatException{"Compressed stream overrun"};
std::copy(data+pos, data+pos+size,
std::back_inserter(ret));
pos += size;
if (replOfs > ret.size())
throw FormatException{"Replication offset exceeds buffer"};
unsigned start = ret.size()-replOfs;
for (unsigned i = 0;i < replSize;++i)
ret.push_back(ret[start+i]);
} else {
unsigned size = (b&0x1f)*4+4;
if (size > 0x70) {
if (pos+(b&0x3) > len)
throw FormatException{"Compressed stream overrun"};
std::copy(data+pos, data+pos+(b&0x3),
std::back_inserter(ret));
pos += (b&0x3);
#ifndef NDEBUG
if (pos < len)
printf("%lu unparsed bytes in compressed data\n", len-pos);
#endif
break;
}
if (pos+size > len)
throw FormatException{"Compressed stream overrun"};
std::copy(data+pos, data+pos+size,
std::back_inserter(ret));
pos += size;
}
}
return ret;
}

14
decompress.hh Normal file
View File

@@ -0,0 +1,14 @@
#ifndef WC3RE_DECOMPRESS_HH__
#define WC3RE_DECOMPRESS_HH__
#include <cstdint>
#include <vector>
/* Decompression routines for various proprietary compression formats found in
WC3 */
/* LZ77-like format consisting of codes specifing copies from the input stream
and/or replication of previously output data */
std::vector<uint8_t> decompressLZ(char const* data, size_t len);
#endif

7
iffexplore.cc
View File

@@ -91,8 +91,11 @@ int main(int argc, char *argv[])
IffFile iff{mmap.data(), mmap.size()};
if (printStructure) {
unsigned blobCount = 0;
iffDumper(iff.getRoot(), dumpBlobs, dumpPath, blobCount);
unsigned blobCount = 0, subdoc = 0;
for (auto& root : iff) {
printf("Subdocument %u:\n", subdoc++);
iffDumper(root, dumpBlobs, dumpPath, blobCount);
}
}
} catch (POSIXException &ex) {
fflush(stdout);

4
treexplore.cc
View File

@@ -70,7 +70,7 @@ int main(int argc, char *argv[]) {
if (dumpIff) {
for(auto name : file.getNames()) {
auto s = file.statName(name);
if (s.flags&0x80)
if ((s.flags&0xc0) == 0x80)
continue;
auto f = file.openName(name);
try {
@@ -84,7 +84,7 @@ int main(int argc, char *argv[]) {
for(auto crc : file.getCRCs()) {
auto s = file.statCRC(crc);
if (s.flags&0x80)
if ((s.flags&0xc0) == 0x80)
continue;
auto f = file.openCRC(crc);
try {