Generalized LZ-style decompression; extend MveDecoder to decode in-flight MVEs

IffFile.cc

@@ -10,15 +10,13 @@ struct ChunkHeader {
 };
 
 IffFile::IffFile(char const* base, size_t length)
-  : base_(base), length_(length), root_(nullptr)
+  : base_(base), length_(length)
 {
-  root_ = parseObject(base, length);
+  size_t pos = 0;
-  
+  while (pos < length) {
-#ifndef NDEBUG
+    roots_.push_back(parseObject(base+pos, length-pos));
-  if (length > (root_->getSize()+8)) {
+    pos += roots_.back()->getSize() + 8;
-    printf("%lu excess bytes parsing IFF\n", length-(root_->getSize()+8));
   }
-#endif 
 }
 
 IffFile::~IffFile()
@@ -47,7 +45,8 @@ static void _printStructure(IffFile::Object const& obj, unsigned level)
 
 void IffFile::printStructure(unsigned level) const
 {
-  _printStructure(*root_, level);
+  for (auto& root : *this)
+    _printStructure(root, level);
 }
 
 

IffFile.hh

@@ -13,55 +13,8 @@ public:
 
   ~IffFile();
 
-  class Object {
+  class Object;
-  public:
+  class Form;
-    Object(std::string type, char const* base, size_t length);
-    Object(Object const& copy) = delete;
-   
-    virtual ~Object() {
-    }
-
-    std::string const& getType() const {
-      return type_;
-    }
-    
-    bool isForm() const {
-      return (typeid(*this) == typeid(Form));
-    }
-    
-    size_t getSize() const {
-      return length_;
-    }
-
-    char const* begin() const {
-      return base_;
-    }
-    
-    char const* end() const {
-      return base_+length_;
-    }
-
-    operator std::string() const;
-    
-  protected:
-    char const* base_;
-    const size_t length_;
-    std::string const type_;
-  };
-  
-  class Form final : public Object {
-  public:
-    Form(std::string type, char const* base, size_t length);
-    
-    ~Form() {}
-    
-    std::string const& getSubtype() const {
-      return subtype_;
-    }
-    
-    size_t getChildCount() const {
-      return children_.size();
-    }
   
   class ObjectIterator : public std::iterator<std::random_access_iterator_tag, const Object> {
   public:
@@ -151,10 +104,61 @@ public:
     }
 
     friend class Form;
+    friend class IffFile;
       
     std::vector<std::unique_ptr<Object> >::const_iterator implIt_;
   };
   
+  class Object {
+  public:
+    Object(std::string type, char const* base, size_t length);
+    Object(Object const& copy) = delete;
+   
+    virtual ~Object() {
+    }
+
+    std::string const& getType() const {
+      return type_;
+    }
+    
+    bool isForm() const {
+      return (typeid(*this) == typeid(Form));
+    }
+    
+    size_t getSize() const {
+      return length_;
+    }
+
+    char const* begin() const {
+      return base_;
+    }
+    
+    char const* end() const {
+      return base_+length_;
+    }
+
+    operator std::string() const;
+    
+  protected:
+    char const* base_;
+    const size_t length_;
+    std::string const type_;
+  };
+  
+  class Form final : public Object {
+  public:
+    Form(std::string type, char const* base, size_t length);
+    
+    ~Form() {}
+    
+    std::string const& getSubtype() const {
+      return subtype_;
+    }
+    
+    size_t getChildCount() const {
+      return children_.size();
+    }
+
     ObjectIterator childrenBegin() const {
       return ObjectIterator(children_.cbegin());
     }
@@ -168,7 +172,15 @@ public:
   };
 
   Object const& getRoot() const {
-    return *root_;
+    return *roots_[0];
+  }
+
+  ObjectIterator begin() const {
+    return ObjectIterator(roots_.cbegin());
+  }
+
+  ObjectIterator end() const {
+    return ObjectIterator(roots_.cend());
   }
   
   void printStructure(unsigned level = 0) const;
@@ -178,7 +190,7 @@ private:
   
   char const* base_;
   const size_t length_;
-  std::unique_ptr<Object> root_;
+  std::vector<std::unique_ptr<Object> > roots_;
 };
 
 #endif

Makefile

@@ -5,13 +5,13 @@ LDOPTS=-Wl,--sort-common,--as-needed
 iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc exceptions.cc
 iffexplore_LIBS ::=
 
-treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc
+treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc decompress.cc
 treexplore_LIBS ::=
 
 font2png_CXXSRCS ::= font2png.cc
 font2png_LIBS ::= -lpng
 
-mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc
+mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc decompress.cc
 mvedecode_LIBS ::= -lSDL2
 
 progs ::= iffexplore treexplore mvedecode

MveDecoder.cc

@@ -4,6 +4,7 @@
 #include "common.hh"
 #include "util.hh"
 #include "IffFile.hh"
+#include "decompress.hh"
 #include "MveDecoder.hh"
 
 MveDecoder::MveDecoder(char const* base, size_t length)
@@ -32,26 +33,31 @@ MveDecoder::MveDecoder(char const* base, size_t length)
   printf("_PC_: %u PALTs\n", PC_PALTcount);
 #endif
 
-  // Parse SOND chunk
+  // // Parse SOND chunk
-  auto& SOND = *it++;
+  // auto& SOND = *it++;
-  if ((SOND.getType() != "SOND") ||
+  // if ((SOND.getType() != "SOND") ||
-      (SOND.getSize() != 4))
+  //     (SOND.getSize() != 4))
-    throw FormatException{"SOND chunk missing or wrong size"};
+  //   throw FormatException{"SOND chunk missing or wrong size"};
 
-  const uint32_t  SONDc = readU32LE(SOND.begin());
+  // const uint32_t  SONDc = readU32LE(SOND.begin());
-  if (SONDc != 3)
+  // if (SONDc != 3)
-    throw FormatException{"Unexpected SOND value"};
+  //   throw FormatException{"Unexpected SOND value"};
 
   // Parse data
   unsigned curPALT = 0, curBRCH = 0;
   bool haveIndex = false;
-  std::vector<IffFile::Object const*> PALTs;
   // Store branch offsets, resolve to shots later
   std::vector<uint32_t> branches;
   Shot *curSHOT = nullptr;
   
   for (;it != rootForm.childrenEnd();++it) {
-    if (it->getType() == "PALT") {
+    if (it->getType() == "SIZE") {
+      if (it->getSize() != 8)
+	throw FormatException{"Unexpected SIZE size"};
+
+      width_ = readU32LE(it->begin());
+      height_ = readU32LE(it->begin()+4); 
+    } else if (it->getType() == "PALT") {
       if (curPALT > PC_PALTcount)
 	throw FormatException{"Number of PALT chunks exceeds amount specified in _PC_"};
       if (it->getSize() != 768)
@@ -99,6 +105,8 @@ MveDecoder::MveDecoder(char const* base, size_t length)
       curBRCH = (idxIt-branches.begin());
     } else if (it->getType() == "TEXT") {
       // Subtitle NYI
+    } else if (it->getType() == "SOND") {
+      // ignore
     } else
       throw FormatException{"Encountered unexpected chunk: " + it->getType()};
   }
@@ -113,26 +121,15 @@ MveDecoder::MveDecoder(char const* base, size_t length)
 	     i++, std::find(palts_.begin(), palts_.end(), shot.palt)-palts_.begin(),
 	     shot.VGAs.size(), shot.AUDIs.size());
 #endif
-      if (shot.VGAs.size() != shot.AUDIs.size())
+      if ((shot.AUDIs.size() != 0) &&
-	throw FormatException{"Video/audio block count mismatch"};
+	  (shot.VGAs.size() != shot.AUDIs.size()))
+      	throw FormatException{"Video/audio block count mismatch: V " +
+	    std::to_string(shot.VGAs.size()) + ", A " +
+	    std::to_string(shot.AUDIs.size())};
     }
   }
 }
 
-char *binify(uint8_t a) {
-  static char buf[9];
-  buf[8] = '\0';
-  for(unsigned i = 0;i < 8;++i) {
-    if(a&0x80)
-      buf[i] = '1';
-    else
-      buf[i] = '0';
-    a <<= 1;
-  }
-
-  return buf;
-}
-
 std::vector<char> parseHuff_(char const* data, size_t len)
 {
   uint8_t numHuffVals;
@@ -201,98 +198,14 @@ std::vector<char> parseHuff_(char const* data, size_t len)
 
 std::vector<uint8_t> parsePixels_(char const* data, size_t len)
 {
-  std::vector<uint8_t> ret;
-
   if (*data == 0x02) {
-    size_t pos = 1;
+    return decompressLZ(data+1, len-1);
-    while (pos < len) {
-      uint8_t b = *(data+pos++);
-      if (!((b&0xe0)==0xe0)) {
-	unsigned size = 0, replSize = 0;
-	unsigned replOfs = 0;
-	if (!(b&0x80)) {
-#ifdef PIXELDEBUG_
-	  printf("Code: Repl1 %.2hhx\n", b);
-#endif
-	  if (pos >= len)
-	    throw FormatException{"Pixel data overrun"};
-	
-	  uint8_t ofs = *(data+pos++);
-	  size = b&0x3;
-	  
-	  replSize = ((b&0x1c)>>2) + 3;
-	  replOfs =  ret.size()-(((b&0x60)<<3)+ofs+1)+size;
-	} else if (!(b&0x40)) {
-#ifdef PIXELDEBUG_
-	  printf("Code: Repl2 %.2hhx\n", b);
-#endif
-	  if (pos+1 >= len)
-	    throw FormatException{"Pixel data overrun"};
-	  
-	  uint8_t b1 = *(data+pos++);
-	  uint8_t b2 = *(data+pos++);
-
-	  size = (b1&0xc0)>>6;
-
-	  replSize = (b&0x3f)+4;
-	  replOfs = ret.size()-(((b1&0x3f)<<8)+b2+1)+size;
-	} else if (!(b&0x20)) {
-#ifdef PIXELDEBUG_
-	  printf("Code: Repl3 %.2hhx\n", b);
-#endif
-	  if (pos+2 >= len)
-	    throw FormatException{"Pixel data overrun"};
-	  
-	  uint8_t b1 = *(data+pos++);
-	  uint8_t b2 = *(data+pos++);
-	  uint8_t b3 = *(data+pos++);
-
-	  size = b&0x3;
-
-	  replSize = b3+5+((b&0xc)<<6);
-	  replOfs = ret.size()-(((b&0x10)<<12)+1+(b1<<8)+b2)+size;
-	}
-	if (pos+size >= len)
-	    throw FormatException{"Pixel data overrun"};
-	std::copy(data+pos, data+pos+size,
-		  std::back_inserter(ret));
-	pos += size;
-
-	if (replOfs >= ret.size())
-	  throw FormatException{"Replication offset exceeds buffer"};
-	for (unsigned i = 0;i < replSize;++i)
-	  ret.push_back(ret[replOfs+i]);
   } else {
-#ifdef PIXELDEBUG_
+    std::vector<uint8_t> ret;
-	printf("Code: Copy %.2hhx\n", b);
-#endif
-	unsigned size = (b&0x1f)*4+4;
-	if (size > 0x70) {
-	  if (pos+(b&0x3) > len)
-	    throw FormatException{"Pixel data overrun"};
-	  std::copy(data+pos, data+pos+(b&0x3),
-		    std::back_inserter(ret));
-	  pos += (b&0x3);
-#ifndef NDEBUG
-	  if (pos < len)
-	    printf("%lu unparsed bytes in pixel data\n", len-pos);
-#endif
-	  break;
-	}
-	
-	if (pos+size > len)
-	  throw FormatException{"Pixel data overrun"};
-	std::copy(data+pos, data+pos+size,
-		  std::back_inserter(ret));
-	pos += size;
-      }
-      
-    }
-  } else
     std::copy(data+1, data+len, std::back_inserter(ret));
-
     return ret;
   }
+}
 
 MveDecoder::Frame MveDecoder::parseVGA(IffFile::Object const& vga, Frame const* prev) const
 {
@@ -583,9 +496,13 @@ void MveDecoder::play(unsigned branch) const
   }
 
   AudioCBData audioCBData;
+  if (curShot->AUDIs.size() > 0) {
     audioCBData.cur = &*curShot;
-  audioCBData.next = nullptr;
     audioCBData.aCur = curShot->AUDIs.cbegin();
+  } else {
+    audioCBData.cur = nullptr;
+  }
+  audioCBData.next = nullptr;
   audioCBData.pos = 0;
   
   SDL_AudioSpec want;
@@ -666,7 +583,7 @@ void MveDecoder::play(unsigned branch) const
 
       SDL_Surface *vidSurfRGB = SDL_ConvertSurface(vidSurf, screen->format, 0);
       if (!vidSurfRGB) {
-	printf("Could not get video surface: %s\n", SDL_GetError());
+	printf("Could not convert video surface: %s\n", SDL_GetError());
 	SDL_FreeSurface(vidSurf);
 	SDL_CloseAudioDevice(audioDev);
 	SDL_FreePalette(palette);
@@ -709,6 +626,7 @@ void MveDecoder::play(unsigned branch) const
   
 	  if (SDL_SetPaletteColors(palette, colors.data(), 0, 256) != 0) {
 	    printf("Could not set palette: %s\n", SDL_GetError());
+	    SDL_CloseAudioDevice(audioDev);
 	    SDL_FreePalette(palette);
 	    SDL_DestroyWindow(window);
 	    SDL_Quit();

TreFile.cc

@@ -4,6 +4,7 @@
 
 #include "common.hh"
 #include "util.hh"
+#include "decompress.hh"
 #include "TreFile.hh"
 
 struct TreHeader { // little endian
@@ -293,21 +294,13 @@ size_t TreFile::findCRC_(uint32_t crc) const
 
 void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const
 {
-  uint32_t dataPtr, length;
+  auto obj = openIdx_(table3Idx);
-  uint8_t flags;
-  std::tie(dataPtr, length, flags) = table3_[table3Idx];
-  
-  if(flags&0x80)
-    throw Exception{"Compressed TRE objects NYI"};
-  
-  if ((dataPtr + length) > length_)
-    throw FormatException{"length exceeds file size"};
   
   FILEUPtr outFile{fopen(name.c_str(), "wb")};
   if (!outFile)
     throw POSIXException{errno, "Could not open " + name};
 
-  if (fwrite(base_+dataPtr, length, 1, outFile.get()) != 1)
+  if (fwrite(obj.data(), obj.size(), 1, outFile.get()) != 1)
       throw POSIXException{errno, "Could not write"};
 }
 
@@ -317,15 +310,19 @@ TreFile::Object TreFile::openIdx_(size_t table3Idx) const
   uint8_t flags;
   std::tie(dataPtr, length, flags) = table3_[table3Idx];
 
-  if(flags&0x80)
-    throw Exception{"Compressed TRE objects NYI"};
-
   if ((dataPtr + length) > length_)
     throw FormatException{"length exceeds file size"};
   
+  if (flags&0x80) {
+    if (flags&0x40) {
+      return Object(decompressLZ(base_+dataPtr, length));
+    } else
+      throw Exception{"Compression type 0 NYI"};
+  } else {
     return Object(base_+dataPtr,
 		  length);
   }
+}
 
 TreFile::Stat TreFile::statIdx_(size_t table3Idx) const
 {

TreFile.hh

@@ -6,6 +6,7 @@
 #include <cstdio>
 #include <vector>
 #include <tuple>
+#include <experimental/optional>
 
 class TreFile {
 public:
@@ -29,21 +30,27 @@ public:
     
     Object(Object const& copy) = delete;
     Object(Object && move)
-      : base_(move.base_), length_(move.length_) {
+      : data_(std::move(move.data_)), base_(move.base_), length_(move.length_) {
+      move.data_ = std::experimental::nullopt;
       move.base_ = nullptr;
       move.length_ = 0;
     }
 
     Object& operator=(Object const& copy) = delete;
     Object& operator=(Object && move) {
+      data_ = std::move(move.data_);
       base_ = move.base_;
       length_ = move.length_;
+      move.data_ = std::experimental::nullopt;
       move.base_ = nullptr;
       move.length_ = 0;
       return *this;
     }
     
     char const* data() const {
+      if (data_)
+	return (char const*)(data_->data());
+      else
 	return base_;
     }
     
@@ -52,20 +59,32 @@ public:
     }
 
     operator bool() const {
-      return base_;
+      return data_ || base_;
     }
     
   private:
+    // Object either owns data (when decompressed) ...
+    std::experimental::optional<std::vector<uint8_t> > data_;
+    // .. or points to area in underlying data (when not compressed)
     char const* base_;
     size_t length_;
 
     Object(char const* base, size_t length)
-      : base_(base), length_(length) {
+      : data_(), base_(base), length_(length) {
+    }
+
+    Object(std::vector<uint8_t> data)
+      : data_(std::move(data)), base_(nullptr), length_(data_->size()) {
     }
     
     friend class TreFile;
   };
 
+  /* Open an object of the TRE file by name or by CRC
+   * The returned object should be destroyed when no longer needed as it may
+   * hold a memory allocation for its data. But it must not be used after the
+   * TreFile object is destroyed.
+   */
   Object openName(std::string const& name) const;
   Object openCRC(uint32_t crc) const;
 

decompress.cc

@@ -0,0 +1,82 @@
+#include "common.hh"
+
+#include "decompress.hh"
+
+std::vector<uint8_t> decompressLZ(char const* data, size_t len)
+{
+  std::vector<uint8_t> ret;
+  
+  size_t pos = 0;
+  while (pos < len) {
+    uint8_t b = *(data+pos++);
+    if (!((b&0xe0)==0xe0)) {
+      unsigned size = 0, replSize = 0;
+      unsigned replOfs = 0;
+      if (!(b&0x80)) {
+	if (pos >= len)
+	  throw FormatException{"Compressed stream overrun"};
+	
+	uint8_t ofs = *(data+pos++);
+	size = b&0x3;
+	  
+	replSize = ((b&0x1c)>>2) + 3;
+	replOfs =  ((b&0x60)<<3)+ofs+1;
+      } else if (!(b&0x40)) {
+	if (pos+1 >= len)
+	  throw FormatException{"Compressed stream overrun"};
+	  
+	uint8_t b1 = *(data+pos++);
+	uint8_t b2 = *(data+pos++);
+
+	size = (b1&0xc0)>>6;
+
+	replSize = (b&0x3f)+4;
+	replOfs = ((b1&0x3f)<<8)+b2+1;
+      } else if (!(b&0x20)) {
+	if (pos+2 >= len)
+	  throw FormatException{"Compressed stream overrun"};
+	  
+	uint8_t b1 = *(data+pos++);
+	uint8_t b2 = *(data+pos++);
+	uint8_t b3 = *(data+pos++);
+
+	size = b&0x3;
+
+	replSize = b3+5+((b&0xc)<<6);
+	replOfs = ((b&0x10)<<12)+1+(b1<<8)+b2;
+      }
+      if (pos+size >= len)
+	throw FormatException{"Compressed stream overrun"};
+      std::copy(data+pos, data+pos+size,
+		std::back_inserter(ret));
+      pos += size;
+
+      if (replOfs > ret.size())
+	throw FormatException{"Replication offset exceeds buffer"};
+      unsigned start = ret.size()-replOfs;
+      for (unsigned i = 0;i < replSize;++i)
+	ret.push_back(ret[start+i]);
+    } else {
+      unsigned size = (b&0x1f)*4+4;
+      if (size > 0x70) {
+	if (pos+(b&0x3) > len)
+	  throw FormatException{"Compressed stream overrun"};
+	std::copy(data+pos, data+pos+(b&0x3),
+		  std::back_inserter(ret));
+	pos += (b&0x3);
+#ifndef NDEBUG
+	if (pos < len)
+	  printf("%lu unparsed bytes in compressed data\n", len-pos);
+#endif
+	break;
+      }
+	
+      if (pos+size > len)
+	throw FormatException{"Compressed stream overrun"};
+      std::copy(data+pos, data+pos+size,
+		std::back_inserter(ret));
+      pos += size;
+    }
+  }
+  return ret;
+}

decompress.hh

@@ -0,0 +1,14 @@
+#ifndef WC3RE_DECOMPRESS_HH__
+#define WC3RE_DECOMPRESS_HH__
+
+#include <cstdint>
+#include <vector>
+
+/* Decompression routines for various proprietary compression formats found in
+   WC3 */
+
+/* LZ77-like format consisting of codes specifing copies from the input stream
+   and/or replication of previously output data */
+std::vector<uint8_t> decompressLZ(char const* data, size_t len);
+
+#endif

iffexplore.cc

@@ -91,8 +91,11 @@ int main(int argc, char *argv[])
     IffFile iff{mmap.data(), mmap.size()};
       
     if (printStructure) {
-      unsigned blobCount = 0;
+      unsigned blobCount = 0, subdoc = 0;
-      iffDumper(iff.getRoot(), dumpBlobs, dumpPath, blobCount);
+      for (auto& root : iff) {
+	printf("Subdocument %u:\n", subdoc++);
+	iffDumper(root, dumpBlobs, dumpPath, blobCount);
+      }
     }
   } catch (POSIXException &ex) {
     fflush(stdout);

treexplore.cc

@@ -70,7 +70,7 @@ int main(int argc, char *argv[]) {
     if (dumpIff) {
       for(auto name : file.getNames()) {
 	auto s = file.statName(name);
-	if (s.flags&0x80)
+	if ((s.flags&0xc0) == 0x80)
 	  continue;
 	auto f = file.openName(name);
 	try {
@@ -84,7 +84,7 @@ int main(int argc, char *argv[]) {
       
       for(auto crc : file.getCRCs()) {
 	auto s = file.statCRC(crc);
-	if (s.flags&0x80)
+	if ((s.flags&0xc0) == 0x80)
 	  continue;
 	auto f = file.openCRC(crc);
 	try {