Merge branch 'master' of ssh://ka.blankertz.org/home/matthias/git/wc3re

Merge compressed length changes in TreFile

IffFile.cc

@@ -10,15 +10,13 @@ struct ChunkHeader {
 };
 
 IffFile::IffFile(char const* base, size_t length)
-  : base_(base), length_(length), root_(nullptr)
+  : base_(base), length_(length)
 {
-  root_ = parseObject(base, length);
+  size_t pos = 0;
-  
+  while (pos < length) {
-#ifndef NDEBUG
+    roots_.push_back(parseObject(base+pos, length-pos));
-  if (length > (root_->getSize()+8)) {
+    pos += roots_.back()->getSize() + 8;
-    printf("%lu excess bytes parsing IFF\n", length-(root_->getSize()+8));
   }
-#endif 
 }
 
 IffFile::~IffFile()
@@ -47,7 +45,8 @@ static void _printStructure(IffFile::Object const& obj, unsigned level)
 
 void IffFile::printStructure(unsigned level) const
 {
-  _printStructure(*root_, level);
+  for (auto& root : *this)
+    _printStructure(root, level);
 }
 
 
@@ -64,7 +63,7 @@ std::unique_ptr<IffFile::Object> IffFile::parseObject(char const* base, size_t l
     if (!isprint(header.typeID[i]))
       throw FormatException{"Not an IFF chunk"};
   
-  if (header.length > length-7)
+  if (header.length > length-8)
     throw FormatException{"length < size in header"};
   
   if(memcmp(header.typeID, "FORM", 4) == 0)

IffFile.hh

@@ -13,55 +13,8 @@ public:
 
   ~IffFile();
 
-  class Object {
+  class Object;
-  public:
+  class Form;
-    Object(std::string type, char const* base, size_t length);
-    Object(Object const& copy) = delete;
-   
-    virtual ~Object() {
-    }
-
-    std::string const& getType() const {
-      return type_;
-    }
-    
-    bool isForm() const {
-      return (typeid(*this) == typeid(Form));
-    }
-    
-    size_t getSize() const {
-      return length_;
-    }
-
-    char const* begin() const {
-      return base_;
-    }
-    
-    char const* end() const {
-      return base_+length_;
-    }
-
-    operator std::string() const;
-    
-  protected:
-    char const* base_;
-    const size_t length_;
-    std::string const type_;
-  };
-  
-  class Form final : public Object {
-  public:
-    Form(std::string type, char const* base, size_t length);
-    
-    ~Form() {}
-    
-    std::string const& getSubtype() const {
-      return subtype_;
-    }
-    
-    size_t getChildCount() const {
-      return children_.size();
-    }
   
   class ObjectIterator : public std::iterator<std::random_access_iterator_tag, const Object> {
   public:
@@ -151,10 +104,61 @@ public:
     }
 
     friend class Form;
+    friend class IffFile;
       
     std::vector<std::unique_ptr<Object> >::const_iterator implIt_;
   };
   
+  class Object {
+  public:
+    Object(std::string type, char const* base, size_t length);
+    Object(Object const& copy) = delete;
+   
+    virtual ~Object() {
+    }
+
+    std::string const& getType() const {
+      return type_;
+    }
+    
+    bool isForm() const {
+      return (typeid(*this) == typeid(Form));
+    }
+    
+    size_t getSize() const {
+      return length_;
+    }
+
+    char const* begin() const {
+      return base_;
+    }
+    
+    char const* end() const {
+      return base_+length_;
+    }
+
+    operator std::string() const;
+    
+  protected:
+    char const* base_;
+    const size_t length_;
+    std::string const type_;
+  };
+  
+  class Form final : public Object {
+  public:
+    Form(std::string type, char const* base, size_t length);
+    
+    ~Form() {}
+    
+    std::string const& getSubtype() const {
+      return subtype_;
+    }
+    
+    size_t getChildCount() const {
+      return children_.size();
+    }
+
     ObjectIterator childrenBegin() const {
       return ObjectIterator(children_.cbegin());
     }
@@ -168,7 +172,15 @@ public:
   };
 
   Object const& getRoot() const {
-    return *root_;
+    return *roots_[0];
+  }
+
+  ObjectIterator begin() const {
+    return ObjectIterator(roots_.cbegin());
+  }
+
+  ObjectIterator end() const {
+    return ObjectIterator(roots_.cend());
   }
   
   void printStructure(unsigned level = 0) const;
@@ -178,7 +190,7 @@ private:
   
   char const* base_;
   const size_t length_;
-  std::unique_ptr<Object> root_;
+  std::vector<std::unique_ptr<Object> > roots_;
 };
 
 #endif

Makefile

@@ -1,17 +1,17 @@
 CXX=g++
-CXXOPTS=-Og -ggdb -Wall -Wextra -pedantic -std=c++14 -flto
+CXXOPTS=-Og -ggdb -fvar-tracking-assignments -Wall -Wextra -pedantic -std=c++14 -march=native -fstack-protector-strong --param=ssp-buffer-size=4 -flto
-LDOPTS=
+LDOPTS=-Wl,--sort-common,--as-needed
 
 iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc exceptions.cc
 iffexplore_LIBS ::=
 
-treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc
+treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc decompress.cc
 treexplore_LIBS ::=
 
 font2png_CXXSRCS ::= font2png.cc
 font2png_LIBS ::= -lpng
 
-mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc
+mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc decompress.cc
 mvedecode_LIBS ::= -lSDL2
 
 progs ::= iffexplore treexplore mvedecode

MveDecoder.cc

@@ -4,13 +4,9 @@
 #include "common.hh"
 #include "util.hh"
 #include "IffFile.hh"
+#include "decompress.hh"
 #include "MveDecoder.hh"
 
-struct PCChunk {
-  uint32_t unknown[2];
-  uint32_t PALTcount;
-} __attribute__((__packed__));
-
 MveDecoder::MveDecoder(char const* base, size_t length)
   : iff_(base, length), width_(320), height_(165)
 {
@@ -37,26 +33,31 @@ MveDecoder::MveDecoder(char const* base, size_t length)
   printf("_PC_: %u PALTs\n", PC_PALTcount);
 #endif
 
-  // Parse SOND chunk
+  // // Parse SOND chunk
-  auto& SOND = *it++;
+  // auto& SOND = *it++;
-  if ((SOND.getType() != "SOND") ||
+  // if ((SOND.getType() != "SOND") ||
-      (SOND.getSize() != 4))
+  //     (SOND.getSize() != 4))
-    throw FormatException{"SOND chunk missing or wrong size"};
+  //   throw FormatException{"SOND chunk missing or wrong size"};
 
-  const uint32_t  SONDc = readU32LE(SOND.begin());
+  // const uint32_t  SONDc = readU32LE(SOND.begin());
-  if (SONDc != 3)
+  // if (SONDc != 3)
-    throw FormatException{"Unexpected SOND value"};
+  //   throw FormatException{"Unexpected SOND value"};
 
   // Parse data
   unsigned curPALT = 0, curBRCH = 0;
   bool haveIndex = false;
-  std::vector<IffFile::Object const*> PALTs;
   // Store branch offsets, resolve to shots later
   std::vector<uint32_t> branches;
   Shot *curSHOT = nullptr;
   
   for (;it != rootForm.childrenEnd();++it) {
-    if (it->getType() == "PALT") {
+    if (it->getType() == "SIZE") {
+      if (it->getSize() != 8)
+	throw FormatException{"Unexpected SIZE size"};
+
+      width_ = readU32LE(it->begin());
+      height_ = readU32LE(it->begin()+4); 
+    } else if (it->getType() == "PALT") {
       if (curPALT > PC_PALTcount)
 	throw FormatException{"Number of PALT chunks exceeds amount specified in _PC_"};
       if (it->getSize() != 768)
@@ -89,24 +90,30 @@ MveDecoder::MveDecoder(char const* base, size_t length)
     } else if (it->getType() == "INDX") {
       if (haveIndex)
 	throw FormatException{"Multiple INDX"};
+      if (curSHOT)
+	throw FormatException{"INDX after SHOT"};
       haveIndex = true;
 
       for(unsigned i = 0;i < it->getSize()/4;++i)
 	branches.push_back(readU32LE(it->begin()+i*4));
       branches_.resize(branches.size());
     } else if (it->getType() == "BRCH") {
-      size_t ofs = it->begin()-base;
+      size_t ofs = it->begin()-base-8;
-      auto idxIt = std::find(branches.begin(), branches.end(), ofs-8);
+      auto idxIt = std::find(branches.begin(), branches.end(), ofs);
       if (idxIt == branches.end())
-	throw FormatException{"Could not resolve branch " + std::to_string(ofs-8)};
+	throw FormatException{"Could not resolve branch " + std::to_string(ofs)};
       curBRCH = (idxIt-branches.begin());
     } else if (it->getType() == "TEXT") {
       // Subtitle NYI
+    } else if (it->getType() == "SOND") {
+      // ignore
     } else
       throw FormatException{"Encountered unexpected chunk: " + it->getType()};
   }
 
+#ifdef VIDDEBUG_ 
   unsigned i = 0;
+#endif
   for (auto& brch : branches_) {
     for (auto& shot : brch) {
 #ifdef VIDDEBUG_
@@ -114,26 +121,15 @@ MveDecoder::MveDecoder(char const* base, size_t length)
 	     i++, std::find(palts_.begin(), palts_.end(), shot.palt)-palts_.begin(),
 	     shot.VGAs.size(), shot.AUDIs.size());
 #endif
-      if (shot.VGAs.size() != shot.AUDIs.size())
+      if ((shot.AUDIs.size() != 0) &&
-	throw FormatException{"Video/audio block count mismatch"};
+	  (shot.VGAs.size() != shot.AUDIs.size()))
+      	throw FormatException{"Video/audio block count mismatch: V " +
+	    std::to_string(shot.VGAs.size()) + ", A " +
+	    std::to_string(shot.AUDIs.size())};
     }
   }
 }
 
-char *binify(uint8_t a) {
-  static char buf[9];
-  buf[8] = '\0';
-  for(unsigned i = 0;i < 8;++i) {
-    if(a&0x80)
-      buf[i] = '1';
-    else
-      buf[i] = '0';
-    a <<= 1;
-  }
-
-  return buf;
-}
-
 std::vector<char> parseHuff_(char const* data, size_t len)
 {
   uint8_t numHuffVals;
@@ -173,9 +169,7 @@ std::vector<char> parseHuff_(char const* data, size_t len)
     } else {
       if (bytePos >= len)
 	throw FormatException{"Huffman stream overrun"};
-      memcpy(&byteBuf, data+bytePos, 1);
+      byteBuf = data[bytePos++];
-      //printf("Byte: %.2hhx (%s)\n", byteBuf, binify(byteBuf));
-      ++bytePos;
       bit = byteBuf&0x1;
       byteBuf >>= 1;
       byteValid = 7;
@@ -183,11 +177,9 @@ std::vector<char> parseHuff_(char const* data, size_t len)
 
     huffIdx = huffTree.at(huffIdx-(bit?1:23));
 
-    //printf("step: %d %.2x\n", bit, huffIdx);
 
     if (huffIdx < 22) {
       commands.push_back(huffIdx);
-      //printf("out: %.2hhx\n", commands.back());
       huffIdx = huffTree.size();
     }
   }
@@ -206,99 +198,13 @@ std::vector<char> parseHuff_(char const* data, size_t len)
 
 std::vector<uint8_t> parsePixels_(char const* data, size_t len)
 {
-  std::vector<uint8_t> ret;
-
   if (*data == 0x02) {
-    size_t pos = 1;
+    return decompressLZ(data+1, len-1);
-    while (pos < len) {
-      uint8_t b = *(data+pos++);
-      if (!((b&0xe0)==0xe0)) {
-	unsigned size = 0, replSize = 0;
-	unsigned replOfs = 0;
-	if (!(b&0x80)) {
-#ifdef PIXELDEBUG_
-	  printf("Code: Repl1 %.2hhx\n", b);
-#endif
-	  if (pos >= len)
-	    throw FormatException{"Pixel data overrun"};
-	
-	  uint8_t ofs = *(data+pos++);
-	  size = b&0x3;
-	  if (pos+size >= len)
-	    throw FormatException{"Pixel data overrun"};
-
-	  replSize = ((b&0x1c)>>2) + 3;
-	  replOfs =  ret.size()-(((b&0x60)<<3)+ofs+1)+size;
-	} else if (!(b&0x40)) {
-#ifdef PIXELDEBUG_
-	  printf("Code: Repl2 %.2hhx\n", b);
-#endif
-	  if (pos+1 >= len)
-	    throw FormatException{"Pixel data overrun"};
-	  
-	  uint8_t b1 = *(data+pos++);
-	  uint8_t b2 = *(data+pos++);
-
-	  size = (b1&0xc0)>>6;
-	  if (pos+size >= len)
-	    throw FormatException{"Pixel data overrun"};
-
-	  replSize = (b&0x3f)+4;
-	  replOfs = ret.size()-(((b1&0x3f)<<8)+b2+1)+size;
-	} else if (!(b&0x20)) {
-#ifdef PIXELDEBUG_
-	  printf("Code: Repl3 %.2hhx\n", b);
-#endif
-	  if (pos+2 >= len)
-	    throw FormatException{"Pixel data overrun"};
-	  
-	  uint8_t b1 = *(data+pos++);
-	  uint8_t b2 = *(data+pos++);
-	  uint8_t b3 = *(data+pos++);
-
-	  size = b&0x3;
-	  if (pos+size >= len)
-	    throw FormatException{"Pixel data overrun"};
-
-	  replSize = b3+5+((b&0xc)<<6);
-	  replOfs = ret.size()-(((b&0x10)<<12)+1+(b1<<8)+b2)+size;
-	}
-	std::copy(data+pos, data+pos+size,
-		  std::back_inserter(ret));
-	pos += size;
-	
-	for (unsigned i = 0;i < replSize;++i)
-	  ret.push_back(ret.at(replOfs+i));
   } else {
-#ifdef PIXELDEBUG_
+    std::vector<uint8_t> ret;
-	printf("Code: Copy %.2hhx\n", b);
-#endif
-	unsigned size = (b&0x1f)*4+4;
-	if (size > 0x70) {
-	  if (pos+(b&0x3) > len)
-	    throw FormatException{"Pixel data overrun"};
-	  std::copy(data+pos, data+pos+(b&0x3),
-		    std::back_inserter(ret));
-	  pos += (b&0x3);
-#ifndef NDEBUG
-	  if (pos < len)
-	    printf("%lu unparsed bytes in pixel data\n", len-pos);
-#endif
-	  break;
-	}
-	
-	if (pos+size > len)
-	  throw FormatException{"Pixel data overrun"};
-	std::copy(data+pos, data+pos+size,
-		  std::back_inserter(ret));
-	pos += size;
-      }
-      
-    }
-  } else
     std::copy(data+1, data+len, std::back_inserter(ret));
-
     return ret;
+  }
 }
 
 MveDecoder::Frame MveDecoder::parseVGA(IffFile::Object const& vga, Frame const* prev) const
@@ -541,7 +447,9 @@ void MveDecoder::play(unsigned branch) const
 			    SDL_WINDOWPOS_UNDEFINED,
 			    SDL_WINDOWPOS_UNDEFINED,
 			    width_*2,
-			    height_*2,
+			    // 1.2 scale in height to compensate for non-square pixels in height
+			    // in original MCGA/VGA 320x200 mode
+			    height_*2*1.2,
 			    0);
   
   if (!window) {
@@ -588,9 +496,13 @@ void MveDecoder::play(unsigned branch) const
   }
 
   AudioCBData audioCBData;
+  if (curShot->AUDIs.size() > 0) {
     audioCBData.cur = &*curShot;
-  audioCBData.next = nullptr;
     audioCBData.aCur = curShot->AUDIs.cbegin();
+  } else {
+    audioCBData.cur = nullptr;
+  }
+  audioCBData.next = nullptr;
   audioCBData.pos = 0;
   
   SDL_AudioSpec want;
@@ -612,9 +524,11 @@ void MveDecoder::play(unsigned branch) const
   
   auto vga = curShot->VGAs.begin();
   uint32_t last = 0;
+  const double frameTime = 1000/15.0;
+  double nextFT = frameTime;
   Frame frame = parseVGA(**vga++, nullptr);  
 
-  std::vector<decltype(last)> frameTimes;
+  decltype(last) minFT = std::numeric_limits<decltype(last)>::max(), maxFT = 0;
   
   bool close = false;
 
@@ -634,9 +548,19 @@ void MveDecoder::play(unsigned branch) const
     }
     
     auto now = SDL_GetTicks();
-    if (now-last >= 67) {
+    if (now-last >= nextFT) {
-      if (last)
+      if (last) {
-	frameTimes.push_back(now-last);
+        minFT = std::min(minFT, (now-last));
+	maxFT = std::max(maxFT, (now-last));
+	
+	if ((now-last)-nextFT <= frameTime)
+	  nextFT = frameTime-((now-last)-nextFT);
+	else {
+	  printf("NYI: Frame dropping\n");
+	  nextFT = frameTime;
+	}
+      }
+      
       SDL_Surface *vidSurf = SDL_CreateRGBSurfaceFrom((void*)frame.pixels.data(), width_, height_, 8, width_, 0, 0, 0, 0);
       if (!vidSurf) {
 	printf("Could not get video surface: %s\n", SDL_GetError());
@@ -659,7 +583,7 @@ void MveDecoder::play(unsigned branch) const
 
       SDL_Surface *vidSurfRGB = SDL_ConvertSurface(vidSurf, screen->format, 0);
       if (!vidSurfRGB) {
-	printf("Could not get video surface: %s\n", SDL_GetError());
+	printf("Could not convert video surface: %s\n", SDL_GetError());
 	SDL_FreeSurface(vidSurf);
 	SDL_CloseAudioDevice(audioDev);
 	SDL_FreePalette(palette);
@@ -702,6 +626,7 @@ void MveDecoder::play(unsigned branch) const
   
 	  if (SDL_SetPaletteColors(palette, colors.data(), 0, 256) != 0) {
 	    printf("Could not set palette: %s\n", SDL_GetError());
+	    SDL_CloseAudioDevice(audioDev);
 	    SDL_FreePalette(palette);
 	    SDL_DestroyWindow(window);
 	    SDL_Quit();
@@ -713,7 +638,8 @@ void MveDecoder::play(unsigned branch) const
 	}
       }
       frame = parseVGA(**vga++, &frame);  
-    }
+    } else if ((nextFT-(now-last)) >= 10)
+      SDL_Delay(nextFT-(now-last));
   }
   
   SDL_PauseAudioDevice(audioDev, 1);
@@ -723,7 +649,6 @@ void MveDecoder::play(unsigned branch) const
   SDL_Quit();
 
   printf("Frame times: [%u, %u]\n",
-	 *std::min_element(frameTimes.begin(), frameTimes.end()),
+	 minFT, maxFT);
-	 *std::max_element(frameTimes.begin(), frameTimes.end()));
 }
  

TreFile.cc

@@ -4,6 +4,7 @@
 
 #include "common.hh"
 #include "util.hh"
+#include "decompress.hh"
 #include "TreFile.hh"
 
 struct TreHeader { // little endian
@@ -305,21 +306,13 @@ size_t TreFile::findCRC_(uint32_t crc) const
 
 void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const
 {
-  uint32_t dataPtr, length, clength;
+  auto obj = openIdx_(table3Idx);
-  uint8_t flags;
-  std::tie(dataPtr, length, clength, flags) = table3_[table3Idx];
-  
-  // if(flags&0x80)
-  //   throw Exception{"Compressed TRE objects NYI"};
-  
-  if ((dataPtr + clength) > length_)
-    throw FormatException{"length exceeds file size"};
   
   FILEUPtr outFile{fopen(name.c_str(), "wb")};
   if (!outFile)
     throw POSIXException{errno, "Could not open " + name};
 
-  if (fwrite(base_+dataPtr, clength, 1, outFile.get()) != 1)
+  if (fwrite(obj.data(), obj.size(), 1, outFile.get()) != 1)
       throw POSIXException{errno, "Could not write"};
 }
 
@@ -329,14 +322,22 @@ TreFile::Object TreFile::openIdx_(size_t table3Idx) const
   uint8_t flags;
   std::tie(dataPtr, length, clength, flags) = table3_[table3Idx];
 
-  if(flags&0x80)
-    throw Exception{"Compressed TRE objects NYI"};
-
   if ((dataPtr + clength) > length_)
     throw FormatException{"length exceeds file size"};
   
-  return Object(base_+dataPtr,
+  if (flags&0x80) {
-		length);
+    if (flags&0x40) {
+      auto dec = decompressLZ(base_+dataPtr, clength);
+#ifndef NDEBUG
+      if (dec.size() != length)
+	printf("WARNING: Decompressed size != expected (%lu, %u)\n", dec.size(), length);
+#endif
+      return Object(std::move(dec));
+    } else
+      throw Exception{"Compression type 0 NYI"};
+  } else {
+    return Object(base_+dataPtr, length);
+  }
 }
 
 TreFile::Stat TreFile::statIdx_(size_t table3Idx) const

TreFile.hh

@@ -6,6 +6,7 @@
 #include <cstdio>
 #include <vector>
 #include <tuple>
+#include <experimental/optional>
 
 class TreFile {
 public:
@@ -29,21 +30,27 @@ public:
     
     Object(Object const& copy) = delete;
     Object(Object && move)
-      : base_(move.base_), length_(move.length_) {
+      : data_(std::move(move.data_)), base_(move.base_), length_(move.length_) {
+      move.data_ = std::experimental::nullopt;
       move.base_ = nullptr;
       move.length_ = 0;
     }
 
     Object& operator=(Object const& copy) = delete;
     Object& operator=(Object && move) {
+      data_ = std::move(move.data_);
       base_ = move.base_;
       length_ = move.length_;
+      move.data_ = std::experimental::nullopt;
       move.base_ = nullptr;
       move.length_ = 0;
       return *this;
     }
     
     char const* data() const {
+      if (data_)
+	return (char const*)(data_->data());
+      else
 	return base_;
     }
     
@@ -52,20 +59,32 @@ public:
     }
 
     operator bool() const {
-      return base_;
+      return data_ || base_;
     }
     
   private:
+    // Object either owns data (when decompressed) ...
+    std::experimental::optional<std::vector<uint8_t> > data_;
+    // .. or points to area in underlying data (when not compressed)
     char const* base_;
     size_t length_;
 
     Object(char const* base, size_t length)
-      : base_(base), length_(length) {
+      : data_(), base_(base), length_(length) {
+    }
+
+    Object(std::vector<uint8_t> data)
+      : data_(std::move(data)), base_(nullptr), length_(data_->size()) {
     }
     
     friend class TreFile;
   };
 
+  /* Open an object of the TRE file by name or by CRC
+   * The returned object should be destroyed when no longer needed as it may
+   * hold a memory allocation for its data. But it must not be used after the
+   * TreFile object is destroyed.
+   */
   Object openName(std::string const& name) const;
   Object openCRC(uint32_t crc) const;
 

decompress.cc

@@ -0,0 +1,82 @@
+#include "common.hh"
+
+#include "decompress.hh"
+
+std::vector<uint8_t> decompressLZ(char const* data, size_t len)
+{
+  std::vector<uint8_t> ret;
+  
+  size_t pos = 0;
+  while (pos < len) {
+    uint8_t b = *(data+pos++);
+    if (!((b&0xe0)==0xe0)) {
+      unsigned size = 0, replSize = 0;
+      unsigned replOfs = 0;
+      if (!(b&0x80)) {
+	if (pos >= len)
+	  throw FormatException{"Compressed stream overrun"};
+	
+	uint8_t ofs = *(data+pos++);
+	size = b&0x3;
+	  
+	replSize = ((b&0x1c)>>2) + 3;
+	replOfs =  ((b&0x60)<<3)+ofs+1;
+      } else if (!(b&0x40)) {
+	if (pos+1 >= len)
+	  throw FormatException{"Compressed stream overrun"};
+	  
+	uint8_t b1 = *(data+pos++);
+	uint8_t b2 = *(data+pos++);
+
+	size = (b1&0xc0)>>6;
+
+	replSize = (b&0x3f)+4;
+	replOfs = ((b1&0x3f)<<8)+b2+1;
+      } else if (!(b&0x20)) {
+	if (pos+2 >= len)
+	  throw FormatException{"Compressed stream overrun"};
+	  
+	uint8_t b1 = *(data+pos++);
+	uint8_t b2 = *(data+pos++);
+	uint8_t b3 = *(data+pos++);
+
+	size = b&0x3;
+
+	replSize = b3+5+((b&0xc)<<6);
+	replOfs = ((b&0x10)<<12)+1+(b1<<8)+b2;
+      }
+      if (pos+size >= len)
+	throw FormatException{"Compressed stream overrun"};
+      std::copy(data+pos, data+pos+size,
+		std::back_inserter(ret));
+      pos += size;
+
+      if (replOfs > ret.size())
+	throw FormatException{"Replication offset exceeds buffer"};
+      unsigned start = ret.size()-replOfs;
+      for (unsigned i = 0;i < replSize;++i)
+	ret.push_back(ret[start+i]);
+    } else {
+      unsigned size = (b&0x1f)*4+4;
+      if (size > 0x70) {
+	if (pos+(b&0x3) > len)
+	  throw FormatException{"Compressed stream overrun"};
+	std::copy(data+pos, data+pos+(b&0x3),
+		  std::back_inserter(ret));
+	pos += (b&0x3);
+#ifndef NDEBUG
+	if (pos < len)
+	  printf("%lu unparsed bytes in compressed data\n", len-pos);
+#endif
+	break;
+      }
+	
+      if (pos+size > len)
+	throw FormatException{"Compressed stream overrun"};
+      std::copy(data+pos, data+pos+size,
+		std::back_inserter(ret));
+      pos += size;
+    }
+  }
+  return ret;
+}

decompress.hh

@@ -0,0 +1,14 @@
+#ifndef WC3RE_DECOMPRESS_HH__
+#define WC3RE_DECOMPRESS_HH__
+
+#include <cstdint>
+#include <vector>
+
+/* Decompression routines for various proprietary compression formats found in
+   WC3 */
+
+/* LZ77-like format consisting of codes specifing copies from the input stream
+   and/or replication of previously output data */
+std::vector<uint8_t> decompressLZ(char const* data, size_t len);
+
+#endif

iffexplore.cc

@@ -91,8 +91,11 @@ int main(int argc, char *argv[])
     IffFile iff{mmap.data(), mmap.size()};
       
     if (printStructure) {
-      unsigned blobCount = 0;
+      unsigned blobCount = 0, subdoc = 0;
-      iffDumper(iff.getRoot(), dumpBlobs, dumpPath, blobCount);
+      for (auto& root : iff) {
+	printf("Subdocument %u:\n", subdoc++);
+	iffDumper(root, dumpBlobs, dumpPath, blobCount);
+      }
     }
   } catch (POSIXException &ex) {
     fflush(stdout);

treexplore.cc

@@ -70,7 +70,7 @@ int main(int argc, char *argv[]) {
     if (dumpIff) {
       for(auto name : file.getNames()) {
 	auto s = file.statName(name);
-	if (s.flags&0x80)
+	if ((s.flags&0xc0) == 0x80)
 	  continue;
 	auto f = file.openName(name);
 	try {
@@ -84,7 +84,7 @@ int main(int argc, char *argv[]) {
       
       for(auto crc : file.getCRCs()) {
 	auto s = file.statCRC(crc);
-	if (s.flags&0x80)
+	if ((s.flags&0xc0) == 0x80)
 	  continue;
 	auto f = file.openCRC(crc);
 	try {

util.cc

@@ -92,16 +92,8 @@ uint16_t readU16BE(char const* data)
   if(!data)
     throw std::logic_error("readU16BE() called with nullptr");
 
-  uint16_t ret;
+  return ((static_cast<uint16_t>(*data)&0xff)<<8) |
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    static_cast<uint8_t>(*(data+1));
-  ret = ((static_cast<uint16_t>(*data)&0xff)<<8) | static_cast<uint8_t>(*(data+1));
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  memcpy(&ret, data, 2);
-#else
-#error Unknown endianess
-#endif
-
-  return ret;
 }
   
 uint32_t readU24BE(char const* data)
@@ -109,18 +101,9 @@ uint32_t readU24BE(char const* data)
   if(!data)
     throw std::logic_error("readU24BE() called with nullptr");
 
-  uint32_t ret = 0;
+  return ((static_cast<uint32_t>(*data)&0xff)<<16) |
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-  ret = ((static_cast<uint32_t>(*data)&0xff)<<16) |
     ((static_cast<uint32_t>(*(data+1))&0xff)<<8) |
     static_cast<uint8_t>(*(data+2));
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  memcpy(static_cast<char*>(&ret)+1, data, 3);
-#else
-#error Unknown endianess
-#endif
-
-  return ret;
 }
 
 uint32_t readU32BE(char const* data)
@@ -128,19 +111,10 @@ uint32_t readU32BE(char const* data)
   if(!data)
     throw std::logic_error("readU32BE() called with nullptr");
 
-  uint32_t ret;
+  return ((static_cast<uint32_t>(*data)&0xff)<<24) |
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-  ret = ((static_cast<uint32_t>(*data)&0xff)<<24) |
     ((static_cast<uint32_t>(*(data+1))&0xff)<<16) |
     ((static_cast<uint32_t>(*(data+2))&0xff)<<8) |
     static_cast<uint8_t>(*(data+3));
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  memcpy(&ret, data, 4);
-#else
-#error Unknown endianess
-#endif
-
-  return ret;
 }
 
 uint16_t readU16LE(char const* data)
@@ -148,16 +122,8 @@ uint16_t readU16LE(char const* data)
   if(!data)
     throw std::logic_error("readU16LE() called with nullptr");
 
-  uint16_t ret;
+  return ((static_cast<uint16_t>(*(data+1))&0xff)<<8) |
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    static_cast<uint8_t>(*data);
-  memcpy(&ret, data, 2);
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  ret = ((static_cast<uint16_t>(*(data+1))&0xff)<<8) | static_cast<uint8_t>(*data);
-#else
-#error Unknown endianess
-#endif
-
-  return ret;
 }
 
 uint32_t readU24LE(char const* data)
@@ -165,18 +131,9 @@ uint32_t readU24LE(char const* data)
   if(!data)
     throw std::logic_error("readU24LE() called with nullptr");
 
-  uint32_t ret = 0;
+  return ((static_cast<uint32_t>(*(data+2))&0xff)<<16) |
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-  memcpy(&ret, data, 3);
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  ret = ((static_cast<uint32_t>(*(data+2))&0xff)<<16) |
     ((static_cast<uint32_t>(*(data+1))&0xff)<<8) |
     static_cast<uint8_t>(*data);
-#else
-#error Unknown endianess
-#endif
-
-  return ret;
 }
 
 uint32_t readU32LE(char const* data)
@@ -184,22 +141,13 @@ uint32_t readU32LE(char const* data)
   if(!data)
     throw std::logic_error("readU32LE() called with nullptr");
 
-  uint32_t ret;
+  return ((static_cast<uint32_t>(*(data+3))&0xff)<<24) |
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-  memcpy(&ret, data, 4);
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  ret = ((static_cast<uint32_t>(*(data+3))&0xff)<<24) |
     ((static_cast<uint32_t>(*(data+2))&0xff)<<16) |
     ((static_cast<uint32_t>(*(data+1))&0xff)<<8) |
     static_cast<uint8_t>(*data);
-#else
-#error Unknown endianess
-#endif
-
-  return ret;
 }
 
-int sextend(unsigned b, int msb)
+int sextend(unsigned b, unsigned msb)
 {
   if (msb >= sizeof(unsigned)*8)
     throw std::logic_error("sextend: msb out of range");

util.hh

@@ -40,6 +40,6 @@ uint32_t readU24LE(char const* data);
 uint32_t readU32LE(char const* data);
 
 // Sign-extend b starting at msb
-int sextend(unsigned b, int msb);
+int sextend(unsigned b, unsigned msb);
 
 #endif