Added ResourceManager; WIP: Decoder for SHAP chunks

IffFile.cc

@@ -9,14 +9,15 @@ struct ChunkHeader {
   uint32_t length;
 };
 
-IffFile::IffFile(Resource const& res)
+IffFile::IffFile(Resource::Handle res)
-  : res_(res)
+  : res_(std::move(res)), footprint_(sizeof(IffFile))
 {
   size_t pos = 0;
-  while (pos < res_.size()) {
+  while (pos < res_->size()) {
-    roots_.push_back(parseObject(res_.data()+pos, res_.size()-pos));
+    roots_.push_back(parseObject(*this, res_->data()+pos, res_->size()-pos, footprint_));
     pos += roots_.back()->size() + 8;
   }
+  footprint_ += sizeof(std::unique_ptr<Object>)*roots_.capacity();
 }
 
 IffFile::~IffFile()
@@ -49,8 +50,9 @@ void IffFile::printStructure(unsigned level) const
     _printStructure(root, level);
 }
 
-
+std::unique_ptr<IffFile::Object>
-std::unique_ptr<IffFile::Object> IffFile::parseObject(uint8_t const* base, size_t length)
+IffFile::parseObject(IffFile& parent, uint8_t const* base, size_t length,
+		     size_t& footprint)
 {    
   if (length < 8)
     throw FormatException{"length < header size"};
@@ -67,14 +69,64 @@ std::unique_ptr<IffFile::Object> IffFile::parseObject(uint8_t const* base, size_
     throw FormatException{"length < size in header"};
   
   if(memcmp(header.typeID, "FORM", 4) == 0)
-    return std::make_unique<Form>("FORM", base+8, static_cast<size_t>(header.length));
+    return std::unique_ptr<Object>(new Form(parent, "FORM", base+8,
+					    static_cast<size_t>(header.length),
+					    footprint));
   else
-    return std::make_unique<Object>(std::string(header.typeID, 4), base+8, static_cast<size_t>(header.length));
+    return std::unique_ptr<Object>(new Object(parent, std::string(header.typeID, 4),
+					      base+8, static_cast<size_t>(header.length),
+					      footprint));
 }
 
-IffFile::Object::Object(std::string type, uint8_t const* base, size_t length)
+namespace {
-  : base_(base), length_(length), type_(std::move(type))
+  IffFile::Object& _getObject(std::string const& spec,
+			      std::vector<std::unique_ptr<IffFile::Object> > const& container)
+  {
+    auto sepPos = spec.find(':');
+    auto name = spec.substr(0, sepPos);
+    unsigned matchCnt = 0;
+    if (sepPos != std::string::npos)
+      matchCnt = std::stoul(spec.substr(sepPos+1));
+    
+    for (auto& obj : container) {
+      if (obj->isForm()) {
+	auto form = dynamic_cast<IffFile::Form const*>(obj.get());
+	if (form->getSubtype() == name) {
+	  if (!matchCnt) {
+	    return *obj;
+	  } else {
+	    --matchCnt;
+	  }
+	}
+      } else if (obj->getType() == name) {
+	if (!matchCnt) {
+	  return *obj;
+	} else {
+	  --matchCnt;
+	} 
+      }
+    }
+
+    throw PathException{"Not found: " + spec};
+  }
+  
+}
+
+IffFile::Object& IffFile::getObject(std::string const& spec) const
 {
+  return _getObject(spec, roots_);
+}
+
+IffFile::Object& IffFile::Form::getObject(std::string const& spec) const
+{
+  return _getObject(spec, children_);
+}
+
+IffFile::Object::Object(IffFile& parent, std::string type, uint8_t const* base,
+			size_t length, size_t& footprint)
+  : parent_(parent), base_(base), length_(length), type_(std::move(type))
+{
+  footprint += sizeof(Object)+type_.capacity();
 }
 
 IffFile::Object::operator std::string() const
@@ -112,27 +164,33 @@ IffFile::Object::operator std::string() const
     throw FormatException{"BLOB not string"};
 }
 
-IffFile::Form::Form(std::string type, uint8_t const* base, size_t length)
+IffFile::Form::Form(IffFile& parent, std::string type, uint8_t const* base,
-  : Object(std::move(type), base, length)
+		    size_t length, size_t& footprint)
+  : Object(parent, std::move(type), base, length, footprint)
 {
   if (length < 4)
     throw FormatException{"length < subtype id length"};
 
+  unsigned strLen = 4;
   for (unsigned i = 0;i < 4;++i) {
-    if ((i > 0) || (base[i] == '\0'))
+    if ((i > 0) && (base_[i] == '\0')) {
+      strLen = i;
       break;
-    if (!isprint(base[i]))
+    }
+    if (!isprint(base_[i]))
       throw FormatException{"Subtype not printable"};
   }
-  subtype_ = std::string(reinterpret_cast<char const*>(base), 4);
+  subtype_ = std::string(reinterpret_cast<char const*>(base_), strLen);
 
   size_t pos = 4;
   while (pos+8 < length) {
-    children_.push_back(parseObject(base+pos, length-pos));
+    children_.push_back(parseObject(parent_, base_+pos, length-pos, footprint));
     pos += 8 + children_.back()->size();
     
     if (pos%2 != 0)
       ++pos;
   }
+
+  footprint += sizeof(Form) + sizeof(std::unique_ptr<Object>)*children_.capacity() + subtype_.size() - sizeof(Object);
 }
 

IffFile.hh

@@ -9,12 +9,14 @@
 
 #include "Resource.hh"
 
-class IffFile {
+class IffFile : public RefCounted {
 public:
-  IffFile(Resource const& res);
+  IffFile(Resource::Handle res);
-
+  
   ~IffFile();
 
+  using Handle = ResourceHandle<IffFile>;
+  
   class Object;
   class Form;
   
@@ -113,8 +115,7 @@ public:
   
   class Object : public Resource {
   public:
-    Object(std::string type, uint8_t const* base, size_t length);
+     Object(Object const& copy) = delete;
-    Object(Object const& copy) = delete;
    
     ~Object() override {
     }
@@ -143,6 +144,18 @@ public:
       return base_+length_;
     }
 
+    void incRef() override {
+      parent_.incRef();
+    }
+
+    void decRef() override {
+      parent_.decRef();
+    }
+
+    unsigned getRef() const override {
+      return parent_.getRef();
+    }
+    
     operator std::string() const;
 
     operator bool() const override {
@@ -150,16 +163,20 @@ public:
     }
     
   protected:
+    Object(IffFile& parent, std::string type, uint8_t const* base, size_t length, size_t& footprint);
+   
+    IffFile& parent_;
     uint8_t const* base_;
-    const size_t length_;
+    size_t const length_;
     std::string const type_;
+
+    friend class IffFile;
   };
   
   class Form final : public Object {
   public:
-    Form(std::string type, uint8_t const* base, size_t length);
+    ~Form() override {
-    
+    }
-    ~Form() {}
     
     std::string const& getSubtype() const {
       return subtype_;
@@ -176,9 +193,15 @@ public:
     ObjectIterator childrenEnd() const {
       return ObjectIterator(children_.cend());
     }
+
+    Object& getObject(std::string const& spec) const;
   private:
+    Form(IffFile& parent, std::string type, uint8_t const* base, size_t length, size_t& footprint);
+
     std::vector<std::unique_ptr<Object> > children_;
     std::string subtype_;
+
+    friend class IffFile;
   };
 
   Object const& getRoot() const {
@@ -194,12 +217,32 @@ public:
   }
   
   void printStructure(unsigned level = 0) const;
+
+  Object& getObject(std::string const& spec) const;
+  
+  size_t size() const {
+    return res_->size();
+  }
+
+  uint8_t const* data() const {
+    return res_->data();
+  }
+
+  operator bool() const {
+    return *res_;
+  }
+
+  size_t footprint() const {
+    return footprint_;
+  }
   
 private:
-  static std::unique_ptr<Object> parseObject(uint8_t const* base, size_t length);
+  static std::unique_ptr<Object> parseObject(IffFile& parent, uint8_t const* base,
+					     size_t length, size_t& footprint);
   
-  Resource const& res_;
+  Resource::Handle res_;
   std::vector<std::unique_ptr<Object> > roots_;
+  size_t footprint_;
 };
 
 #endif

Makefile

@@ -1,28 +1,31 @@
 CXX=g++
-CXXOPTS=-Og -ggdb -fvar-tracking-assignments -Wall -Wextra -Wno-unused-function -pedantic -std=c++14 -march=native -fstack-protector-strong --param=ssp-buffer-size=4 -flto -I.
+CXXOPTS=-Og -ggdb -fvar-tracking-assignments -Wall -Wextra -Wno-unused-function -pedantic -std=c++14 -march=native -fstack-protector-strong --param=ssp-buffer-size=4 -flto -I. -pthread
-LDOPTS=-Wl,--sort-common,--as-needed
+LDOPTS=-Wl,--sort-common
 
 render_CXXSRCS ::= render/Renderer.cc render/GlResource.cc render/ProgramProvider.cc render/renderutil.cc render/Overlay.cc render/VBOManager.cc render/Object.cc render/AlResource.cc render/AudioStream.cc
 render_LIBS ::= -lSDL2 -lSDL2_ttf -lglbinding -lopenal
 game_CXXSRCS ::= game/GSMvePlay.cc game/GSShowObject.cc
 
 
-iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc exceptions.cc
+iffexplore_CXXSRCS ::= iffexplore.cc IffFile.cc util.cc exceptions.cc ResourceProvider.cc TreFile.cc decompress.cc
 iffexplore_LIBS ::=
 
-treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc decompress.cc
+treexplore_CXXSRCS ::= treexplore.cc TreFile.cc IffFile.cc util.cc exceptions.cc decompress.cc ResourceProvider.cc
 treexplore_LIBS ::=
 
 font2png_CXXSRCS ::= font2png.cc
 font2png_LIBS ::= -lpng
 
-mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc decompress.cc $(render_CXXSRCS) $(game_CXXSRCS)
+mvedecode_CXXSRCS ::= mvedecode.cc TreFile.cc IffFile.cc util.cc MveDecoder.cc exceptions.cc decompress.cc ResourceProvider.cc $(render_CXXSRCS) $(game_CXXSRCS)
 mvedecode_LIBS ::= $(render_LIBS)
 
-objdecode_CXXSRCS ::= objdecode.cc TreFile.cc IffFile.cc util.cc ObjDecoder.cc exceptions.cc decompress.cc PaletteDecoder.cc $(render_CXXSRCS) $(game_CXXSRCS)
+objdecode_CXXSRCS ::= objdecode.cc TreFile.cc IffFile.cc util.cc ObjDecoder.cc exceptions.cc decompress.cc PaletteDecoder.cc ResourceProvider.cc $(render_CXXSRCS) $(game_CXXSRCS)
 objdecode_LIBS ::= $(render_LIBS)
 
-progs ::= iffexplore treexplore mvedecode objdecode
+shapdecode_CXXSRCS ::= shapdecode.cc exceptions.cc util.cc ShapDecoder.cc IffFile.cc ResourceProvider.cc TreFile.cc decompress.cc
+shapdecode_LIBS ::=
+
+progs ::= iffexplore treexplore mvedecode objdecode shapdecode
 
 all:	$(progs)
 

MveDecoder.cc

@@ -6,14 +6,19 @@
 #include "util.hh"
 #include "IffFile.hh"
 #include "decompress.hh"
+#include "ResourceProvider.hh"
 #include "MveDecoder.hh"
 
-MveDecoder::MveDecoder(Resource const& res)
+MveDecoder::MveDecoder(std::string const& path)
-  : iff_(res), width_(320), height_(165)
+  : MveDecoder(ResourceProvider::getInstance().getIff(path)) {
+}
+
+MveDecoder::MveDecoder(IffFile::Handle iff)
+  : iff_(std::move(iff)), width_(320), height_(165)
 {
   //iff_.printStructure();
 
-  auto& root = iff_.getRoot();
+  auto& root = iff_->getRoot();
   if (!root.isForm())
     throw FormatException{"Root node not FORM"};
 
@@ -99,7 +104,7 @@ MveDecoder::MveDecoder(Resource const& res)
 	branches.push_back(readU32LE(it->begin()+i*4));
       branches_.resize(branches.size());
     } else if (it->getType() == "BRCH") {
-      size_t ofs = it->begin()-res.data()-8;
+      size_t ofs = it->begin()-iff_->data()-8;
       auto idxIt = std::find(branches.begin(), branches.end(), ofs);
       if (idxIt == branches.end())
 	throw FormatException{"Could not resolve branch " + std::to_string(ofs)};

MveDecoder.hh

@@ -6,11 +6,10 @@
 
 #include "IffFile.hh"
 
-class Resource;
-
 class MveDecoder {
 public:
-  MveDecoder(Resource const& res);
+  MveDecoder(std::string const& path);
+  MveDecoder(IffFile::Handle iff);
 
   size_t numBranches() const {
     return branches_.size();
@@ -70,7 +69,7 @@ public:
   Movie open(size_t branch) const;
   
 private:
-  IffFile iff_;
+  IffFile::Handle iff_;
 
   unsigned width_, height_;
 

ObjDecoder.cc

@@ -3,11 +3,17 @@
 #include "ObjDecoder.hh"
 #include "exceptions.hh"
 #include "util.hh"
+#include "ResourceProvider.hh"
 
-ObjDecoder::ObjDecoder(Resource const& res)
+ObjDecoder::ObjDecoder(std::string const& path)
-  : iff_(res)
+  : ObjDecoder(ResourceProvider::getInstance().getIff(path))
 {
-  auto& root = iff_.getRoot();
+}
+
+ObjDecoder::ObjDecoder(IffFile::Handle iff)
+  : iff_(std::move(iff))
+{
+  auto& root = iff_->getRoot();
   if (!root.isForm())
     throw FormatException{"Root node not FORM"};
 

ObjDecoder.hh

@@ -8,11 +8,10 @@
 
 #include "IffFile.hh"
 
-class Resource;
-
 class ObjDecoder {
 public:
-  ObjDecoder(Resource const& res);
+  ObjDecoder(std::string const& path);
+  ObjDecoder(IffFile::Handle iff);
 
   using Vertex = std::array<int32_t, 3>;
   using Vertices = std::vector<Vertex>;
@@ -70,7 +69,7 @@ public:
     return texAnims_;
   }
 private:
-  IffFile iff_;
+  IffFile::Handle iff_;
   std::string name_;
 
   void parseOBJT_(IffFile::Form const& form);

PaletteDecoder.cc

@@ -2,11 +2,17 @@
 
 #include "PaletteDecoder.hh"
 #include "exceptions.hh"
+#include "ResourceProvider.hh"
 
-PaletteDecoder::PaletteDecoder(Resource const& res)
+PaletteDecoder::PaletteDecoder(std::string const& path)
-  : iff_(res)
+  : PaletteDecoder(ResourceProvider::getInstance().getIff(path))
 {
-  auto& root = iff_.getRoot();
+}
+
+PaletteDecoder::PaletteDecoder(IffFile::Handle iff)
+  : iff_(std::move(iff))
+{
+  auto& root = iff_->getRoot();
   if (!root.isForm())
     throw FormatException{"Root node not FORM"};
 

PaletteDecoder.hh

@@ -8,7 +8,8 @@
 
 class PaletteDecoder {
 public:
-  PaletteDecoder(Resource const& res);
+  PaletteDecoder(std::string const& path);
+  PaletteDecoder(IffFile::Handle iff);
 
   using Palette = std::array<uint8_t, 768>;
 
@@ -17,7 +18,7 @@ public:
   }
   
 private:
-  IffFile iff_;
+  IffFile::Handle iff_;
   Palette palette_;
 };
 

Resource.hh

@@ -1,18 +1,129 @@
 #ifndef WC3RE_RESOURCE_HH__
 #define WC3RE_RESOURCE_HH__
 
+#include <cstdint>
+#include <cstddef>
+#include <string>
+#include <memory>
+#include <atomic>
+#include <cassert>
+
+class RefCounted {
+public:
+  RefCounted() : ref_(0) {
+  }
+
+  virtual ~RefCounted() {
+    assert(ref_ == 0);
+  }
+  
+  virtual void incRef() {
+    ++ref_;
+  }
+  
+  virtual void decRef() {
+    auto old = ref_.fetch_sub(1);
+    assert(old > 0);
+  }
+
+  virtual unsigned getRef() const {
+    return ref_;
+  }
+
+protected:
+  std::atomic_uint ref_;
+};
+
+template<class Resource>
+class ResourceHandle {
+public:
+  ResourceHandle()
+    : res_(nullptr) {
+  }
+  
+  ResourceHandle(Resource& res)
+    : res_(&res) {
+    res_->incRef();
+  }
+
+  ResourceHandle(ResourceHandle const& copy)
+    : res_(copy.res_) {
+    if (res_)
+      res_->incRef();
+  }
+
+  ResourceHandle(ResourceHandle&& move)
+    : res_(move.res_) {
+    move.res_ = nullptr;
+  } 
+  
+  ~ResourceHandle() {
+    if(res_)
+      res_->decRef();
+  }
+
+  ResourceHandle& operator=(ResourceHandle const& copy) {
+    Resource* oldRes = res_;
+    res_ = copy.res_;
+    if (res_)
+      res_->incRef();
+    if (oldRes)
+      oldRes->decRef();
+
+    return *this;
+  }
+
+  ResourceHandle& operator=(ResourceHandle&& move) {
+    res_ = move.res_;
+    move.res_ = nullptr;
+
+    return *this;
+  }
+
+  operator bool() const {
+    return res_;
+  }
+
+  Resource& operator*() {
+    return *res_;
+  }
+
+  Resource const& operator*() const {
+    return *res_;
+  }
+
+  Resource* operator->() {
+    return res_;
+  }
+
+  Resource const* operator->() const {
+    return res_;
+  }
+  
+private:
+  Resource *res_;
+};
+  
 // Interface for any memory-resident resource
 // For example mmap'd data file, TRE object, IFF Object, ...
 
-class Resource {
+class Resource : public RefCounted {
-public:
+public:  
-  virtual ~Resource() {}
+  virtual ~Resource() {
+  }
 
   virtual uint8_t const* data() const = 0;
   virtual size_t size() const = 0;
 
   virtual operator bool() const = 0;
-};
+
+  // The memory footprint of this resource, that is, how much memory would be
+  // freed if this resource is discarded. Used for cache management.
+  virtual size_t footprint() const {
+    return size();
+  }
   
+  using Handle = ResourceHandle<Resource>;
+};
 
 #endif

ResourceProvider.cc

@@ -0,0 +1,257 @@
+#include <algorithm>
+
+#include "Resource.hh"
+#include "exceptions.hh"
+#include "util.hh"
+#include "IffFile.hh"
+//#include "TreFile.hh"
+#include "ResourceProvider.hh"
+
+template<>
+std::unique_ptr<ResourceProvider> Singleton<ResourceProvider>::instance{nullptr};
+
+template<>
+std::once_flag Singleton<ResourceProvider>::instance_flag{};
+
+using std::tie;
+using std::ignore;
+using std::make_unique;
+
+ResourceProvider::ResourceProvider()
+  : cacheSize_(0), cacheLimit_(64*1024*1024)
+{
+}
+
+ResourceProvider::~ResourceProvider()
+{
+
+}
+
+void ResourceProvider::dumpCache_() const
+{
+  for (auto& iff: iffs_) {
+    printf("IFF: %s: %u users, %zu size\n", iff.first.c_str(), iff.second->getRef(),
+	   iff.second->footprint());
+  }
+
+  for (auto& treObj: treObjs_) {
+    printf("TRE-Obj: %s: %u users, %zu size\n", treObj.first.c_str(), treObj.second->getRef(),
+	   treObj.second->footprint());
+  }
+
+  for (auto& tre: tres_) {
+    printf("TRE: %s: %u users, %zu size\n", tre.first.c_str(), tre.second->getRef(),
+	   tre.second->footprint());
+  }
+
+  for (auto& mmap: files_) {
+    printf("File: %s: %u users, %zu size\n", mmap.first.c_str(), mmap.second->getRef(),
+	   mmap.second->footprint());
+  }
+}
+
+void ResourceProvider::gc_()
+{
+  printf("ResourceProvider::gc_()\nBefore:\n");
+  dumpCache_();
+
+  if (cacheSize_ <= cacheLimit_) {
+    printf("No GC necessary\n");
+    return;
+  }
+  
+  {
+    auto it = iffs_.begin();
+    while (it != iffs_.end() && cacheSize_ > cacheLimit_) {
+      if (it->second->getRef() == 0) {
+	auto del = it++;
+	cacheSize_ -= del->second->footprint();
+	iffs_.erase(del);
+      } else
+	++it;
+    }
+  }
+
+  {
+    auto it = treObjs_.begin();
+    while (it != treObjs_.end() && cacheSize_ > cacheLimit_) {
+      if (it->second->getRef() == 0) {
+	auto del = it++;
+	cacheSize_ -= del->second->footprint();
+	treObjs_.erase(del);
+      } else
+	++it;
+    }
+  }
+
+  {
+    auto it = tres_.begin();
+    while (it != tres_.end() && cacheSize_ > cacheLimit_) {
+      if (it->second->getRef() == 0) {
+	auto del = it++;
+	cacheSize_ -= del->second->footprint();
+	tres_.erase(del);
+      } else
+	++it;
+    }
+  }
+
+  {
+    auto it = files_.begin();
+    while (it != files_.end() && cacheSize_ > cacheLimit_) {
+      if (it->second->getRef() == 0) {
+	auto del = it++;
+	cacheSize_ -= del->second->footprint();
+	files_.erase(del);
+      } else
+	++it;
+    }
+  }
+
+  printf("After:\n");
+  dumpCache_();
+}
+
+MmapFile& ResourceProvider::getMmap_(std::string const& ospath)
+{
+  auto it = files_.find(ospath);
+  if (it == files_.end()) {
+    tie(it, ignore) = files_.emplace(ospath, make_unique<MmapFile>(ospath));
+    cacheSize_ += it->second->footprint();
+  }
+  
+  return *it->second;
+}
+
+TreFile& ResourceProvider::getTre_(std::string const& ospath)
+{
+  auto it = tres_.find(ospath);
+  if (it == tres_.end()) {
+    auto& mmap = getMmap_(ospath);
+    tie(it, ignore) = tres_.emplace(ospath, make_unique<TreFile>(mmap));
+    cacheSize_ += it->second->footprint();
+  }
+  
+  return *it->second;
+}
+
+TreFile::Object& ResourceProvider::getTreObj_(std::string const& path)
+{
+  auto sepPos = path.find(':');
+  if (sepPos == std::string::npos)
+    throw PathException{"Incomplete TRE object path"};
+
+  auto& tre = getTre_(path.substr(0, sepPos));
+  auto normName = tre.normalizeName(path.substr(sepPos+1));
+
+  auto it = treObjs_.find(path.substr(0, sepPos) + ':' + normName);
+  if (it == treObjs_.end()) {
+    tie(it, ignore) = treObjs_.emplace(path.substr(0, sepPos) + ':' + normName,
+				       tre.getObject(normName));
+    cacheSize_ += it->second->footprint();
+  }
+
+  return *it->second;
+}
+
+IffFile& ResourceProvider::getIff_(std::string const& path)
+{
+  std::string normPath;
+  auto sepPos = path.find(':');
+  if (sepPos == std::string::npos) {
+    normPath = path;
+  } else {
+    auto& tre = getTre_(path.substr(0, sepPos));
+    normPath = path.substr(0, sepPos) + ':' + tre.normalizeName(path.substr(sepPos+1));
+  }
+  
+  auto it = iffs_.find(normPath);
+  if (it == iffs_.end()) {
+    if (sepPos == std::string::npos) {
+      auto& mmap = getMmap_(normPath);
+      tie(it, ignore) = iffs_.emplace(normPath, make_unique<IffFile>(mmap));
+    } else {
+      auto& treObj = getTreObj_(normPath);
+      tie(it, ignore) = iffs_.emplace(normPath, make_unique<IffFile>(treObj));
+    }
+    cacheSize_ += it->second->footprint();
+  }
+
+  return *it->second;
+}
+
+IffFile::Object& ResourceProvider::getIffObj_(IffFile& iff, std::string const& nodepath)
+{
+  size_t pos = 0, sepPos = nodepath.find('/');
+  auto obj = &iff.getObject(nodepath.substr(0, sepPos));
+  pos = sepPos;
+  while (pos != std::string::npos) {
+    sepPos = nodepath.find('/', pos+1);
+    auto form = dynamic_cast<IffFile::Form*>(obj);
+    if (!form)
+      throw PathException{"Non-form " + nodepath.substr(0, pos) +
+	  "with leftover path elements: " + nodepath.substr(pos+1)};
+    obj = &form->getObject(nodepath.substr(pos+1, sepPos-pos-1));
+
+    pos = sepPos;
+  }
+
+  return *obj;
+}
+
+Resource::Handle ResourceProvider::getResource(std::string const& path)
+{
+  std::string normPath;
+  auto sepPos = path.find(':');
+  Resource::Handle ret;
+  
+  if (sepPos == std::string::npos) {
+    // No path seperator, requested resource is a plain OS file
+    ret = getMmap_(path);
+  } else {
+    try {
+      // Assume root resource is TRE...
+      auto sepPos2 = path.find('/', sepPos);
+      if (sepPos2 == std::string::npos) {
+	// Only one path component, requested resource is TRE object
+	ret = getTreObj_(path);
+      } else {
+	// Requested resource is probably IFF node inside TRE
+	auto& iff = getIff_(path.substr(0, sepPos2));
+	ret = getIffObj_(iff, path.substr(sepPos2+1));
+      }
+    } catch (FormatException& ex) {
+    }
+
+    if (!ret) {
+      try {
+	// Hmm, maybe root resource is a naked IFF (no TRE)?
+	auto& iff = getIff_(path.substr(0, sepPos));
+	ret = getIffObj_(iff, path.substr(sepPos+1));
+      } catch (FormatException& ex) {
+      }
+    }
+
+    if (!ret) {
+      // No dice
+      throw PathException{"Could not load requested resource: " + path};
+    }
+  }
+
+  gc_();
+  return ret;
+}
+
+IffFile::Handle ResourceProvider::getIff(std::string const& path)
+{
+  IffFile::Handle ret(getIff_(path));
+  gc_();
+  return ret;
+}
+
+TreFile::Handle ResourceProvider::getTre(std::string const& path)
+{
+  TreFile::Handle ret(getTre_(path));
+  gc_();
+  return ret;
+}

ResourceProvider.hh

@@ -0,0 +1,44 @@
+#ifndef WC3RE_RESOURCEPROVIDER_HH__
+#define WC3RE_RESOURCEPROVIDER_HH__
+
+#include <unordered_map>
+#include <memory>
+
+#include "Singleton.hh"
+#include "TreFile.hh"
+#include "IffFile.hh"
+
+class Resource;
+class MmapFile;
+
+class ResourceProvider : public Singleton<ResourceProvider> {
+private:
+  ResourceProvider();
+  friend class Singleton<ResourceProvider>;
+  
+public:
+  ~ResourceProvider();
+  
+  Resource::Handle getResource(std::string const& path);
+
+  IffFile::Handle getIff(std::string const& path);
+  TreFile::Handle getTre(std::string const& path);
+ 
+private:
+  void gc_();
+  void dumpCache_() const;
+  
+  MmapFile& getMmap_(std::string const& ospath);
+  TreFile& getTre_(std::string const& ospath);
+  TreFile::Object& getTreObj_(std::string const& path);
+  IffFile& getIff_(std::string const& path);
+  IffFile::Object& getIffObj_(IffFile& iff, std::string const& nodepath);
+
+  size_t cacheSize_, cacheLimit_;
+  std::unordered_map<std::string, std::unique_ptr<MmapFile> > files_;
+  std::unordered_map<std::string, std::unique_ptr<TreFile> > tres_;
+  std::unordered_map<std::string, std::unique_ptr<TreFile::Object> > treObjs_;
+  std::unordered_map<std::string, std::unique_ptr<IffFile> > iffs_;
+};
+
+#endif

ShapDecoder.cc

@@ -0,0 +1,85 @@
+#include <cstring>
+#include <vector>
+#include <map>
+#include <algorithm>
+
+#include <exceptions.hh>
+
+#include "util.hh"
+#include "Resource.hh"
+#include "ResourceProvider.hh"
+#include "ShapDecoder.hh"
+
+ShapDecoder::ShapDecoder(std::string const& path)
+  : ShapDecoder(ResourceProvider::getInstance().getResource(path)) {
+}
+
+ShapDecoder::ShapDecoder(Resource::Handle res)
+  : res_(std::move(res))
+{
+  if (res_->size() < 8)
+    throw FormatException{"Object smaller than header size"};
+
+  if (memcmp(res_->data(), "1.11", 4) != 0)
+    throw FormatException{"Wrong magic, not a SHAP?"};
+
+  uint32_t numSubObjs = readU32LE(res_->data()+4);
+
+  if (res_->size() < 8+numSubObjs*8)
+    throw FormatException{"Object smaller than subobject table size"};
+
+  uint32_t lastofs = 0;
+  for (uint32_t i = 0;i < numSubObjs;++i) {
+    uint32_t ofs = readU32LE(res_->data()+8+i*8);
+    if (lastofs)
+      printf("\t%u bytes of data\n", ofs-(lastofs+24));
+    lastofs = ofs;
+    uint32_t flags = readU32LE(res_->data()+8+i*8+4);
+
+    printf("Subobject %u: Ofs %u (0x%x), flags %.8x\n",
+	   i, ofs, ofs, flags);
+
+    if (res_->size() < ofs+24)
+      throw FormatException{"Subobject header exceeds resource"};
+    
+    uint16_t unk1 = readU16LE(res_->data()+ofs), unk2 = readU16LE(res_->data()+ofs+2),
+      unk3 = readU16LE(res_->data()+ofs+4), unk4 = readU16LE(res_->data()+ofs+6);
+    int32_t unk5 = readU32LE(res_->data()+ofs+8), unk6 = readU32LE(res_->data()+ofs+12);
+    uint32_t unk7 = readU32LE(res_->data()+ofs+16), unk8 = readU32LE(res_->data()+ofs+20);
+
+    printf("\tunk: %hu, %hu, %hu, %hu, %d, %d, %u, %u\n",
+	   unk1, unk2, unk3, unk4, unk5, unk6, unk7, unk8);
+
+    std::vector<unsigned> hist(256, 0);
+    std::map<uint16_t, unsigned> hist2;
+    uint32_t end;
+    if (i < numSubObjs-1)
+      end = readU32LE(res_->data()+8+(i+1)*8);
+    else
+      end = res_->size();
+    uint16_t sreg = 0;
+    for (uint8_t const* it = res_->data()+ofs+24;it < res_->data()+end;++it) {
+      ++hist.at(*it);
+      sreg = (sreg<<8)|*it;
+      ++hist2[sreg];
+    }
+    
+    for (unsigned i = 0;i < 256;++i)
+      if (hist[i])
+	printf("\t%.2x: %u\n", i, hist[i]);
+
+    std::vector<std::pair<uint16_t, unsigned> > sortedHist2(20);
+    std::partial_sort_copy
+      (hist2.begin(), hist2.end(),
+       sortedHist2.begin(), sortedHist2.end(),
+       [](std::pair<uint16_t, unsigned> const& a, std::pair<uint16_t, unsigned> const& b) -> bool {
+	return a.second > b.second;
+      });
+    
+    for (auto& ent: sortedHist2)
+      if (ent.second > 1)
+	printf("\t%.4x: %u\n", ent.first, ent.second);
+    
+  }
+  printf("\t%zu bytes of data\n", res_->size()-(lastofs+24));
+}

ShapDecoder.hh

@@ -0,0 +1,18 @@
+#ifndef WC3RE_SHAPDECODER_HH__
+#define WC3RE_SHAPDECODER_HH__
+
+#include <cstddef>
+
+class Resource;
+
+class ShapDecoder {
+public:
+  ShapDecoder(Resource::Handle res);
+  ShapDecoder(std::string const& path);
+  
+private:
+  Resource::Handle res_;
+};
+
+#endif
+

TreFile.cc

@@ -1,6 +1,7 @@
 #include <cstdio>
 #include <cstdint>
 #include <cstring>
+#include <limits>
 
 #include "common.hh"
 #include "util.hh"
@@ -20,27 +21,27 @@ static const size_t headerSize = 24;
 static const size_t table1EntrySize = 8;
 static const size_t table3EntrySize = 8;
 
-TreFile::TreFile(uint8_t const* base, size_t length)
+TreFile::TreFile(Resource::Handle res)
-  : base_(base), length_(length)
+  : res_(std::move(res)), footprint_(sizeof(TreFile))
 {
-  if (length_ < 6*4)
+  if (res_->size() < 6*4)
     throw FormatException{"Size < header size"};
 
   TreHeader header;
-  memcpy(header.magic, base, 4);
+  memcpy(header.magic, res_->data(), 4);
 
   if (memcmp(header.magic, "XTRE", 4) != 0)
     throw FormatException{"Wrong magic, not a TRE?"};
 
-  header.table1Ofs = readU32LE(base_+8);
+  header.table1Ofs = readU32LE(res_->data()+8);
-  header.table2Ofs = readU32LE(base_+12);
+  header.table2Ofs = readU32LE(res_->data()+12);
-  header.table3Ofs = readU32LE(base_+16);
+  header.table3Ofs = readU32LE(res_->data()+16);
-  header.dataOfs = readU32LE(base_+20);
+  header.dataOfs = readU32LE(res_->data()+20);
   
-  if ((header.table1Ofs > length_) ||
+  if ((header.table1Ofs > res_->size()) ||
-      (header.table2Ofs > length_) ||
+      (header.table2Ofs > res_->size()) ||
-      (header.table3Ofs > length_) ||
+      (header.table3Ofs > res_->size()) ||
-      (header.dataOfs > length_))
+      (header.dataOfs > res_->size()))
     throw FormatException{"Table offset exceeds length"};
 
   if ((header.table1Ofs > header.table2Ofs) ||
@@ -61,22 +62,22 @@ TreFile::TreFile(uint8_t const* base, size_t length)
 	 header.table3Ofs, header.dataOfs-header.table3Ofs,
 	 numTable3Entries);
   printf("Data: start %u, length %lu\n",
-	 header.dataOfs, length_ - header.dataOfs);
+	 header.dataOfs, res_->size() - header.dataOfs);
 #endif
 
   // Read table 3
   for (size_t i = 0;i < numTable3Entries;++i) {
-    uint8_t const* const entryBase = base_+header.table3Ofs+i*table3EntrySize;
+    uint8_t const* const entryBase = res_->data()+header.table3Ofs+i*table3EntrySize;
     
     const uint32_t dataPtr = readU32LE(entryBase);
     const uint32_t length = readU32LE(entryBase+4)&0x0fffffffu;
     const uint8_t flags = *(entryBase+7)&0xf0;
     
-    if ((dataPtr > length_) ||
+    if ((dataPtr > res_->size()) ||
 	(dataPtr < header.dataOfs))
       throw FormatException{"Data pointer out of range"};
 
-    if ((dataPtr + length) > length_) {
+    if ((dataPtr + length) > res_->size()) {
       fprintf(stderr, "Data length exceeds file length: %.8x %.8x\n", dataPtr, length);
     }
 
@@ -100,11 +101,11 @@ TreFile::TreFile(uint8_t const* base, size_t length)
     uint32_t lastPtr, lastLen;
     uint8_t lastFlags;
     std::tie(lastPtr, std::ignore, lastLen, lastFlags) = table3_.back();
-    if (lastPtr+lastLen > length_) {
+    if (lastPtr+lastLen > res_->size()) {
       if (lastFlags&0x80)
-	std::get<2>(table3_.back()) = length_ - lastPtr;
+	std::get<2>(table3_.back()) = res_->size() - lastPtr;
       else
-	fprintf(stderr, "Overrun? %u %u (%hhu) -> %lu\n",lastPtr, lastLen, lastFlags, length_);
+	fprintf(stderr, "Overrun? %u %u (%hhu) -> %lu\n",lastPtr, lastLen, lastFlags, res_->size());
     }
   }
   
@@ -114,18 +115,18 @@ TreFile::TreFile(uint8_t const* base, size_t length)
     
   size_t pos = header.table2Ofs;
   while((pos+5) < header.table3Ofs) {
-    uint8_t nameLen = *(base+pos);
+    uint8_t nameLen = *(res_->data()+pos);
 
     if (pos+nameLen+5 > header.table3Ofs)
       throw FormatException{"Table 2 entry exceeds table " + std::to_string(nameLen)};
 
     for (unsigned i = 0;i < nameLen;++i)
-      if (!isprint(base[pos+1+i]))
+      if (!isprint(res_->data()[pos+1+i]))
 	throw FormatException{"Filename not printable"};
     
-    std::string nameStr(reinterpret_cast<char const*>(base)+pos+1, nameLen);
+    std::string nameStr(reinterpret_cast<char const*>(res_->data())+pos+1, nameLen);
 
-    const uint32_t table3Ptr = readU32LE(base+pos+nameLen+1);
+    const uint32_t table3Ptr = readU32LE(res_->data()+pos+nameLen+1);
     
     if ((table3Ptr < header.table3Ofs) ||
 	(table3Ptr >= header.dataOfs)) {
@@ -150,7 +151,7 @@ TreFile::TreFile(uint8_t const* base, size_t length)
 
   // Read Table 1
   for (size_t i = 0;i < numTable1Entries;++i) {
-    uint8_t const* const entryBase = base+header.table1Ofs+i*table1EntrySize;
+    uint8_t const* const entryBase = res_->data()+header.table1Ofs+i*table1EntrySize;
 
     const uint32_t crc = readU32LE(entryBase);
     uint32_t table3Ptr = readU32LE(entryBase+4);
@@ -185,6 +186,10 @@ TreFile::TreFile(uint8_t const* base, size_t length)
 	printf("Collision for CRC %.8x: prev %lu, new %lu\n", crc, ins.first->second, table3Index);
     }
   }
+
+  footprint_ += sizeof(std::map<uint32_t, size_t>::value_type)*table1_.size()+
+    sizeof(std::map<std::string, size_t>::value_type)*table2_.size()+
+    sizeof(std::tuple<uint32_t, uint32_t, uint32_t, uint8_t>)*table3_.capacity();
 }
 
 
@@ -212,16 +217,40 @@ std::vector<uint32_t> TreFile::getCRCs() const
   return ret;
 }
 
-TreFile::Object TreFile::openName(std::string const& name) const
+std::unique_ptr<TreFile::Object> TreFile::openName(std::string const& name) const
 {
   return openIdx_(findName_(name));
 }
 
-TreFile::Object TreFile::openCRC(uint32_t crc) const
+std::unique_ptr<TreFile::Object> TreFile::openCRC(uint32_t crc) const
 {
   return openIdx_(findCRC_(crc));
 }
-  
+
+std::unique_ptr<TreFile::Object> TreFile::getObject(std::string const& spec) const
+{
+  uint64_t crc;
+  try {
+    size_t pos;
+    crc = std::stoul(spec, &pos, 16);
+    if (pos < spec.size())
+      crc = std::numeric_limits<uint64_t>::max();
+  } catch (std::invalid_argument &ex) {
+    crc = std::numeric_limits<uint64_t>::max();
+  } catch (std::out_of_range &ex) {
+    crc = std::numeric_limits<uint64_t>::max();
+  }
+
+  if (crc <= std::numeric_limits<uint32_t>::max()) {
+    try {
+      return openCRC(crc);
+    } catch (Exception &ex) {
+    }
+  }
+
+  return openName(spec);
+}
+
 TreFile::Stat TreFile::statName(std::string const& name) const
 {
   return statIdx_(findName_(name));
@@ -325,7 +354,41 @@ uint32_t TreFile::calcCRC(std::string const& path)
 
   return sum;
 }
-  
+
+std::string TreFile::normalizeName(std::string const& name) const
+{
+  // Is name already a CRC?
+  if(name.size() == 8) {
+    uint64_t crc;
+    try {
+      size_t pos;
+      crc = std::stoul(name, &pos, 16);
+      if (pos < name.size())
+	crc = std::numeric_limits<uint64_t>::max();
+    } catch (std::invalid_argument &ex) {
+      crc = std::numeric_limits<uint64_t>::max();
+    } catch (std::out_of_range &ex) {
+      crc = std::numeric_limits<uint64_t>::max();
+    }
+
+    if (crc <= std::numeric_limits<uint32_t>::max()) {
+      // yes it is a CRC, keep name
+      return name;
+    }
+  }
+
+  // Is name a named file?
+  auto it = table2_.find(name);
+  if (it == table2_.end()) {
+    // no, reduce to CRC
+    char crcStr[9];
+    snprintf(crcStr, 9, "%.8X", calcCRC(name));
+    return crcStr;
+  } else
+    // yes it is a named file, keep name
+    return name;
+}
+
 size_t TreFile::findName_(std::string const& name) const
 {
   auto it = table2_.find(name);
@@ -360,37 +423,37 @@ void TreFile::dumpIdx_(std::string const& name, size_t table3Idx) const
   if (!outFile)
     throw POSIXException{errno, "Could not open " + name};
 
-  if (fwrite(obj.data(), obj.size(), 1, outFile.get()) != 1)
+  if (fwrite(obj->data(), obj->size(), 1, outFile.get()) != 1)
       throw POSIXException{errno, "Could not write"};
 }
 
-TreFile::Object TreFile::openIdx_(size_t table3Idx) const
+std::unique_ptr<TreFile::Object> TreFile::openIdx_(size_t table3Idx) const
 {
   uint32_t dataPtr, length, clength;
   uint8_t flags;
   std::tie(dataPtr, length, clength, flags) = table3_[table3Idx];
 
-  if ((dataPtr + clength) > length_)
+  if ((dataPtr + clength) > res_->size())
     throw FormatException{"length exceeds file size"};
   
   if (flags&0x80) {
     if (flags&0x40) {
-      auto dec = decompressLZ(base_+dataPtr, clength, length);
+      auto dec = decompressLZ(res_->data()+dataPtr, clength, length);
 #ifndef NDEBUG
       if (dec.size() != length)
 	printf("WARNING: Decompressed size != expected (%lu, %u)\n", dec.size(), length);
 #endif
-      return Object(std::move(dec));
+      return std::make_unique<Object>(std::move(dec));
     } else {
-      auto dec = decompressLZW(base_+dataPtr, clength, length);
+      auto dec = decompressLZW(res_->data()+dataPtr, clength, length);
 #ifndef NDEBUG
       if (dec.size() != length)
 	printf("WARNING: Decompressed size != expected (%lu, %u)\n", dec.size(), length);
 #endif
-      return Object(std::move(dec));
+      return std::make_unique<Object>(std::move(dec));
     }
   } else {
-    return Object(base_+dataPtr, clength);
+    return std::make_unique<Object>(res_->data()+dataPtr, clength, res_);
   }
 }
 

TreFile.hh

@@ -10,12 +10,14 @@
 
 #include "Resource.hh"
 
-class TreFile {
+class TreFile : public RefCounted {
 public:
-  TreFile(uint8_t const* base, size_t length);
+  TreFile(Resource::Handle base);
  
   ~TreFile();
 
+  using Handle = ResourceHandle<TreFile>;
+  
   std::vector<std::string> getNames() const;
   std::vector<uint32_t> getCRCs() const;
   
@@ -31,22 +33,35 @@ public:
   
   class Object : public Resource {
   public:
-    Object() : base_(nullptr), length_(0) {
+    Object() : data_(), base_(nullptr), length_(0), res_() {
+    }
+    
+    Object(Object const& copy)
+      : data_(copy.data_), base_(copy.base_), length_(copy.length_),
+	res_(copy.res_) {
     }
     
-    Object(Object const& copy) = delete;
     Object(Object && move)
-      : data_(std::move(move.data_)), base_(move.base_), length_(move.length_) {
+      : data_(std::move(move.data_)), base_(move.base_), length_(move.length_),
+	res_(std::move(move.res_)) {
       move.data_ = std::experimental::nullopt;
       move.base_ = nullptr;
       move.length_ = 0;
     }
 
-    Object& operator=(Object const& copy) = delete;
+    Object& operator=(Object const& copy) {
+      data_ = copy.data_;
+      base_ = copy.base_;
+      length_ = copy.length_;
+      res_ = copy.res_;
+      return *this;
+    }
+       
     Object& operator=(Object && move) {
       data_ = std::move(move.data_);
       base_ = move.base_;
       length_ = move.length_;
+      res_ = std::move(move.res_);
       move.data_ = std::experimental::nullopt;
       move.base_ = nullptr;
       move.length_ = 0;
@@ -55,7 +70,7 @@ public:
 
     ~Object() override {
     }
-    
+
     uint8_t const* data() const override {
       if (data_)
 	return data_->data();
@@ -69,34 +84,41 @@ public:
 
     operator bool() const override {
       return data_ || base_;
+    }	
+
+    size_t footprint() const override {
+      if (res_)
+	return sizeof(Object);
+      
+      return length_+sizeof(Object);
+    }
+    
+    Object(uint8_t const* base, size_t length, Resource::Handle res)
+      : data_(), base_(base), length_(length), res_(std::move(res)) {
+    }
+
+    Object(std::vector<uint8_t> data)
+      : data_(std::move(data)), base_(nullptr), length_(data_->size()), res_() {
     }
     
   private:
-    // Object either owns data (when decompressed) ...
+    // Object either owns data (when decompressed) (res_ is null in this case)...
     std::experimental::optional<std::vector<uint8_t> > data_;
     // .. or points to area in underlying data (when not compressed)
     uint8_t const* base_;
     size_t length_;
-
+    Resource::Handle res_;
-    Object(uint8_t const* base, size_t length)
-      : data_(), base_(base), length_(length) {
-    }
-
-    Object(std::vector<uint8_t> data)
-      : data_(std::move(data)), base_(nullptr), length_(data_->size()) {
-    }
-    
-    friend class TreFile;
   };
 
   /* Open an object of the TRE file by name or by CRC
-   * The returned object should be destroyed when no longer needed as it may
-   * hold a memory allocation for its data. But it must not be used after the
-   * TreFile object is destroyed.
    */
-  Object openName(std::string const& name) const;
+  std::unique_ptr<Object> openName(std::string const& name) const;
-  Object openCRC(uint32_t crc) const;
+  std::unique_ptr<Object> openCRC(uint32_t crc) const;
 
+  std::unique_ptr<Object> getObject(std::string const& spec) const;
+
+  std::string normalizeName(std::string const& name) const;
+  
   struct Stat {
     uint32_t size;
     uint32_t csize;
@@ -106,14 +128,16 @@ public:
   Stat statName(std::string const& name) const;
   Stat statCRC(uint32_t crc) const;
   
+  size_t footprint() const {
+    return footprint_;
+  }
+  
 private:
-  void construct_();
-
   size_t findName_(std::string const& name) const;
   size_t findCRC_(uint32_t crc) const;
   
   void dumpIdx_(std::string const& name, size_t table3Idx) const;
-  Object openIdx_(size_t table3Idx) const;
+  std::unique_ptr<Object> openIdx_(size_t table3Idx) const;
   Stat statIdx_(size_t table3Idx) const;
   
   std::map<uint32_t, size_t> table1_;
@@ -121,8 +145,8 @@ private:
   // base, size, comp. size, flags
   std::vector<std::tuple<uint32_t, uint32_t, uint32_t, uint8_t> > table3_;
 
-  uint8_t const* base_;
+  Resource::Handle res_;
-  size_t length_;
+  size_t footprint_;
 };
 
 #endif

decompress.cc

@@ -15,7 +15,7 @@ std::vector<uint8_t> decompressLZ(uint8_t const* data, size_t len, size_t retlen
   size_t pos = 0;
   while (pos < len) {
     uint8_t b = *(data+pos++);
-    if (!((b&0xe0)==0xe0)) {
+    if ((b&0xe0) != 0xe0) {
       unsigned size = 0, replSize = 0;
       unsigned replOfs = 0;
       if (!(b&0x80)) {

exceptions.cc

@@ -31,6 +31,17 @@ std::string FormatException::toString() const
 }
 
 
+PathException::PathException(std::string msg)
+  : Exception(std::move(msg))
+{
+}
+
+std::string PathException::toString() const
+{
+  return "PathException: " + msg_;
+}
+
+
 POSIXException::POSIXException(int err, std::string msg)
   : Exception(std::move(msg)), err_(err)
 {

exceptions.hh

@@ -22,6 +22,13 @@ public:
   std::string toString() const override;
 };
 
+class PathException : public Exception {
+public:
+  PathException(std::string msg);
+
+  std::string toString() const override;
+};
+
 class POSIXException : public Exception {
 public:
   POSIXException(int err, std::string msg = "");

iffexplore.cc

@@ -8,6 +8,7 @@
 
 #include "common.hh"
 #include "util.hh"
+#include "ResourceProvider.hh"
 #include "IffFile.hh"
 
 void blobDump(Resource const& obj, std::string const& filename)
@@ -86,13 +87,11 @@ int main(int argc, char *argv[])
   }
   
   try {
-    MmapFile mmap{iffFile};
+    auto iff = ResourceProvider::getInstance().getIff(iffFile);
-   
-    IffFile iff{mmap};
       
     if (printStructure) {
       unsigned blobCount = 0, subdoc = 0;
-      for (auto& root : iff) {
+      for (auto& root : *iff) {
 	printf("Subdocument %u:\n", subdoc++);
 	iffDumper(root, dumpBlobs, dumpPath, blobCount);
       }
@@ -101,7 +100,7 @@ int main(int argc, char *argv[])
     fflush(stdout);
     fprintf(stderr, "%s\n", ex.toString().c_str());
     return 2;
-  } catch (FormatException &ex) {
+  } catch (Exception &ex) {
     fflush(stdout);
     fprintf(stderr, "%s\n", ex.toString().c_str());
     return 3;

mvedecode.cc

@@ -9,8 +9,6 @@
 
 #include "common.hh"
 #include "util.hh"
-#include "TreFile.hh"
-#include "IffFile.hh"
 #include "MveDecoder.hh"
 #include "render/Renderer.hh"
 #include "game/GSMvePlay.hh"
@@ -25,9 +23,9 @@ void usage(char *argv0) {
 }
 
 int main(int argc, char *argv[]) {
-  std::string inFile, objectSpec;
+  std::string inPath;
   int branch = -1;
-  bool useTre = false, play = false;
+  bool play = false;
   {
     int opt;
     while ((opt = getopt(argc, argv, "hp::")) != -1) {
@@ -51,46 +49,17 @@ int main(int argc, char *argv[]) {
       }
     }
 
-    if (optind >= argc) {
+    if (optind != argc-1) {
       usage(argv[0]);
       return 1;
     }
 
-    inFile = argv[optind++];
+    inPath = argv[optind];
-
-    if (optind < argc) {
-      useTre = true;
-      objectSpec = argv[optind];
-    }
   }
 
   try {
-    MmapFile mmap{inFile};
+    auto mve = std::make_unique<MveDecoder>(inPath);
     
-    std::unique_ptr<TreFile> tre;
-    TreFile::Object object;
-    std::unique_ptr<MveDecoder> mve;
-    if (useTre) {
-      tre = std::make_unique<TreFile>(mmap.data(), mmap.size());
-
-      // Try to parse objectSpec as CRC
-      try {
-	unsigned long CRC = std::stoul(objectSpec, nullptr, 16);
-	if (CRC <= std::numeric_limits<uint32_t>::max()) {
-	  object = tre->openCRC(CRC);
-	}
-      } catch (std::invalid_argument &ex) {
-      } catch (std::out_of_range &ex) {
-      }
-
-      
-      if (!object) // Wasn't a CRC, try as name
-	object = tre->openName(objectSpec);
-	
-      mve = std::make_unique<MveDecoder>(object);
-    } else
-      mve = std::make_unique<MveDecoder>(mmap);
-
     if (play) {
       Renderer renderer;
 

objdecode.cc

@@ -17,15 +17,15 @@
 #include "game/GSShowObject.hh"
 
 void usage(char *argv0) {
-  fprintf(stderr, "Usage: %s [-h] [-ppalt] (tre-file name/crc)/iff-file\n", argv0);
+  fprintf(stderr, "Usage: %s [-h] [-ppalt] resource-path\n", argv0);
-  fprintf(stderr, "\tAttempt to decode the object stored in iff-file, or in the\n\tiff-object \"name\"/\"crc\" contained in tre-file\n");
+  fprintf(stderr, "\tAttempt to decode the object stored in the resource\n");
   fprintf(stderr, "\t-p palt\tUse palette \"palt\"");
   fprintf(stderr, "\t-h\tPrint this help\n");
 }
 
 int main(int argc, char *argv[]) {
-  std::string inFile, objectSpec, paltFile = "tmp/1DC61F50";
+  std::string inPath, paltFile = "tmp/1DC61F50";
-  bool useTre = false;
+  
   {
     int opt;
     while ((opt = getopt(argc, argv, "hp:")) != -1) {
@@ -42,48 +42,18 @@ int main(int argc, char *argv[]) {
       }
     }
 
-    if (optind >= argc) {
+    if (optind != argc-1) {
       usage(argv[0]);
       return 1;
     }
 
-    inFile = argv[optind++];
+    inPath = argv[optind];
-
-    if (optind < argc) {
-      useTre = true;
-      objectSpec = argv[optind];
-    }
   }
 
   try {
-    MmapFile paltMmap{paltFile};
+    PaletteDecoder palt{paltFile};
-    PaletteDecoder palt{paltMmap};
     
-    MmapFile mmap{inFile};
+    auto obj = std::make_unique<ObjDecoder>(inPath);
-    
-    std::unique_ptr<TreFile> tre;
-    TreFile::Object object;
-    std::unique_ptr<ObjDecoder> obj;
-    if (useTre) {
-      tre = std::make_unique<TreFile>(mmap.data(), mmap.size());
-
-      // Try to parse objectSpec as CRC
-      try {
-	unsigned long CRC = std::stoul(objectSpec, nullptr, 16);
-	if (CRC <= std::numeric_limits<uint32_t>::max()) {
-	  object = tre->openCRC(CRC);
-	}
-      } catch (std::invalid_argument &ex) {
-      } catch (std::out_of_range &ex) {
-      }
-
-      
-      if (!object) // Wasn't a CRC, try as name
-	object = tre->openName(objectSpec);
-	
-      obj = std::make_unique<ObjDecoder>(object);
-    } else
-      obj = std::make_unique<ObjDecoder>(mmap);
 
     render::Renderer renderer;
     renderer.pushGS(std::make_unique<game::GSShowObject>(renderer, *obj, palt));

render/Object.cc

@@ -70,7 +70,8 @@ namespace render {
 	
 	return src[y*srcWidth+x];
       };
-      
+
+      uint8_t min = 0xff, max = 0;
       for (unsigned y = 0;y < srcHeight;++y) {
 	for (unsigned x = 0;x < srcWidth;++x) {
 	  auto col = getPixel(x, y);
@@ -109,10 +110,15 @@ namespace render {
 	    *dst++ = palt[col*3u+1u]; // g
 	    *dst++ = palt[col*3u]; // r
 	    *dst++ = 255u; // a
+	    if (col > 0) {
+	      min = std::min(min, col);
+	      max = std::max(max, col);
+	    }
 	  }
 	}
 	dst += (dstStride - srcWidth*4);
-      }	    
+      }
+       printf("tex value range %hhu..%hhu\n", min, max);
     }
 
     /* Generate Mipmaps for BGRA texture 'src'.

shapdecode.cc

@@ -0,0 +1,56 @@
+#include <cstdio>
+#include <cstring>
+
+#include <unistd.h>
+
+#include "common.hh"
+#include "util.hh"
+#include "ShapDecoder.hh"
+#include "IffFile.hh"
+#include "ResourceProvider.hh"
+
+void usage(char *argv0) {
+  fprintf(stderr, "Usage: %s [-h] file\n", argv0);
+  fprintf(stderr, "\tAttempt to decode the object stored in file\n");
+  fprintf(stderr, "\t-h\tPrint this help\n");
+}
+
+int main(int argc, char *argv[]) {
+  std::vector<std::string> specs;
+  {
+    int opt;
+    while ((opt = getopt(argc, argv, "h")) != -1) {
+      switch (opt) {
+      case 'h':
+	usage(argv[0]);
+	return 0;
+      default:
+	usage(argv[0]);
+	return 1;
+      }
+    }
+
+    if (optind >= argc) {
+      usage(argv[0]);
+      return 1;
+    }
+
+    while (optind < argc)
+      specs.push_back(argv[optind++]);
+  }
+  
+  try {
+    for(auto& spec : specs)
+      auto shap = std::make_unique<ShapDecoder>(spec);
+  } catch (POSIXException &ex) {
+    fflush(stdout);
+    fprintf(stderr, "%s\n", ex.toString().c_str());
+    return 2;
+  } catch (Exception &ex) {
+    fflush(stdout);
+    fprintf(stderr, "%s\n", ex.toString().c_str());
+    return 3;
+  }
+  
+  return 0;
+}

treexplore.cc

@@ -11,6 +11,7 @@
 #include "util.hh"
 #include "TreFile.hh"
 #include "IffFile.hh"
+#include "ResourceProvider.hh"
 
 void usage(char *argv0) {
   fprintf(stderr, "Usage: %s [-sh] [-d dest] tre-file [filenames/crcs...]\n", argv0);
@@ -60,21 +61,16 @@ int main(int argc, char *argv[]) {
   }
 
   try {
-    MmapFile mmap{treFile}; 
+    auto tre = ResourceProvider::getInstance().getTre(treFile);
-    
-    TreFile file{mmap.data(), mmap.size()};
 
     if (printStructure)
-      file.printStructure();
+      tre->printStructure();
 
     if (dumpIff) {
-      for(auto name : file.getNames()) {
+      for(auto name : tre->getNames()) {
-	auto s = file.statName(name);
+	auto f = tre->openName(name);
-	if ((s.flags&0xc0) == 0x80)
-	  continue;
-	auto f = file.openName(name);
 	try {
-	  IffFile iff{f};
+	  IffFile iff{*f};
 	  printf("%s:\n", name.c_str());
 	  iff.printStructure(1);
 	} catch(FormatException &ex) {
@@ -82,10 +78,10 @@ int main(int argc, char *argv[]) {
 	}
       }
       
-      for(auto crc : file.getCRCs()) {
+      for(auto crc : tre->getCRCs()) {
-	auto f = file.openCRC(crc);
+	auto f = tre->openCRC(crc);
 	try {
-	  IffFile iff{f};
+	  IffFile iff{*f};
 	  printf("%.8x:\n", crc);
 	  iff.printStructure(1);
 	} catch(FormatException &ex) {
@@ -96,19 +92,19 @@ int main(int argc, char *argv[]) {
     
     if (dumpFiles) {
       if (fileSpecs.empty())
-	file.dumpAll(dumpPath);
+	tre->dumpAll(dumpPath);
       else {
 	for (auto spec : fileSpecs) {
 	  try {
 	    unsigned long fileCRC = std::stoul(spec, nullptr, 16);
 	    if (fileCRC <= std::numeric_limits<uint32_t>::max()) {
-	      file.dumpCRC(dumpPath, fileCRC);
+	      tre->dumpCRC(dumpPath, fileCRC);
 	      continue;
 	    }
 	  } catch (std::invalid_argument &ex) {
 	  } catch (std::out_of_range &ex) {
 	  }
-	  file.dumpName(dumpPath, spec);
+	  tre->dumpName(dumpPath, spec);
 	}
       }
     }

util.hh

@@ -23,6 +23,10 @@ public:
   uint8_t const* data() const override;
 
   operator bool() const override;
+
+  size_t footprint() const {
+    return sizeof(MmapFile)+name_.capacity()+sizeof(FILE);
+  }
 private:
   std::string name_;
   FILEUPtr fd_;