BTH/axyz
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

WIP, multithreaded loading based on exceptions, very slow probably.

Browse Source
This commit is contained in:
William Moberg
2016-01-13 10:58:31 +01:00
parent 49884d7e60
commit a58cc33ed0
5 changed files with 178 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/Engine/Core/ResourceManager.h
+71 -24
View File
@@ -40,6 +40,30 @@ private:
ResourceManager();
public:
//TODO: Check if this is ever used, and remove it if it isn't.
//Why would a resource load another resource async. in the ctor?
//Should be thrown in a Resource's constructor if it cannot complete because another resource is still loading.
//Eg. If a Model loads a Texture asyncronously in the constructor, and it is not done yet.
struct StillLoadingException : public std::exception
{
virtual const char* what() const throw()
{
return "Resource is still loading.";
}
};
//Should be thrown in a Resource's constructor if certain work needs to be handled by
//the main thread, and not a parallel worker thread. Eg. opengl commands.
struct WorkerCannotExecute : public std::exception
{
virtual const char* what() const throw()
{
return "Worker thread is unable to execute, main thread need to handle the code.";
}
};
static void AssertIsMainThread();
static bool IsMainThread();
/*static ResourceManager& Instance()
{
static ResourceManager s;
@@ -49,15 +73,6 @@ public:
template <typename T>
static void RegisterType(std::string typeName);
/** Preloads a resource and caches it for future use
@tparam T Resource type.
@param resourceName Fully qualified name of the resource to preload.
*/
template <typename T>
static void Preload(std::string resourceName);
static void Preload(std::string resourceType, std::string resourceName);
/** Checks if a resource is in cache
@param resourceType Resource type as string.
@@ -66,14 +81,25 @@ public:
// TODO: Templateify
static bool IsResourceLoaded(std::string resourceType, std::string resourceName);
/** Hot-loads a resource and caches it for future use
/** If the resource is not in cache, starts loading the resource and returns nullptr immediately.
If the resource has been loaded already, return a pointer to it.
@tparam T Resource type.
@param resourceName Fully qualified name of the resource to load.
*/
template <typename T>
static T* LoadAsync(std::string resourceName, Resource* parent = nullptr);
static Resource* LoadAsync(std::string resourceType, std::string resourceName, Resource* parent = nullptr);
/** Hot-loads a resource and caches it for future use.
Fairly safe to assume that return value is a valid pointer, will only return nullptr on error.
@tparam T Resource type.
@param resourceName Fully qualified name of the resource to load.
*/
template <typename T>
static T* Load(std::string resourceName, Resource* parent = nullptr);
static Resource* Load(std::string resourceType, std::string resourceName, Resource* parent = nullptr);
static Resource* Load(std::string resourceType, std::string resourceName, Resource* parent = nullptr);
/** Reloads an already loaded resource, keeping its resource ID intact.
@@ -87,19 +113,40 @@ public:
static void Update();
private:
//Represents pointer values to signify that a resource haven't failed, but is not fully loaded.
class SpecialResourcePointer
{
public:
SpecialResourcePointer()
: m_Val(new Resource())
{}
~SpecialResourcePointer()
{
delete m_Val;
}
//Make this class implicitly convertible to the Resource*.
operator Resource* const() const { return m_Val; }
private:
Resource* const m_Val;
};
//TODO: Check if this is ever used, and remove it if it isn't.
static SpecialResourcePointer m_StillLoading;
static SpecialResourcePointer m_LoadWithMainThread;
static std::unordered_map<std::string, std::string> m_CompilerTypenameToResourceType;
static std::unordered_map<std::string, std::function<Resource*(std::string)>> m_FactoryFunctions; // type -> factory function
static std::unordered_map<std::pair<std::string, std::string>, Resource*> m_ResourceCache; // (type, name) -> resource
static std::unordered_map<std::string, Resource*> m_ResourceFromName; // name -> resource
static std::unordered_map<Resource*, Resource*> m_ResourceParents; // resource -> parent resource
static std::unordered_map<std::pair<std::string, std::string>, boost::thread> m_LoadingThreads; // (type, name) -> loading thread
static boost::recursive_mutex m_Mutex;
// TODO: Getters for IDs
static unsigned int m_CurrentResourceTypeID;
static std::unordered_map<std::string, unsigned int> m_ResourceTypeIDs;
// Number of resources of a type. Doubles as local ID.
static std::unordered_map<unsigned int, unsigned int> m_ResourceCount;
// Flag to suppress hot-load warnings when a preloading resource chain loads another resource
static bool m_Preloading;
static FileWatcher m_FileWatcher;
static void fileWatcherCallback(std::string path, FileWatcher::FileEventFlags flags);
@@ -108,20 +155,20 @@ private:
static unsigned int GetNewResourceID(unsigned int typeID);
// Internal: Create a resource and cache it
static Resource* CreateResource(std::string resourceType, std::string resourceName, Resource* parent);
static Resource* createResource(std::string resourceType, std::string resourceName, Resource* parent);
};
template <typename T>
T* ResourceManager::Load(std::string resourceName, Resource* parent /* = nullptr */)
{
auto resourceTypename = typeid(T).name();
auto it = m_CompilerTypenameToResourceType.find(resourceTypename);
if (it == m_CompilerTypenameToResourceType.end()) {
LOG_ERROR("Failed to load resource \"%s\" of type \"%s\": type not registered", resourceName.c_str(), resourceTypename);
return nullptr;
}
auto resourceTypename = typeid(T).name();
auto it = m_CompilerTypenameToResourceType.find(resourceTypename);
if (it == m_CompilerTypenameToResourceType.end()) {
LOG_ERROR("Failed to load resource \"%s\" of type \"%s\": type not registered", resourceName.c_str(), resourceTypename);
return nullptr;
}
return static_cast<T*>(Load(it->second, resourceName, parent));
return static_cast<T*>(Load(it->second, resourceName, parent));
}
template <typename T>
@@ -132,16 +179,16 @@ void ResourceManager::RegisterType(std::string typeName)
}
template <typename T>
void ResourceManager::Preload(std::string resourceName)
T* ResourceManager::LoadAsync(std::string resourceName, Resource* parent /* = nullptr */)
{
auto resourceTypename = typeid(T).name();
auto it = m_CompilerTypenameToResourceType.find(resourceTypename);
if (it == m_CompilerTypenameToResourceType.end()) {
LOG_ERROR("Failed to load resource \"%s\" of type \"%s\": type not registered", resourceName.c_str(), resourceTypename);
return;
return nullptr;
}
Preload(it->second, resourceName);
return static_cast<T*>(LoadAsync(it->second, resourceName, parent));
}
#endif
resources/Schema/Entities/CollisionTestLevel.xml
+1 -1
View File
@@ -17,7 +17,7 @@
<Scale X="1" Y="1" Z="1"/>
</c:Transform>
<c:Model>
<Resource>Models/ScaleWidget.obj</Resource>
<Resource>Models/Core/UnitSphere.obj</Resource>
</c:Model>
</Components>
</Entity>
src/Engine/Core/ResourceManager.cpp
+103 -26
View File
@@ -1,4 +1,7 @@
#include "Core/ResourceManager.h"
#include "boost/thread/thread.hpp"
#include "boost/thread/mutex.hpp"
#include "boost/thread/lock_guard.hpp"
std::unordered_map<std::string, std::string> ResourceManager::m_CompilerTypenameToResourceType;
std::unordered_map<std::string, std::function<Resource*(std::string)>> ResourceManager::m_FactoryFunctions;
@@ -8,8 +11,11 @@ std::unordered_map<Resource*, Resource*> ResourceManager::m_ResourceParents;
unsigned int ResourceManager::m_CurrentResourceTypeID = 0;
std::unordered_map<std::string, unsigned int> ResourceManager::m_ResourceTypeIDs;
std::unordered_map<unsigned int, unsigned int> ResourceManager::m_ResourceCount;
bool ResourceManager::m_Preloading = false;
FileWatcher ResourceManager::m_FileWatcher;
std::unordered_map<std::pair<std::string, std::string>, boost::thread> ResourceManager::m_LoadingThreads;
boost::recursive_mutex ResourceManager::m_Mutex;
ResourceManager::SpecialResourcePointer ResourceManager::m_StillLoading;
ResourceManager::SpecialResourcePointer ResourceManager::m_LoadWithMainThread;
unsigned int ResourceManager::GetTypeID(std::string resourceType)
{
@@ -74,37 +80,90 @@ void ResourceManager::Update()
m_FileWatcher.Check();
}
void ResourceManager::Preload(std::string resourceType, std::string resourceName)
Resource* ResourceManager::LoadAsync(std::string resourceType, std::string resourceName, Resource* parent /*= nullptr*/)
{
if (IsResourceLoaded(resourceType, resourceName)) {
//LOG_WARNING("Attempted to preload resource \"%s\" multiple times!", resourceName.c_str());
return;
}
auto cacheKey = std::make_pair(resourceType, resourceName);
decltype(m_ResourceCache)::iterator it;
//If a thread has already been launched to load this resource.
auto tIt = m_LoadingThreads.find(cacheKey);
if (tIt != m_LoadingThreads.end()) {
//If the thread is still working.
if (tIt->second.joinable()) {
return nullptr;
}
//Else, the thread is done.
m_LoadingThreads.erase(tIt);
it = m_ResourceCache.find(cacheKey);
if (it != m_ResourceCache.end()) {
//If the thread is done, but it cannot complete the rest, main thread must complete construction.
if (it->second == m_LoadWithMainThread) {
AssertIsMainThread();
return createResource(resourceType, resourceName, parent);
}
return it->second;
} else {
//If cache is still empty at cacheKey after thread finishes, it failed.
return nullptr;
}
}
m_Preloading = true;
LOG_INFO("Preloading resource \"%s\"", resourceName.c_str());
CreateResource(resourceType, resourceName, nullptr);
m_Preloading = false;
//If resource has already been loaded and cached.
it = m_ResourceCache.find(cacheKey);
if (it != m_ResourceCache.end()) {
//if ConstructByMainThread, createResource from Main.
return it->second;
}
//Create a thread that loads the resource into cache.
m_LoadingThreads[cacheKey] = boost::thread(createResource, resourceType, resourceName, parent);
return nullptr;
}
Resource* ResourceManager::Load(std::string resourceType, std::string resourceName, Resource* parent /*= nullptr*/)
{
auto it = m_ResourceCache.find(std::make_pair(resourceType, resourceName));
if (it != m_ResourceCache.end()) {
return it->second;
}
Resource* resource;
auto cacheKey = std::make_pair(resourceType, resourceName);
decltype(m_ResourceCache)::iterator it;
//If a thread has already been launched to load this resource.
auto tIt = m_LoadingThreads.find(cacheKey);
if (tIt != m_LoadingThreads.end()) {
//Wait for the thread to finish loading.
tIt->second.join();
//Then delete the thread.
m_LoadingThreads.erase(tIt);
it = m_ResourceCache.find(cacheKey);
if (it != m_ResourceCache.end()) {
//If the thread is done, but it cannot complete the rest, main thread must complete construction.
if (it->second == m_LoadWithMainThread) {
AssertIsMainThread();
return createResource(resourceType, resourceName, parent);
}
return it->second;
} else {
//If cache is still empty at cacheKey after thread finishes, it failed.
return nullptr;
}
}
//If resource has already been loaded and cached.
it = m_ResourceCache.find(cacheKey);
if (it != m_ResourceCache.end()) {
return it->second;
}
if (m_Preloading) {
LOG_INFO("Preloading resource \"%s\"", resourceName.c_str());
} else {
LOG_WARNING("Hot-loading resource \"%s\"", resourceName.c_str());
}
return CreateResource(resourceType, resourceName, parent);
//If resource is not cached, load and return it.
resource = createResource(resourceType, resourceName, parent);
if (resource == m_LoadWithMainThread) {
//If we entered here then we know the caller is a worker thread.
//And we know the resource must be loaded from master thread.
throw(WorkerCannotExecute());
}
return resource;
}
Resource* ResourceManager::CreateResource(std::string resourceType, std::string resourceName, Resource* parent)
Resource* ResourceManager::createResource(std::string resourceType, std::string resourceName, Resource* parent)
{
//Lock the mutex immediately, and unlock it when leaving the function.
boost::lock_guard<decltype(m_Mutex)> guard(m_Mutex);
auto facIt = m_FactoryFunctions.find(resourceType);
if (facIt == m_FactoryFunctions.end()) {
LOG_ERROR("Failed to load resource \"%s\" of type \"%s\": type not registered", resourceName.c_str(), resourceType.c_str());
@@ -112,25 +171,43 @@ Resource* ResourceManager::CreateResource(std::string resourceType, std::string
}
// Call the factory function
Resource* resource;
Resource* resource = nullptr;
try {
resource = facIt->second(resourceName);
} catch (const WorkerCannotExecute& e) {
resource = m_LoadWithMainThread;
} catch (const std::exception& e) {
LOG_ERROR("Failed to load resource \"%s\" of type \"%s\": %s", resourceName.c_str(), resourceType.c_str(), e.what());
}
if (resource != nullptr && resource != m_LoadWithMainThread) {
// Store IDs
resource->TypeID = GetTypeID(resourceType);
resource->ResourceID = GetNewResourceID(resource->TypeID);
} catch (const std::exception& e) {
resource = nullptr;
LOG_ERROR("Failed to load resource \"%s\" of type \"%s\": %s", resourceName.c_str(), resourceType.c_str(), e.what());
}
// Cache
m_ResourceCache[std::make_pair(resourceType, resourceName)] = resource;
m_ResourceFromName[resourceName] = resource;
if (parent != nullptr) {
m_ResourceParents[resource] = parent;
}
if (!boost::filesystem::is_directory(resourceName)) {
LOG_DEBUG("Adding watch for %s", resourceName.c_str());
m_FileWatcher.AddWatch(resourceName, fileWatcherCallback);
}
return resource;
}
bool ResourceManager::IsMainThread()
{
static boost::thread::id MainThreadId = boost::this_thread::get_id();
return boost::this_thread::get_id() == MainThreadId;
}
void ResourceManager::AssertIsMainThread()
{
if (!IsMainThread()) {
throw WorkerCannotExecute();
}
}
src/Engine/Rendering/RenderQueueFactory.cpp
+2 -1
View File
@@ -84,7 +84,8 @@ void RenderQueueFactory::FillModels(World* world, RenderQueue* renderQueue)
continue;
}
glm::vec4 color = modelC["Color"];
Model* model = ResourceManager::Load<Model>(resource);
//Model* model = ResourceManager::Load<Model>(resource);
Model* model = ResourceManager::LoadAsync<Model>(resource);
if (model == nullptr) {
model = ResourceManager::Load<Model>("Models/Core/Error.obj");
}
src/Game/Game.cpp
+1
View File
@@ -4,6 +4,7 @@
Game::Game(int argc, char* argv[])
{
ResourceManager::AssertIsMainThread();
ResourceManager::RegisterType<ConfigFile>("ConfigFile");
ResourceManager::RegisterType<Model>("Model");
ResourceManager::RegisterType<Texture>("Texture");