2146 lines
68 KiB
C++
2146 lines
68 KiB
C++
//===== Copyright 1996-2005, Valve Corporation, All rights reserved. ======//
|
|
//
|
|
// Queued Loading of map resources. !!!!Specifically!!! designed for the map loading process.
|
|
//
|
|
// Not designed for application startup or gameplay time. Layered on top of async i/o.
|
|
// Queued loading is allowed during the map load process until full connection only,
|
|
// but can complete the remaining low priority jobs during the game render.
|
|
// The normal loading path can run in units of seconds if it does not have to do I/O,
|
|
// which is why this system runs first and gets all the data in memory unhindered
|
|
// by dependency blocking. The I/O delivery process achieves its speed by having all the I/O
|
|
// requests at once, performing the I/O, and handing the actual consumption
|
|
// of the I/O buffer to another available core/thread (via job pool) for computation work.
|
|
// The I/O (should be all unbuffered) is then only throttled by physical transfer rates.
|
|
//
|
|
// The Load process is broken into three phases. The first phase build up I/O requests.
|
|
// The second phase fulfills only the high priority I/O requests. This gets the critical
|
|
// data in memory, that has to be there for the normal load path to query, or the renderer
|
|
// to run (i.e. models and shaders). The third phase is the normal load process.
|
|
// The low priority jobs run concurrently with the normal load process. Low priority jobs
|
|
// are those that have been specially built such that the game or loading can operate unblocked
|
|
// without the actual data (i.e. d3d texture bits).
|
|
//
|
|
// Phase 1: The reslist is parsed into seperate lists based on handled extensions. Each list
|
|
// call its own loader which in turn generates its own dictionaries and I/O requests through
|
|
// "AddJob". A single reslist entry could cause a laoder to request multiple jobs. ( i.e. models )
|
|
// A loader marks its jobs as high or low priority.
|
|
// Phase 2: The I/O requests are sorted (which achieves seek offset order) and
|
|
// async i/o commences. Phase 2 does not end until all the high priority jobs
|
|
// are complete. This ensures critical data is resident.
|
|
// Phase 3: The !!!NORMAL!!! loading path can commence. The legacy loading path then
|
|
// is not expected to do I/O (it can, but that's a hole in the reslist), as all of the data
|
|
// that it queries, should be resident.
|
|
//
|
|
// Late added jobs are non-optimal (should have been in reslist), warned, but handled.
|
|
//
|
|
//===========================================================================//
|
|
|
|
#include "basefilesystem.h"
|
|
|
|
#include "tier0/vprof.h"
|
|
#include "tier0/tslist.h"
|
|
#include "tier1/utlbuffer.h"
|
|
#include "tier1/convar.h"
|
|
#include "tier1/keyvalues.h"
|
|
#include "tier1/utllinkedlist.h"
|
|
#include "tier1/utlstring.h"
|
|
#include "tier1/utlsortvector.h"
|
|
#include "tier1/utldict.h"
|
|
#include "basefilesystem.h"
|
|
#include "tier0/icommandline.h"
|
|
#include "vstdlib/jobthread.h"
|
|
#include "filesystem/IQueuedLoader.h"
|
|
#include "tier2/tier2.h"
|
|
#include "characterset.h"
|
|
#include "tier1/lzmaDecoder.h"
|
|
#if !defined( _X360 )
|
|
#include "xbox/xboxstubs.h"
|
|
#endif
|
|
#ifdef _PS3
|
|
#include "tls_ps3.h"
|
|
#endif
|
|
|
|
// memdbgon must be the last include file in a .cpp file!!!
|
|
#include "tier0/memdbgon.h"
|
|
|
|
#define PRIORITY_HIGH 1
|
|
#define PRIORITY_NORMAL 0
|
|
#define PRIORITY_LOW -1
|
|
|
|
// main thread has reason to block and wait for thread pool to finish jobs
|
|
#define MAIN_THREAD_YIELD_TIME 20
|
|
|
|
// discrete stages in the preload process to tick the progress bar
|
|
#define PROGRESS_START 0.10f
|
|
#define PROGRESS_GOTRESLIST 0.12f
|
|
#define PROGRESS_PARSEDRESLIST 0.15f
|
|
#define PROGRESS_CREATEDRESOURCES 0.20f
|
|
#define PROGRESS_PREPURGE 0.22f
|
|
#define PROGRESS_FILESORT 0.23f
|
|
#define PROGRESS_IO 0.25f
|
|
#define PROGRESS_END 0.999f // up to 1.0
|
|
|
|
struct FileJob_t
|
|
{
|
|
FileJob_t()
|
|
{
|
|
Q_memset( this, 0, sizeof( FileJob_t ) );
|
|
}
|
|
|
|
FileNameHandle_t m_hFilename;
|
|
QueuedLoaderCallback_t m_pCallback;
|
|
FSAsyncControl_t m_hAsyncControl;
|
|
void *m_pContext;
|
|
void *m_pContext2;
|
|
void *m_pTargetData;
|
|
int m_nBytesToRead;
|
|
unsigned int m_nStartOffset;
|
|
LoaderPriority_t m_Priority;
|
|
|
|
unsigned int m_SubmitTime;
|
|
unsigned int m_FinishTime;
|
|
int m_SubmitTag;
|
|
int m_nActualBytesRead;
|
|
LoaderError_t m_LoaderError;
|
|
unsigned int m_ThreadId;
|
|
|
|
unsigned int m_bFinished : 1;
|
|
unsigned int m_bFreeTargetAfterIO : 1;
|
|
unsigned int m_bFileExists : 1;
|
|
unsigned int m_bClaimed : 1;
|
|
unsigned int m_bLateQueued : 1;
|
|
unsigned int m_bAnonymousDecode : 1;
|
|
};
|
|
|
|
// dummy stubbed progress interface
|
|
class CDummyProgress : public ILoaderProgress
|
|
{
|
|
void BeginProgress() {}
|
|
void UpdateProgress( float progress, bool bForce = false ) {}
|
|
void PauseNonInteractiveProgress( bool bPause ) {}
|
|
void EndProgress() {}
|
|
};
|
|
static CDummyProgress s_DummyProgress;
|
|
|
|
class CQueuedLoader : public CTier2AppSystem< IQueuedLoader >
|
|
{
|
|
typedef CTier2AppSystem< IQueuedLoader > BaseClass;
|
|
|
|
public:
|
|
CQueuedLoader();
|
|
virtual ~CQueuedLoader();
|
|
|
|
// Inherited from IAppSystem
|
|
virtual InitReturnVal_t Init();
|
|
virtual void Shutdown();
|
|
|
|
// IQueuedLoader
|
|
virtual void InstallLoader( ResourcePreload_t type, IResourcePreload *pLoader );
|
|
virtual void InstallProgress( ILoaderProgress *pProgress );
|
|
// Set bOptimizeReload if you want appropriate data (such as static prop lighting)
|
|
// to persist - rather than being purged and reloaded - when going from map A to map A.
|
|
virtual bool BeginMapLoading( const char *pMapName, bool bLoadForHDR, bool bOptimizeMapReload, void (*pfnBeginMapLoadingCallback)( int nStage ) );
|
|
virtual void EndMapLoading( bool bAbort );
|
|
virtual bool AddJob( const LoaderJob_t *pLoaderJob );
|
|
virtual void AddMapResource( const char *pFilename );
|
|
virtual bool ClaimAnonymousJob( const char *pFilename, QueuedLoaderCallback_t pCallback, void *pContext, void *pContext2 );
|
|
virtual bool ClaimAnonymousJob( const char *pFilename, void **pData, int *pDataSize, LoaderError_t *pError );
|
|
virtual bool IsMapLoading() const;
|
|
virtual bool IsSameMapLoading() const;
|
|
virtual bool IsFinished() const;
|
|
virtual bool IsBatching() const;
|
|
virtual int GetSpewDetail() const;
|
|
|
|
char *GetFilename( const FileNameHandle_t hFilename, char *pBuff, int nBuffSize );
|
|
FileNameHandle_t FindFilename( const char *pFilename );
|
|
void SpewInfo();
|
|
unsigned int GetStartTime();
|
|
|
|
// submit any queued jobs to the async loader, called by main or async thread to get more work
|
|
void SubmitPendingJobs();
|
|
|
|
void PurgeAll( ResourcePreload_t *pDontPurgeList = NULL, int nPurgeListSize = 0 );
|
|
#ifdef _PS3
|
|
// hack to prevent PS/3 deadlock on queued loader render mutex when quitting during loading a map
|
|
// PLEASE REMOVE THIS (AND THE MUTEX) AFTER WE SHIP
|
|
virtual uint UnlockProgressBarMutex()
|
|
{
|
|
uint nCount = m_nRendererMutexProgressBarUnlockCount;
|
|
for( uint i = 0; i < nCount ; ++i )
|
|
{
|
|
m_sRendererMutex.Unlock();
|
|
}
|
|
return nCount;
|
|
}
|
|
virtual void LockProgressBarMutex( uint nLockCount )
|
|
{
|
|
for( uint i = 0; i < nLockCount ; ++i )
|
|
{
|
|
m_sRendererMutex.Lock();
|
|
}
|
|
}
|
|
#endif
|
|
private:
|
|
|
|
class CFileJobsLessFunc
|
|
{
|
|
public:
|
|
int GetLayoutOrderForFilename( const char *pFilename );
|
|
bool Less( FileJob_t* const &pFileJobLHS, FileJob_t* const &pFileJobRHS, void *pCtx );
|
|
};
|
|
|
|
class CResourceNameLessFunc
|
|
{
|
|
public:
|
|
bool Less( const FileNameHandle_t &hFilenameLHS, const FileNameHandle_t &hFilenameRHS, void *pCtx );
|
|
};
|
|
typedef CUtlSortVector< FileNameHandle_t, CResourceNameLessFunc > ResourceList_t;
|
|
|
|
ILoaderProgress * GetProgress() { return m_pProgress; };
|
|
|
|
static void BuildResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime );
|
|
static void BuildMaterialResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime );
|
|
|
|
void PurgeQueue();
|
|
void CleanQueue();
|
|
void SubmitBatchedJobsAndWait();
|
|
void ParseResourceList( CUtlBuffer &resourceList );
|
|
void GetJobRequests();
|
|
void PurgeUnreferencedResources();
|
|
void AddResourceToTable( const char *pFilename );
|
|
void GetDVDLayout();
|
|
|
|
bool m_bStarted;
|
|
bool m_bActive;
|
|
bool m_bBatching;
|
|
bool m_bCanBatch;
|
|
bool m_bLoadForHDR;
|
|
bool m_bDoProgress;
|
|
bool m_bSameMap;
|
|
int m_nSubmitCount;
|
|
unsigned int m_StartTime;
|
|
unsigned int m_EndTime;
|
|
char m_szMapNameToCompareSame[MAX_PATH];
|
|
|
|
CUtlFilenameSymbolTable m_Filenames;
|
|
CTSList< FileJob_t* > m_PendingJobs;
|
|
CTSList< FileJob_t* > m_BatchedJobs;
|
|
CUtlLinkedList< FileJob_t* > m_SubmittedJobs;
|
|
CUtlDict< FileJob_t*, int > m_AnonymousJobs;
|
|
CUtlSymbolTable m_AdditionalResources;
|
|
|
|
CUtlSortVector< FileNameHandle_t, CResourceNameLessFunc > m_ResourceNames[RESOURCEPRELOAD_COUNT];
|
|
CUtlSortVector< FileNameHandle_t, CResourceNameLessFunc > m_ExcludeResourceNames;
|
|
|
|
IResourcePreload *m_pLoaders[RESOURCEPRELOAD_COUNT];
|
|
float m_LoaderTimes[RESOURCEPRELOAD_COUNT];
|
|
ILoaderProgress *m_pProgress;
|
|
CThreadFastMutex m_Mutex;
|
|
#if defined( _PS3 )
|
|
// PLEASE REMOVE THIS MUTEX AFTER WE SHIP, IT IS NOT NEEDED. But leaving it here because it's too close to ship date.
|
|
static CThreadFastMutex m_sRendererMutex;
|
|
// this is the counter of locks we lock the renderer mutex for the ProgressBar call
|
|
// it's used to unlock the mutex when going into Host_Error state to prevent deadlocks (it's a hack)
|
|
static CInterlockedInt m_nRendererMutexProgressBarUnlockCount;
|
|
#endif
|
|
};
|
|
|
|
#if defined( _PS3 )
|
|
CThreadFastMutex CQueuedLoader::m_sRendererMutex;
|
|
CInterlockedInt CQueuedLoader::m_nRendererMutexProgressBarUnlockCount;
|
|
class CInterlockedIntAutoIncrementer
|
|
{
|
|
protected:
|
|
CInterlockedInt &m_refInt;
|
|
public:
|
|
CInterlockedIntAutoIncrementer(CInterlockedInt &refInt): m_refInt( refInt ){ m_refInt ++; }
|
|
~CInterlockedIntAutoIncrementer(){ m_refInt --;}
|
|
};
|
|
#endif
|
|
|
|
static CQueuedLoader g_QueuedLoader;
|
|
EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CQueuedLoader, IQueuedLoader, QUEUEDLOADER_INTERFACE_VERSION, g_QueuedLoader );
|
|
|
|
class CResourcePreloadAnonymous : public IResourcePreload
|
|
{
|
|
virtual void PrepareForCreate( bool bSameMap ) {}
|
|
|
|
virtual bool CreateResource( const char *pName )
|
|
{
|
|
// create an anonymous job to get the data in memory, claimed during load, or auto-freed
|
|
LoaderJob_t loaderJob;
|
|
loaderJob.m_pFilename = pName;
|
|
loaderJob.m_pPathID = "GAME";
|
|
loaderJob.m_Priority = LOADERPRIORITY_DURINGPRELOAD;
|
|
g_QueuedLoader.AddJob( &loaderJob );
|
|
return true;
|
|
}
|
|
|
|
virtual void PurgeUnreferencedResources() {}
|
|
virtual void OnEndMapLoading( bool bAbort ) {}
|
|
virtual void PurgeAll() {}
|
|
#if defined( _PS3 )
|
|
virtual bool RequiresRendererLock() { return true; } // do we know that anonymous resource loads won't hit the renderer?
|
|
#endif // _PS3
|
|
};
|
|
static CResourcePreloadAnonymous s_ResourcePreloadAnonymous;
|
|
|
|
const char *g_ResourceLoaderNames[RESOURCEPRELOAD_COUNT] =
|
|
{
|
|
"???", // RESOURCEPRELOAD_UNKNOWN
|
|
"Sounds", // RESOURCEPRELOAD_SOUND
|
|
"Materials", // RESOURCEPRELOAD_MATERIAL
|
|
"Models", // RESOURCEPRELOAD_MODEL
|
|
"Cubemaps", // RESOURCEPRELOAD_CUBEMAP
|
|
"PropLighting", // RESOURCEPRELOAD_STATICPROPLIGHTING
|
|
"Anonymous", // RESOURCEPRELOAD_ANONYMOUS
|
|
};
|
|
|
|
static CInterlockedInt g_nActiveJobs;
|
|
static CInterlockedInt g_nQueuedJobs;
|
|
static CInterlockedInt g_nHighPriorityJobs; // tracks jobs that must finish during preload
|
|
static CInterlockedInt g_nJobsToFinishBeforePlay; // tracks jobs that must finish before gameplay
|
|
static CInterlockedInt g_nIOMemory; // tracks I/O data from async delivery until consumed
|
|
static CInterlockedInt g_nAnonymousIOMemory; // tracks anonymous I/O data from async delivery until consumed
|
|
static CInterlockedInt g_SuspendIO; // used to throttle the I/O
|
|
static int g_nIOMemoryPeak;
|
|
static int g_nAnonymousIOMemoryPeak;
|
|
static int g_nHighIOSuspensionMark;
|
|
static int g_nLowIOSuspensionMark;
|
|
static CInterlockedInt g_nForceSuspendIO;
|
|
static CUtlVector< CUtlString > g_DVDLayout;
|
|
|
|
ConVar loader_spew_info( "loader_spew_info", "0", 0, "0:Off, 1:Timing, 2:Completions, 3:Late Completions, 4:Creations/Purges, -1:All" );
|
|
ConVar loader_throttle_io( "loader_throttle_io", "1" );
|
|
|
|
// debugging only state to defer all non high priority jobs until the very end of loading
|
|
// essentially making the EndMapLoading() do a bunch of work to stress the loading bar
|
|
ConVar loader_defer_non_critical_jobs( "loader_defer_non_critical_jobs", "0" );
|
|
|
|
CON_COMMAND( loader_dump_table, "" )
|
|
{
|
|
g_QueuedLoader.SpewInfo();
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Constructor
|
|
//-----------------------------------------------------------------------------
|
|
CQueuedLoader::CQueuedLoader()
|
|
{
|
|
m_bStarted = false;
|
|
m_bActive = false;
|
|
m_bSameMap = false;
|
|
m_szMapNameToCompareSame[0] = '\0';
|
|
|
|
m_pProgress = &s_DummyProgress;
|
|
V_memset( m_pLoaders, 0, sizeof( m_pLoaders ) );
|
|
|
|
// set resource dictionaries sort context
|
|
m_ExcludeResourceNames.SetLessContext( (void *)RESOURCEPRELOAD_UNKNOWN );
|
|
for ( intp i = 0; i < RESOURCEPRELOAD_COUNT; i++ )
|
|
{
|
|
m_ResourceNames[i].SetLessContext( (void *)i );
|
|
}
|
|
|
|
InstallLoader( RESOURCEPRELOAD_ANONYMOUS, &s_ResourcePreloadAnonymous );
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Destructor
|
|
//-----------------------------------------------------------------------------
|
|
CQueuedLoader::~CQueuedLoader()
|
|
{
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Computation job to build out objects
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::BuildResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime )
|
|
{
|
|
float t0 = Plat_FloatTime();
|
|
|
|
Assert( pLoader );
|
|
if ( pLoader )
|
|
{
|
|
pLoader->PrepareForCreate( g_QueuedLoader.IsSameMapLoading() );
|
|
|
|
pList->RedoSort();
|
|
|
|
for ( int i = 0; i < pList->Count(); i++ )
|
|
{
|
|
char szFilename[MAX_PATH];
|
|
g_QueuedLoader.GetFilename( pList->Element( i ), szFilename, sizeof( szFilename ) );
|
|
if ( szFilename[0] )
|
|
{
|
|
if ( g_QueuedLoader.GetSpewDetail() & LOADER_DETAIL_CREATIONS )
|
|
{
|
|
Msg( "QueuedLoader: Creating: %s\n", szFilename );
|
|
}
|
|
|
|
bool bResourceCreated = false;
|
|
#if defined( _PS3 )
|
|
// PSGL does not allow us to update its memory from two threads simultaneously,
|
|
// even for unrelated textures, so we need to throttle these down to one at a time
|
|
if( pLoader->RequiresRendererLock() )
|
|
{
|
|
AUTO_LOCK_FM( m_sRendererMutex );
|
|
bResourceCreated = pLoader->CreateResource( szFilename );
|
|
}
|
|
else
|
|
#endif
|
|
{
|
|
bResourceCreated = pLoader->CreateResource( szFilename );
|
|
}
|
|
|
|
if ( !bResourceCreated )
|
|
{
|
|
Warning( "QueuedLoader: Failed to create resource %s\n", szFilename );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// finished with list
|
|
pList->Purge();
|
|
|
|
*pBuildTime = Plat_FloatTime() - t0;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Computation job to build out material objects
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::BuildMaterialResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime )
|
|
{
|
|
float t0 = Plat_FloatTime();
|
|
|
|
char szLastFilename[MAX_PATH];
|
|
szLastFilename[0] = '\0';
|
|
|
|
// ensure cubemaps are first
|
|
pList->RedoSort();
|
|
|
|
// run a clean operation to cull the non-patched env_cubemap materials, which are not built directly
|
|
for ( int i = 0; i < pList->Count(); i++ )
|
|
{
|
|
char szFilename[MAX_PATH];
|
|
char *pFilename = g_QueuedLoader.GetFilename( pList->Element( i ), szFilename, sizeof( szFilename ) );
|
|
if ( !V_stristr( pFilename, "maps\\" ) )
|
|
{
|
|
// list is sorted, first non-cubemap marks end of relevant list
|
|
break;
|
|
}
|
|
|
|
// skip past maps/mapname/
|
|
pFilename += 5;
|
|
pFilename = strchr( pFilename, '\\' ) + 1;
|
|
// back up until end of material name is found, need to strip off _%d_%d_%d.vmt
|
|
char *pEndFilename = V_stristr( pFilename, ".vmt" );
|
|
if ( !pEndFilename )
|
|
{
|
|
pEndFilename = pFilename + strlen( pFilename );
|
|
}
|
|
int numUnderscores = 3;
|
|
while ( pEndFilename != pFilename && numUnderscores > 0 )
|
|
{
|
|
pEndFilename--;
|
|
if ( pEndFilename[0] == '_' )
|
|
{
|
|
numUnderscores--;
|
|
}
|
|
}
|
|
if ( numUnderscores == 0 )
|
|
{
|
|
*pEndFilename = '\0';
|
|
if ( !V_strcmp( szLastFilename, pFilename ) )
|
|
{
|
|
// same cubemap material base already processed, skip it
|
|
continue;
|
|
}
|
|
V_strncpy( szLastFilename, pFilename, sizeof( szLastFilename ) );
|
|
|
|
strcat( pFilename, ".vmt" );
|
|
FileNameHandle_t hFilename = g_QueuedLoader.FindFilename( pFilename );
|
|
if ( hFilename )
|
|
{
|
|
pList->Remove( hFilename );
|
|
}
|
|
}
|
|
}
|
|
|
|
// process clean list
|
|
BuildResources( pLoader, pList, pBuildTime );
|
|
|
|
*pBuildTime = Plat_FloatTime() - t0;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Called by multiple worker threads. Throttle the I/O to ensure too many
|
|
// buffers don't flood the work queue. Anonymous I/O is allowed to grow unbounded.
|
|
//-----------------------------------------------------------------------------
|
|
void AdjustAsyncIOSpeed()
|
|
{
|
|
// throttle back the I/O to keep the pending buffers from exhausting memory
|
|
if ( g_SuspendIO == 0 )
|
|
{
|
|
if ( ( g_nForceSuspendIO == 1 || g_nIOMemory >= g_nHighIOSuspensionMark ) && g_nActiveJobs != 0 )
|
|
{
|
|
// protect against another worker thread
|
|
if ( g_SuspendIO.AssignIf( 0, 1 ) )
|
|
{
|
|
if ( g_QueuedLoader.GetSpewDetail() )
|
|
{
|
|
Msg( "QueuedLoader: Suspending I/O at %.2f MB\n", (float)g_nIOMemory / ( 1024.0f * 1024.0f ) );
|
|
}
|
|
g_pFullFileSystem->AsyncSuspend();
|
|
}
|
|
}
|
|
}
|
|
else if ( g_SuspendIO == 1 )
|
|
{
|
|
if ( g_nForceSuspendIO != 1 && g_nIOMemory <= g_nLowIOSuspensionMark )
|
|
{
|
|
// protect against another worker thread
|
|
if ( g_SuspendIO.AssignIf( 1, 0 ) )
|
|
{
|
|
if ( g_QueuedLoader.GetSpewDetail() )
|
|
{
|
|
Msg( "QueuedLoader: Resuming I/O at %.2f MB\n", (float)g_nIOMemory / ( 1024.0f * 1024.0f ) );
|
|
}
|
|
g_pFullFileSystem->AsyncResume();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Computation job to do work after IO, runs callback
|
|
//-----------------------------------------------------------------------------
|
|
void IOComputationJob( FileJob_t *pFileJob, void *pData, int nSize, LoaderError_t loaderError )
|
|
{
|
|
int spewDetail = g_QueuedLoader.GetSpewDetail();
|
|
if ( spewDetail & ( LOADER_DETAIL_COMPLETIONS|LOADER_DETAIL_LATECOMPLETIONS ) )
|
|
{
|
|
const char *pLateString = "";
|
|
if ( !g_QueuedLoader.IsMapLoading() )
|
|
{
|
|
// completed outside of load process
|
|
pLateString = "(Late) ";
|
|
}
|
|
|
|
if ( ( spewDetail & LOADER_DETAIL_COMPLETIONS ) || ( ( spewDetail & LOADER_DETAIL_LATECOMPLETIONS ) && pLateString[0] ) )
|
|
{
|
|
char szFilename[MAX_PATH];
|
|
g_QueuedLoader.GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) );
|
|
Msg( "QueuedLoader: Computation:%8.8llx, Size:%7d %s%s\n", (unsigned long long)ThreadGetCurrentId(), nSize, pLateString, szFilename );
|
|
}
|
|
}
|
|
|
|
if ( loaderError != LOADERERROR_NONE && pFileJob->m_bFileExists )
|
|
{
|
|
char szFilename[MAX_PATH];
|
|
g_QueuedLoader.GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) );
|
|
Warning( "QueuedLoader:: I/O Error on %s\n", szFilename );
|
|
}
|
|
|
|
pFileJob->m_nActualBytesRead = nSize;
|
|
pFileJob->m_LoaderError = loaderError;
|
|
|
|
if ( !pFileJob->m_pCallback )
|
|
{
|
|
// absent callback means resource loader want this system to delay buffer until ready for it
|
|
if ( !pFileJob->m_pTargetData )
|
|
{
|
|
if ( pFileJob->m_bAnonymousDecode )
|
|
{
|
|
// caller want us to decode
|
|
// the actual inflated memory is not tracked, it should be
|
|
CLZMA lzma;
|
|
if ( lzma.IsCompressed( (unsigned char *)pData ) )
|
|
{
|
|
int originalSize = lzma.GetActualSize( (unsigned char *)pData );
|
|
unsigned char *pDecodedData = (unsigned char *)g_pFullFileSystem->AllocOptimalReadBuffer( FILESYSTEM_INVALID_HANDLE, originalSize, 0 );
|
|
lzma.Uncompress( (unsigned char *)pData, pDecodedData );
|
|
|
|
// release the i/o buffer
|
|
g_pFullFileSystem->FreeOptimalReadBuffer( pData );
|
|
g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead;
|
|
|
|
// now own the decoded data as if it was the i/o buffer
|
|
pData = pDecodedData;
|
|
pFileJob->m_nActualBytesRead = originalSize;
|
|
g_nAnonymousIOMemory += originalSize;
|
|
}
|
|
}
|
|
|
|
// track it for later, unclaimed buffers will get freed
|
|
pFileJob->m_pTargetData = pData;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// regardless of error, call job callback so caller can do cleanup of their context
|
|
pFileJob->m_pCallback( pFileJob->m_pContext, pFileJob->m_pContext2, pData, nSize, loaderError );
|
|
if ( pFileJob->m_bFreeTargetAfterIO && pData )
|
|
{
|
|
// free our data only
|
|
g_pFullFileSystem->FreeOptimalReadBuffer( pData );
|
|
}
|
|
|
|
// memory has been consumed
|
|
g_nIOMemory -= nSize;
|
|
}
|
|
|
|
// mark as completed
|
|
pFileJob->m_bFinished = true;
|
|
pFileJob->m_FinishTime = Plat_MSTime();
|
|
pFileJob->m_ThreadId = ThreadGetCurrentId();
|
|
|
|
if ( pFileJob->m_Priority == LOADERPRIORITY_DURINGPRELOAD )
|
|
{
|
|
g_nHighPriorityJobs--;
|
|
}
|
|
else if ( pFileJob->m_Priority == LOADERPRIORITY_BEFOREPLAY )
|
|
{
|
|
g_nJobsToFinishBeforePlay--;
|
|
}
|
|
|
|
g_nQueuedJobs--;
|
|
|
|
if ( g_nQueuedJobs == 0 && ( spewDetail & LOADER_DETAIL_TIMING ) )
|
|
{
|
|
Warning( "QueuedLoader: Finished I/O of all queued jobs! at: %u\n", Plat_MSTime() - g_QueuedLoader.GetStartTime() );
|
|
}
|
|
|
|
if ( ( g_nForceSuspendIO == -1 ) && g_nJobsToFinishBeforePlay && ( g_nHighPriorityJobs == 0 ) )
|
|
{
|
|
// can only suspend when there are no high priority jobs
|
|
// these cannot be delayed, they must finish unhindered, otherwise we would spin lock
|
|
if ( g_nForceSuspendIO.AssignIf( -1, 1 ) )
|
|
{
|
|
Warning( "QueuedLoader: Force Suspending I/O at: %u\n", Plat_MSTime() - g_QueuedLoader.GetStartTime() );
|
|
}
|
|
}
|
|
|
|
AdjustAsyncIOSpeed();
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Computation job to do work after anonymous job was asynchronously claimed, runs callback.
|
|
//-----------------------------------------------------------------------------
|
|
void FinishAnonymousJob( FileJob_t *pFileJob, QueuedLoaderCallback_t pCallback, void *pContext, void *pContext2 )
|
|
{
|
|
// regardless of error, call job callback so caller can do cleanup of their context
|
|
pCallback( pContext, pContext2, pFileJob->m_pTargetData, pFileJob->m_nActualBytesRead, pFileJob->m_LoaderError );
|
|
if ( pFileJob->m_bFreeTargetAfterIO && pFileJob->m_pTargetData )
|
|
{
|
|
// free our data only
|
|
g_pFullFileSystem->FreeOptimalReadBuffer( pFileJob->m_pTargetData );
|
|
pFileJob->m_pTargetData = NULL;
|
|
}
|
|
|
|
pFileJob->m_bClaimed = true;
|
|
|
|
// memory has been consumed
|
|
g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Callback from I/O job thread. Purposely lightweight as possible to keep i/o from stalling.
|
|
//-----------------------------------------------------------------------------
|
|
void IOAsyncCallback( const FileAsyncRequest_t &asyncRequest, int numReadBytes, FSAsyncStatus_t asyncStatus )
|
|
{
|
|
FileJob_t *pFileJob = (FileJob_t *)asyncRequest.pContext;
|
|
|
|
// interpret the async error
|
|
LoaderError_t loaderError;
|
|
switch ( asyncStatus )
|
|
{
|
|
case FSASYNC_OK:
|
|
loaderError = LOADERERROR_NONE;
|
|
break;
|
|
case FSASYNC_ERR_FILEOPEN:
|
|
loaderError = LOADERERROR_FILEOPEN;
|
|
break;
|
|
default:
|
|
loaderError = LOADERERROR_READING;
|
|
}
|
|
|
|
// track how much i/o data is in flight, consumption will decrement
|
|
if ( !pFileJob->m_pCallback )
|
|
{
|
|
// anonymous io memory is tracked seperatley
|
|
g_nAnonymousIOMemory += numReadBytes;
|
|
if ( g_nAnonymousIOMemory > g_nAnonymousIOMemoryPeak )
|
|
{
|
|
g_nAnonymousIOMemoryPeak = g_nAnonymousIOMemory;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
g_nIOMemory += numReadBytes;
|
|
if ( g_nIOMemory > g_nIOMemoryPeak )
|
|
{
|
|
g_nIOMemoryPeak = g_nIOMemory;
|
|
}
|
|
}
|
|
|
|
// have data or error, do callback as a computation job
|
|
g_pThreadPool->QueueCall( IOComputationJob, pFileJob, asyncRequest.pData, numReadBytes, loaderError )->Release();
|
|
|
|
// don't let the i/o starve, possibly get some more work from the pending queue
|
|
g_QueuedLoader.SubmitPendingJobs();
|
|
|
|
// possibly goes to zero atomically, AFTER submission
|
|
// prevents contention between main thread
|
|
--g_nActiveJobs;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Public method to filename dictionary
|
|
//-----------------------------------------------------------------------------
|
|
char *CQueuedLoader::GetFilename( const FileNameHandle_t hFilename, char *pBuff, int nBuffSize )
|
|
{
|
|
m_Filenames.String( hFilename, pBuff, nBuffSize );
|
|
return pBuff;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Public method to filename dictionary
|
|
//-----------------------------------------------------------------------------
|
|
FileNameHandle_t CQueuedLoader::FindFilename( const char *pFilename )
|
|
{
|
|
return m_Filenames.FindFileName( pFilename );
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Sort function for resource names.
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::CResourceNameLessFunc::Less( const FileNameHandle_t &hFilenameLHS, const FileNameHandle_t &hFilenameRHS, void *pCtx )
|
|
{
|
|
switch ( (intp)pCtx )
|
|
{
|
|
case RESOURCEPRELOAD_MATERIAL:
|
|
{
|
|
// Cubemap materials are expected to be at top of list
|
|
char szNameLHS[MAX_PATH];
|
|
char szNameRHS[MAX_PATH];
|
|
|
|
const char *pNameLHS = g_QueuedLoader.GetFilename( hFilenameLHS, szNameLHS, sizeof( szNameLHS ) );
|
|
const char *pNameRHS = g_QueuedLoader.GetFilename( hFilenameRHS, szNameRHS, sizeof( szNameRHS ) );
|
|
|
|
bool bIsCubemapLHS = V_stristr( pNameLHS, "maps\\" ) != NULL;
|
|
bool bIsCubemapRHS = V_stristr( pNameRHS, "maps\\" ) != NULL;
|
|
if ( bIsCubemapLHS != bIsCubemapRHS )
|
|
{
|
|
return ( bIsCubemapLHS == true && bIsCubemapRHS == false );
|
|
}
|
|
return ( V_stricmp( pNameLHS, pNameRHS ) < 0 );
|
|
}
|
|
break;
|
|
|
|
default:
|
|
// sort not really needed, just use numeric handles
|
|
return ( hFilenameLHS < hFilenameRHS );
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// The layout order of zips is assumed to match the search path order. The zip
|
|
// at the head of the search path should be the closest to the outer (faster)
|
|
// edge progressing to the tail of the search path whose zips should be
|
|
// towards the inner (slower) edge. The validity and necessity of this is really
|
|
// specific to the actual game's data distribution in the zips. For now,
|
|
// good enough.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::GetDVDLayout()
|
|
{
|
|
g_DVDLayout.Purge();
|
|
|
|
char searchPaths[32*MAX_PATH];
|
|
g_pFullFileSystem->GetSearchPath( NULL, true, searchPaths, sizeof( searchPaths ) );
|
|
|
|
for ( char *pPath = strtok( searchPaths, ";" ); pPath; pPath = strtok( NULL, ";" ) )
|
|
{
|
|
if ( V_stristr( pPath, ".zip" ) || V_stristr( pPath, ".bsp" ) )
|
|
{
|
|
// only want zip paths
|
|
g_DVDLayout.AddToTail( pPath );
|
|
}
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Resolve filenames to expected disc layout order. Files within the same zip will
|
|
// resolve to the same numeric order and be alpha sub sorted, same as zip constructed.
|
|
// Files that are from different zips should sort to match the relative placement
|
|
// of the zips on the disc image.
|
|
//-----------------------------------------------------------------------------
|
|
int CQueuedLoader::CFileJobsLessFunc::GetLayoutOrderForFilename( const char *pFilename )
|
|
{
|
|
for ( int i = 0; i < g_DVDLayout.Count(); i++ )
|
|
{
|
|
if ( V_stristr( pFilename, g_DVDLayout[i].String() ) )
|
|
{
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return INT_MAX;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::CFileJobsLessFunc::Less( FileJob_t* const &pFileJobLHS, FileJob_t* const &pFileJobRHS, void *pCtx )
|
|
{
|
|
static int nCalls = 0;
|
|
|
|
nCalls++; // 60,000 times loading background map for l4d.
|
|
|
|
if ( ( nCalls % 200 ) == 0 )
|
|
{
|
|
static float flLastUpdateTime = -1.0f;
|
|
float flTime = Plat_FloatTime();
|
|
|
|
if ( flTime - flLastUpdateTime > .06f )
|
|
{
|
|
flLastUpdateTime = flTime;
|
|
float t = ( ( float )nCalls ) * ( 1.0f / 1000000.0f );
|
|
g_QueuedLoader.GetProgress()->UpdateProgress( PROGRESS_FILESORT + t * ( PROGRESS_IO - PROGRESS_FILESORT ) );
|
|
}
|
|
}
|
|
|
|
if ( pFileJobLHS->m_Priority != pFileJobRHS->m_Priority )
|
|
{
|
|
// higher priorities sort to top
|
|
return ( pFileJobLHS->m_Priority > pFileJobRHS->m_Priority );
|
|
}
|
|
|
|
if ( pFileJobLHS->m_hFilename == pFileJobRHS->m_hFilename )
|
|
{
|
|
// same file (zip), sort by offset
|
|
return pFileJobLHS->m_nStartOffset < pFileJobRHS->m_nStartOffset;
|
|
}
|
|
|
|
char szFilenameLHS[MAX_PATH];
|
|
char szFilenameRHS[MAX_PATH];
|
|
g_QueuedLoader.GetFilename( pFileJobLHS->m_hFilename, szFilenameLHS, sizeof( szFilenameLHS ) );
|
|
g_QueuedLoader.GetFilename( pFileJobRHS->m_hFilename, szFilenameRHS, sizeof( szFilenameRHS ) );
|
|
|
|
// resolve filename to match disk layout of zips
|
|
int layoutLHS = GetLayoutOrderForFilename( szFilenameLHS );
|
|
int layoutRHS = GetLayoutOrderForFilename( szFilenameRHS );
|
|
if ( layoutLHS != layoutRHS )
|
|
{
|
|
return layoutLHS < layoutRHS;
|
|
}
|
|
|
|
return CaselessStringLessThan( szFilenameLHS, szFilenameRHS );
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Dump the queue contents to the file system.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::SubmitPendingJobs()
|
|
{
|
|
// prevents contention between I/O and main thread attempting to submit
|
|
if ( ThreadInMainThread() && g_nActiveJobs != 0 )
|
|
{
|
|
// main thread can only kick start work if the I/O is idle
|
|
// once the I/O is kicked off, the I/O thread is responsible for continual draining
|
|
return;
|
|
}
|
|
else if ( !ThreadInMainThread() && g_nActiveJobs != 1 )
|
|
{
|
|
// I/O thread requests more work, but will only fall through and get some when it expects to go idle
|
|
// I/O thread still has jobs and doesn't need any more yet
|
|
return;
|
|
}
|
|
|
|
CTSList<FileJob_t *>::Node_t *pNode = m_PendingJobs.Detach();
|
|
if ( !pNode )
|
|
{
|
|
return;
|
|
}
|
|
|
|
// used by spew to indicate submission blocks
|
|
m_nSubmitCount++;
|
|
|
|
// sort entries
|
|
CUtlSortVector< FileJob_t*, CFileJobsLessFunc > sortedFiles( 0, 128 );
|
|
while ( pNode )
|
|
{
|
|
FileJob_t *pFileJob = pNode->elem;
|
|
|
|
sortedFiles.InsertNoSort( pFileJob );
|
|
|
|
CTSList<FileJob_t *>::Node_t *pNext = (CTSList<FileJob_t *>::Node_t*)pNode->Next;
|
|
delete pNode;
|
|
pNode = pNext;
|
|
}
|
|
sortedFiles.RedoSort();
|
|
|
|
FileAsyncRequest_t asyncRequest;
|
|
asyncRequest.pfnCallback = IOAsyncCallback;
|
|
|
|
char szFilename[MAX_PATH];
|
|
for ( int i = 0; i<sortedFiles.Count(); i++ )
|
|
{
|
|
FileJob_t *pFileJob = sortedFiles[i];
|
|
|
|
pFileJob->m_SubmitTag = m_nSubmitCount;
|
|
pFileJob->m_SubmitTime = Plat_MSTime();
|
|
|
|
m_SubmittedJobs.AddToTail( pFileJob );
|
|
|
|
// build an async request
|
|
if ( pFileJob->m_Priority == LOADERPRIORITY_DURINGPRELOAD )
|
|
{
|
|
// must finish during preload
|
|
asyncRequest.priority = PRIORITY_HIGH;
|
|
g_nHighPriorityJobs++;
|
|
}
|
|
else if ( pFileJob->m_Priority == LOADERPRIORITY_BEFOREPLAY )
|
|
{
|
|
// must finish before gameplay
|
|
asyncRequest.priority = PRIORITY_NORMAL;
|
|
g_nJobsToFinishBeforePlay++;
|
|
}
|
|
else
|
|
{
|
|
// can finish during gameplay, normal priority
|
|
asyncRequest.priority = PRIORITY_NORMAL;
|
|
}
|
|
|
|
// async will allocate unless caller provided a target
|
|
// loader always takes ownership of buffer
|
|
asyncRequest.pData = pFileJob->m_pTargetData;
|
|
asyncRequest.flags = pFileJob->m_pTargetData ? 0 : FSASYNC_FLAGS_ALLOCNOFREE;
|
|
asyncRequest.nOffset = pFileJob->m_nStartOffset;
|
|
asyncRequest.nBytes = pFileJob->m_nBytesToRead;
|
|
asyncRequest.pszFilename = GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) );
|
|
asyncRequest.pContext = (void *)pFileJob;
|
|
|
|
while( g_SuspendIO )
|
|
{
|
|
ThreadSleep( MAIN_THREAD_YIELD_TIME );
|
|
}
|
|
|
|
if ( pFileJob->m_bFileExists )
|
|
{
|
|
// start the valid async request
|
|
g_nActiveJobs++;
|
|
g_pFullFileSystem->AsyncRead( asyncRequest, &pFileJob->m_hAsyncControl );
|
|
}
|
|
else
|
|
{
|
|
// prevent dragging the i/o system down for known failures
|
|
// still need to do callback so subsystems can do the right thing based on file absence
|
|
g_pThreadPool->QueueCall( IOComputationJob, pFileJob, pFileJob->m_pTargetData, 0, LOADERERROR_FILEOPEN )->Release();
|
|
}
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Add to queue
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::AddJob( const LoaderJob_t *pLoaderJob )
|
|
{
|
|
if ( !m_bActive )
|
|
{
|
|
return false;
|
|
}
|
|
|
|
Assert( pLoaderJob && pLoaderJob->m_pFilename );
|
|
|
|
bool bLateQueued = false;
|
|
if ( m_bCanBatch && !m_bBatching )
|
|
{
|
|
// should have been part of pre-load batch
|
|
DevWarning( "QueuedLoader: Late Queued Job: %s\n", pLoaderJob->m_pFilename );
|
|
bLateQueued = true;
|
|
}
|
|
|
|
// anonymous jobs lack callbacks and are heavily restricted to ensure their stability
|
|
// the caller is expected to claim these before load ends (which auto-purges them)
|
|
if ( !pLoaderJob->m_pCallback && pLoaderJob->m_Priority == LOADERPRIORITY_ANYTIME )
|
|
{
|
|
Assert( 0 );
|
|
DevWarning( "QueuedLoader: Ignoring Anonymous Job: %s\n", pLoaderJob->m_pFilename );
|
|
return false;
|
|
}
|
|
|
|
FileNameHandle_t hFilename = m_Filenames.FindFileName( pLoaderJob->m_pFilename );
|
|
if ( hFilename && m_ExcludeResourceNames.Find( hFilename ) != m_ExcludeResourceNames.InvalidIndex() )
|
|
{
|
|
// marked as excluded, ignoring
|
|
return false;
|
|
}
|
|
|
|
MEM_ALLOC_CREDIT();
|
|
|
|
// all bsp based files get forced to a higher priority in order to achieve a clustered sort
|
|
// the bsp files are not going to be anywhere near the zips, thus we don't want head thrashing
|
|
bool bFileIsFromBSP;
|
|
bool bExists = false;
|
|
|
|
char *pFullPath;
|
|
char szFullPath[MAX_PATH];
|
|
if ( V_IsAbsolutePath( pLoaderJob->m_pFilename ) )
|
|
{
|
|
// an absolute path is trusted, take as is
|
|
pFullPath = (char *)pLoaderJob->m_pFilename;
|
|
bFileIsFromBSP = V_stristr( pFullPath, ".bsp" ) != NULL;
|
|
bExists = true;
|
|
}
|
|
else
|
|
{
|
|
// must resolve now, all submitted paths must be absolute for proper sort which achieves seek linearization
|
|
// a resolved absolute file ensures its existence
|
|
PathTypeFilter_t pathFilter = FILTER_NONE;
|
|
if ( IsX360() )
|
|
{
|
|
if ( V_stristr( pLoaderJob->m_pFilename, ".bsp" ) || V_stristr( pLoaderJob->m_pFilename, ".ain" ) )
|
|
{
|
|
// only the bsp/ain are allowed to be external
|
|
pathFilter = FILTER_CULLPACK;
|
|
}
|
|
else
|
|
{
|
|
// all files are expected to be in zip
|
|
pathFilter = FILTER_CULLNONPACK;
|
|
}
|
|
}
|
|
|
|
PathTypeQuery_t pathType;
|
|
g_pFullFileSystem->RelativePathToFullPath( pLoaderJob->m_pFilename, pLoaderJob->m_pPathID, szFullPath, sizeof( szFullPath ), pathFilter, &pathType );
|
|
bExists = V_IsAbsolutePath( szFullPath );
|
|
pFullPath = szFullPath;
|
|
bFileIsFromBSP = ( (pathType & PATH_IS_MAPPACKFILE) != 0 );
|
|
}
|
|
|
|
// create a file job
|
|
FileJob_t *pFileJob = new FileJob_t;
|
|
|
|
pFileJob->m_hFilename = m_Filenames.FindOrAddFileName( pFullPath );
|
|
pFileJob->m_bFileExists = bExists;
|
|
pFileJob->m_pCallback = pLoaderJob->m_pCallback;
|
|
pFileJob->m_pContext = pLoaderJob->m_pContext;
|
|
pFileJob->m_pContext2 = pLoaderJob->m_pContext2;
|
|
pFileJob->m_pTargetData = pLoaderJob->m_pTargetData;
|
|
pFileJob->m_nBytesToRead = pLoaderJob->m_nBytesToRead;
|
|
pFileJob->m_nStartOffset = pLoaderJob->m_nStartOffset;
|
|
pFileJob->m_Priority = bFileIsFromBSP ? LOADERPRIORITY_DURINGPRELOAD : pLoaderJob->m_Priority;
|
|
pFileJob->m_bLateQueued = bLateQueued;
|
|
pFileJob->m_bAnonymousDecode = pLoaderJob->m_bAnonymousDecode;
|
|
|
|
if ( pLoaderJob->m_pTargetData )
|
|
{
|
|
// never free caller's buffer, if they provide, they have to free it
|
|
pFileJob->m_bFreeTargetAfterIO = false;
|
|
}
|
|
else
|
|
{
|
|
// caller can take over ownership, otherwise it gets freed after I/O
|
|
pFileJob->m_bFreeTargetAfterIO = ( pLoaderJob->m_bPersistTargetData == false );
|
|
}
|
|
|
|
if ( !pLoaderJob->m_pCallback )
|
|
{
|
|
// track anonymous jobs
|
|
AUTO_LOCK_FM( m_Mutex );
|
|
char szFixedName[MAX_PATH];
|
|
V_strncpy( szFixedName, pLoaderJob->m_pFilename, sizeof( szFixedName ) );
|
|
V_FixSlashes( szFixedName );
|
|
m_AnonymousJobs.Insert( szFixedName, pFileJob );
|
|
}
|
|
|
|
g_nQueuedJobs++;
|
|
|
|
if ( m_bBatching )
|
|
{
|
|
m_BatchedJobs.PushItem( pFileJob );
|
|
}
|
|
else
|
|
{
|
|
m_PendingJobs.PushItem( pFileJob );
|
|
SubmitPendingJobs();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Allows an external system to append to a map's reslist. The next map load
|
|
// will append these specified files. Unhandled resources will just get
|
|
// quietly discarded. An external system could use this to patch a hole
|
|
// or prevent a purge.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::AddMapResource( const char *pFilename )
|
|
{
|
|
if ( !pFilename || !pFilename[0] )
|
|
{
|
|
// pointless
|
|
return;
|
|
}
|
|
|
|
// normalize the provided name as a filename
|
|
char szFilename[MAX_PATH];
|
|
V_strncpy( szFilename, pFilename, sizeof( szFilename ) );
|
|
V_FixSlashes( szFilename );
|
|
V_strlower( szFilename );
|
|
|
|
if ( m_AdditionalResources.Find( szFilename ) != UTL_INVAL_SYMBOL )
|
|
{
|
|
// already added
|
|
return;
|
|
}
|
|
|
|
m_AdditionalResources.AddString( szFilename );
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Asynchronous claim for an anonymous job.
|
|
// This allows loaders with deep dependencies to get their data in flight, and then claim it
|
|
// when the they are in a state to consume it.
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::ClaimAnonymousJob( const char *pFilename, QueuedLoaderCallback_t pCallback, void *pContext, void *pContext2 )
|
|
{
|
|
Assert( ThreadInMainThread() );
|
|
Assert( pFilename && pCallback && !m_bBatching );
|
|
|
|
char szFixedName[MAX_PATH];
|
|
V_strncpy( szFixedName, pFilename, sizeof( szFixedName ) );
|
|
V_FixSlashes( szFixedName );
|
|
pFilename = szFixedName;
|
|
|
|
int iIndex = m_AnonymousJobs.Find( pFilename );
|
|
if ( iIndex == m_AnonymousJobs.InvalidIndex() )
|
|
{
|
|
// unknown
|
|
DevWarning( "QueuedLoader: Anonymous Job '%s' not found\n", pFilename );
|
|
return false;
|
|
}
|
|
|
|
// caller is claiming
|
|
FileJob_t *pFileJob = m_AnonymousJobs[iIndex];
|
|
if ( !pFileJob->m_bFinished )
|
|
{
|
|
// unfinished shouldn't happen and caller can't have it
|
|
// anonymous jobs and their claims are very restrictive in such a way to provide stability
|
|
// this dead job will get auto-cleaned at end of map loading
|
|
Assert( 0 );
|
|
return false;
|
|
}
|
|
|
|
m_AnonymousJobs.RemoveAt( iIndex );
|
|
CJob *pClaimedAnonymousJob = g_pThreadPool->QueueCall( FinishAnonymousJob, pFileJob, pCallback, pContext, pContext2 );
|
|
if ( IsPS3() )
|
|
{
|
|
// PS3 static prop lighting (legacy async IO still in flight catching
|
|
// non reslist-lighting buffers) is writing data into raw pointers
|
|
// to RSX memory which have been acquired before material system
|
|
// switches to multithreaded mode. During switch to multithreaded
|
|
// mode RSX moves its memory so pointers become invalid and thus
|
|
// all IO must be finished and callbacks fired before
|
|
// Host_AllowQueuedMaterialSystem
|
|
pClaimedAnonymousJob->WaitForFinishAndRelease();
|
|
}
|
|
else
|
|
{
|
|
pClaimedAnonymousJob->Release();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Synchronous claim for an anonymous job. This allows loaders
|
|
// with deep dependencies to get their data in flight, and then claim it
|
|
// when the they are in a state to consume it.
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::ClaimAnonymousJob( const char *pFilename, void **pData, int *pDataSize, LoaderError_t *pError )
|
|
{
|
|
Assert( ThreadInMainThread() );
|
|
Assert( pFilename && !m_bBatching );
|
|
|
|
char szFixedName[MAX_PATH];
|
|
V_strncpy( szFixedName, pFilename, sizeof( szFixedName ) );
|
|
V_FixSlashes( szFixedName );
|
|
pFilename = szFixedName;
|
|
|
|
int iIndex = m_AnonymousJobs.Find( pFilename );
|
|
if ( iIndex == m_AnonymousJobs.InvalidIndex() )
|
|
{
|
|
// unknown
|
|
DevWarning( "QueuedLoader: Anonymous Job '%s' not found\n", pFilename );
|
|
return false;
|
|
}
|
|
|
|
// caller is claiming
|
|
FileJob_t *pFileJob = m_AnonymousJobs[iIndex];
|
|
if ( !pFileJob->m_bFinished )
|
|
{
|
|
// unfinished shouldn't happen and caller can't have it
|
|
// anonymous jobs and their claims are very restrictive in such a way to provide stability
|
|
// this dead job will get auto-cleaned at end of map loading
|
|
Assert( 0 );
|
|
return false;
|
|
}
|
|
|
|
pFileJob->m_bClaimed = true;
|
|
|
|
m_AnonymousJobs.RemoveAt( iIndex );
|
|
|
|
*pData = pFileJob->m_pTargetData;
|
|
*pDataSize = pFileJob->m_LoaderError == LOADERERROR_NONE ? pFileJob->m_nActualBytesRead : 0;
|
|
if ( pError )
|
|
{
|
|
*pError = pFileJob->m_LoaderError;
|
|
}
|
|
|
|
// caller owns the data, regardless of how the job was setup
|
|
pFileJob->m_pTargetData = NULL;
|
|
|
|
// memory has been consumed
|
|
g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead;
|
|
|
|
return true;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// End of batching. High priority jobs are guaranteed completed before function returns.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::SubmitBatchedJobsAndWait()
|
|
{
|
|
// end of batching
|
|
m_bBatching = false;
|
|
|
|
CTSList<FileJob_t *>::Node_t *pNode = m_BatchedJobs.Detach();
|
|
if ( !pNode )
|
|
{
|
|
return;
|
|
}
|
|
|
|
// must wait for any initial i/o to finish
|
|
// i/o thread must stop in order to submit all the batched jobs atomically
|
|
// and get an accurate accounting of high priority jobs
|
|
while ( g_nActiveJobs != 0 )
|
|
{
|
|
g_pThreadPool->Yield( MAIN_THREAD_YIELD_TIME );
|
|
}
|
|
|
|
// dump batched jobs to pending jobs
|
|
while ( pNode )
|
|
{
|
|
FileJob_t *pFileJob = pNode->elem;
|
|
|
|
m_PendingJobs.PushItem( pFileJob );
|
|
|
|
CTSList<FileJob_t *>::Node_t *pNext = (CTSList<FileJob_t *>::Node_t*)pNode->Next;
|
|
delete pNode;
|
|
pNode = pNext;
|
|
}
|
|
|
|
SubmitPendingJobs();
|
|
|
|
if ( GetSpewDetail() )
|
|
{
|
|
Msg( "QueuedLoader: High Priority Jobs: %d\n", ( int ) g_nHighPriorityJobs );
|
|
}
|
|
|
|
if ( loader_defer_non_critical_jobs.GetBool() )
|
|
{
|
|
// allow to trigger at next available oppportunity
|
|
g_nForceSuspendIO = -1;
|
|
}
|
|
|
|
// finish only the high priority jobs
|
|
// high priority jobs are expected to be complete at the conclusion of batching
|
|
int total = g_nHighPriorityJobs;
|
|
while ( g_nHighPriorityJobs != 0 )
|
|
{
|
|
float t = (float)( total - g_nHighPriorityJobs ) / (float)total;
|
|
m_pProgress->UpdateProgress( PROGRESS_IO + t * ( PROGRESS_END - PROGRESS_IO ) );
|
|
|
|
// yield some time
|
|
g_pThreadPool->Yield( MAIN_THREAD_YIELD_TIME );
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Clean queue of stale entries. Active entries are skipped.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::CleanQueue()
|
|
{
|
|
for ( int i = 0; i<RESOURCEPRELOAD_COUNT; i++ )
|
|
{
|
|
m_ResourceNames[i].Purge();
|
|
}
|
|
|
|
m_ExcludeResourceNames.Purge();
|
|
m_BatchedJobs.Purge();
|
|
|
|
int iIndex = m_SubmittedJobs.Head();
|
|
while ( iIndex != m_SubmittedJobs.InvalidIndex() )
|
|
{
|
|
int iNext = m_SubmittedJobs.Next( iIndex );
|
|
|
|
FileJob_t *pFileJob = m_SubmittedJobs[iIndex];
|
|
if ( pFileJob->m_bFinished )
|
|
{
|
|
// job is complete, safe to free
|
|
m_SubmittedJobs.Free( iIndex );
|
|
g_pFullFileSystem->AsyncRelease( pFileJob->m_hAsyncControl );
|
|
delete pFileJob;
|
|
}
|
|
iIndex = iNext;
|
|
}
|
|
|
|
m_Filenames.RemoveAll();
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Abandon queue
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::PurgeQueue()
|
|
{
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Spew info abut queued load
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::SpewInfo()
|
|
{
|
|
Msg( "Queued Loader:\n\n" );
|
|
|
|
int totalClaimed = 0;
|
|
int totalUnclaimed = 0;
|
|
|
|
if ( IsFinished() )
|
|
{
|
|
// can only access submitted jobs safely when io thread complete
|
|
int lastPriority = -1;
|
|
int lastLateState = -1;
|
|
int iIndex = m_SubmittedJobs.Head();
|
|
while ( iIndex != m_SubmittedJobs.InvalidIndex() )
|
|
{
|
|
FileJob_t *pFileJob = m_SubmittedJobs[iIndex];
|
|
|
|
int asyncDuration = -1;
|
|
if ( pFileJob->m_FinishTime )
|
|
{
|
|
asyncDuration = pFileJob->m_FinishTime - pFileJob->m_SubmitTime;
|
|
}
|
|
|
|
if ( pFileJob->m_bLateQueued != lastLateState )
|
|
{
|
|
if ( pFileJob->m_bLateQueued )
|
|
{
|
|
Warning( "---- LATE QUEUED JOBS ----\n" );
|
|
}
|
|
lastLateState = pFileJob->m_bLateQueued;
|
|
}
|
|
|
|
if ( !pFileJob->m_bLateQueued && pFileJob->m_Priority != lastPriority )
|
|
{
|
|
switch ( pFileJob->m_Priority )
|
|
{
|
|
case LOADERPRIORITY_DURINGPRELOAD:
|
|
Msg( "---- FINISH DURING PRELOAD ( HIGH PRIORITY )----\n" );
|
|
break;
|
|
case LOADERPRIORITY_BEFOREPLAY:
|
|
Msg( "---- FINISH BEFORE GAMEPLAY ( NORMAL PRIORITY )----\n" );
|
|
break;
|
|
case LOADERPRIORITY_ANYTIME:
|
|
Msg( "---- FINISH ANYTIME ( NORMAL PRIORITY )----\n" );
|
|
break;
|
|
}
|
|
lastPriority = pFileJob->m_Priority;
|
|
}
|
|
|
|
char szAnonymousString[MAX_PATH];
|
|
const char *pAnonymousStatus = "";
|
|
if ( !pFileJob->m_pCallback )
|
|
{
|
|
V_snprintf( szAnonymousString, sizeof( szAnonymousString ), "(%s) ", pFileJob->m_bClaimed ? "Claimed" : "Unclaimed" );
|
|
pAnonymousStatus = szAnonymousString;
|
|
|
|
if ( pFileJob->m_bClaimed )
|
|
{
|
|
totalClaimed += pFileJob->m_nActualBytesRead;
|
|
}
|
|
else
|
|
{
|
|
totalUnclaimed += pFileJob->m_nActualBytesRead;
|
|
}
|
|
}
|
|
|
|
char szFilename[MAX_PATH];
|
|
char szMessage[1024];
|
|
V_snprintf(
|
|
szMessage,
|
|
sizeof( szMessage ),
|
|
"Submit:%5dms AsyncDuration:%5dms Tag:%d Thread:%8.8x Size:%7d %s%s",
|
|
pFileJob->m_SubmitTime - m_StartTime,
|
|
asyncDuration,
|
|
pFileJob->m_SubmitTag,
|
|
pFileJob->m_ThreadId,
|
|
pFileJob->m_nActualBytesRead,
|
|
pAnonymousStatus,
|
|
GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) ) );
|
|
|
|
if ( pFileJob->m_bLateQueued )
|
|
{
|
|
Warning( "%s\n", szMessage );
|
|
}
|
|
else
|
|
{
|
|
Msg( "%s\n", szMessage );
|
|
}
|
|
|
|
iIndex = m_SubmittedJobs.Next( iIndex );
|
|
}
|
|
|
|
Msg( "%d Total Jobs\n", m_SubmittedJobs.Count() );
|
|
}
|
|
|
|
Msg( "\nExcludes:\n" );
|
|
for ( int i = 0; i < m_ExcludeResourceNames.Count(); i++ )
|
|
{
|
|
char szFilename[MAX_PATH] = "";
|
|
GetFilename( m_ExcludeResourceNames[i], szFilename, sizeof( szFilename ) );
|
|
Msg( "%s\n", szFilename );
|
|
}
|
|
|
|
Msg( "\nLayout Order:\n" );
|
|
for ( int i = 0; i < g_DVDLayout.Count(); i++ )
|
|
{
|
|
Msg( "%s\n", g_DVDLayout[i].String() );
|
|
}
|
|
|
|
Msg( "\n" );
|
|
Msg( "%d Queued Jobs\n", ( int ) g_nQueuedJobs );
|
|
Msg( "%d Active Jobs\n", ( int ) g_nActiveJobs );
|
|
Msg( "Peak IO Memory: %.2f MB\n", (float)g_nIOMemoryPeak / ( 1024.0f * 1024.0f ) );
|
|
Msg( "Peak Anonymous IO Memory: %.2f MB\n", (float)g_nAnonymousIOMemoryPeak / ( 1024.0f * 1024.0f ) );
|
|
Msg( " Total Anonymous Claimed: %d\n", totalClaimed );
|
|
Msg( " Total Anonymous Unclaimed: %d\n", totalUnclaimed );
|
|
if ( m_EndTime )
|
|
{
|
|
Msg( "Queuing Duration: %dms\n", m_EndTime - m_StartTime );
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Initialization
|
|
//-----------------------------------------------------------------------------
|
|
InitReturnVal_t CQueuedLoader::Init()
|
|
{
|
|
InitReturnVal_t nRetVal = BaseClass::Init();
|
|
if ( nRetVal != INIT_OK )
|
|
{
|
|
return nRetVal;
|
|
}
|
|
|
|
return INIT_OK;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Shutdown
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::Shutdown()
|
|
{
|
|
BaseClass::Shutdown();
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Install a type specific interface from managing system.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::InstallLoader( ResourcePreload_t type, IResourcePreload *pLoader )
|
|
{
|
|
m_pLoaders[type] = pLoader;
|
|
}
|
|
|
|
void CQueuedLoader::InstallProgress( ILoaderProgress *pProgress )
|
|
{
|
|
m_pProgress = pProgress;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Invoke the loader systems to purge dead resources
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::PurgeUnreferencedResources()
|
|
{
|
|
ResourcePreload_t purgeOrder[RESOURCEPRELOAD_COUNT];
|
|
|
|
// the purge operations require a specific order (models and cubemaps before materials)
|
|
int numPurges = 0;
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_SOUND;
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_STATICPROPLIGHTING;
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_MODEL;
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_CUBEMAP;
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_MATERIAL;
|
|
// 'gpubufferallocator' must be LAST so that it can reclaim released physical memory
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_GPUBUFFERALLOCATOR;
|
|
|
|
// iterate according to order
|
|
for ( int i = 0; i < numPurges; i++ )
|
|
{
|
|
ResourcePreload_t loader = purgeOrder[i];
|
|
if ( m_pLoaders[loader] )
|
|
{
|
|
m_pLoaders[loader]->PurgeUnreferencedResources();
|
|
}
|
|
}
|
|
|
|
m_pProgress->UpdateProgress( PROGRESS_PREPURGE );
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Invoke the loader systems to purge all resources, if possible.
|
|
// NOT meant to be used when transitioning to the same map.
|
|
// This trips expensive resource eviction.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::PurgeAll( ResourcePreload_t *pDontPurgeList, int nPurgeListSize )
|
|
{
|
|
ResourcePreload_t purgeOrder[RESOURCEPRELOAD_COUNT];
|
|
|
|
// default purge all
|
|
unsigned int bPurgeMask = 0xFFFFFFFF;
|
|
|
|
// caller can narrow purge and inhibit specific resources
|
|
if ( pDontPurgeList && nPurgeListSize > 0 )
|
|
{
|
|
for ( int i = 0; i < nPurgeListSize; i++ )
|
|
{
|
|
bPurgeMask &= ~( 1 << pDontPurgeList[i] );
|
|
}
|
|
}
|
|
|
|
// the purge operations require a specific order (models and cubemaps before materials)
|
|
int numPurges = 0;
|
|
if ( bPurgeMask & ( 1 << RESOURCEPRELOAD_SOUND ) )
|
|
{
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_SOUND;
|
|
}
|
|
if ( bPurgeMask & ( 1 << RESOURCEPRELOAD_STATICPROPLIGHTING ) )
|
|
{
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_STATICPROPLIGHTING;
|
|
}
|
|
if ( bPurgeMask & ( 1 << RESOURCEPRELOAD_MODEL ) )
|
|
{
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_MODEL;
|
|
}
|
|
if ( bPurgeMask & ( 1 << RESOURCEPRELOAD_CUBEMAP ) )
|
|
{
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_CUBEMAP;
|
|
}
|
|
if ( bPurgeMask & ( 1 << RESOURCEPRELOAD_MATERIAL ) )
|
|
{
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_MATERIAL;
|
|
}
|
|
// 'gpubufferallocator' must be LAST so that it can reclaim released physical memory
|
|
if ( bPurgeMask & ( 1 << RESOURCEPRELOAD_GPUBUFFERALLOCATOR ) )
|
|
{
|
|
purgeOrder[numPurges++] = RESOURCEPRELOAD_GPUBUFFERALLOCATOR;
|
|
}
|
|
|
|
// iterate according to order
|
|
for ( int i = 0; i < numPurges; i++ )
|
|
{
|
|
ResourcePreload_t loader = purgeOrder[i];
|
|
if ( m_pLoaders[loader] )
|
|
{
|
|
m_pLoaders[loader]->PurgeAll();
|
|
}
|
|
}
|
|
|
|
// any purge means that we can no longer supply the "same map" loading hint
|
|
// hopefully the caller does not invoke a PurgeAll() when transitioning to the same map
|
|
*m_szMapNameToCompareSame = '\0';
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Invoke the loader systems to request i/o jobs, which are batched.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::GetJobRequests()
|
|
{
|
|
COM_TimestampedLog( "CQueuedLoader::GetJobRequests - Start" );
|
|
|
|
// causes the batch queue to fill with i/o requests
|
|
m_bCanBatch = true;
|
|
m_bBatching = true;
|
|
|
|
float t0, flLastUpdateT;
|
|
t0 = Plat_FloatTime();
|
|
CJob *jobs[5];
|
|
|
|
// cubemap textures must be first to install correctly before their cubemap materials are built (and precache the cubmeap textures)
|
|
// cannot be overlapped, must run serially
|
|
BuildResources( m_pLoaders[RESOURCEPRELOAD_CUBEMAP], &m_ResourceNames[RESOURCEPRELOAD_CUBEMAP], &m_LoaderTimes[RESOURCEPRELOAD_CUBEMAP] );
|
|
|
|
// Overlapping these is not critical in any way, total time is currently < 2 seconds.
|
|
// These operations flood calls (AddJob) back into the queued loader (which has to mutex its lists),
|
|
// so in fact it's slightly slower to queue these at this stage. As these routines age they may become more heavyweight.
|
|
jobs[0] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_SOUND], &m_ResourceNames[RESOURCEPRELOAD_SOUND], &m_LoaderTimes[RESOURCEPRELOAD_SOUND] );
|
|
jobs[1] = g_pThreadPool->QueueCall( BuildMaterialResources, m_pLoaders[RESOURCEPRELOAD_MATERIAL], &m_ResourceNames[RESOURCEPRELOAD_MATERIAL], &m_LoaderTimes[RESOURCEPRELOAD_MATERIAL] );
|
|
jobs[2] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_STATICPROPLIGHTING], &m_ResourceNames[RESOURCEPRELOAD_STATICPROPLIGHTING], &m_LoaderTimes[RESOURCEPRELOAD_STATICPROPLIGHTING] );
|
|
jobs[3] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_MODEL], &m_ResourceNames[RESOURCEPRELOAD_MODEL], &m_LoaderTimes[RESOURCEPRELOAD_MODEL] );
|
|
jobs[4] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_ANONYMOUS], &m_ResourceNames[RESOURCEPRELOAD_ANONYMOUS], &m_LoaderTimes[RESOURCEPRELOAD_ANONYMOUS] );
|
|
|
|
// all jobs must finish
|
|
flLastUpdateT = -1000.0f;
|
|
// Update as if this takes 2 seconds
|
|
float flDelta = ( PROGRESS_CREATEDRESOURCES - PROGRESS_PARSEDRESLIST ) * 0.03 / 2.0f;
|
|
float flProgress = PROGRESS_PARSEDRESLIST;
|
|
while( true )
|
|
{
|
|
bool bIsDone = true;
|
|
for ( int i=0; i<ARRAYSIZE( jobs ); i++ )
|
|
{
|
|
if ( !jobs[i]->IsFinished() )
|
|
{
|
|
bIsDone = false;
|
|
break;
|
|
}
|
|
}
|
|
if ( bIsDone )
|
|
break;
|
|
|
|
// Can't sleep; that will allow this thread to be used by the thread pool
|
|
float newt = Plat_FloatTime();
|
|
if ( newt - flLastUpdateT > .03 )
|
|
{
|
|
#if defined( _PS3 )
|
|
AUTO_LOCK_FM( m_sRendererMutex );
|
|
CInterlockedIntAutoIncrementer autoIncrementRendererMutex( m_nRendererMutexProgressBarUnlockCount );
|
|
#endif
|
|
m_pProgress->UpdateProgress( flProgress );
|
|
flProgress = clamp( flProgress + flDelta, PROGRESS_PARSEDRESLIST, PROGRESS_CREATEDRESOURCES );
|
|
|
|
// Necessary to take into account any waits for vsync
|
|
flLastUpdateT = Plat_FloatTime();
|
|
}
|
|
}
|
|
|
|
for ( int i=0; i<ARRAYSIZE( jobs ); i++ )
|
|
{
|
|
jobs[i]->Release();
|
|
}
|
|
|
|
if ( g_QueuedLoader.GetSpewDetail() & LOADER_DETAIL_TIMING )
|
|
{
|
|
for ( int i = RESOURCEPRELOAD_UNKNOWN+1; i<RESOURCEPRELOAD_COUNT; i++ )
|
|
{
|
|
Msg( "QueuedLoader: %s Creating: %.2f seconds\n", g_ResourceLoaderNames[i], m_LoaderTimes[i] );
|
|
}
|
|
Msg( "QueuedLoader: Total Creating: %.2f seconds\n", Plat_FloatTime() - t0 );
|
|
}
|
|
|
|
m_pProgress->UpdateProgress( PROGRESS_CREATEDRESOURCES );
|
|
|
|
COM_TimestampedLog( "CQueuedLoader::GetJobRequests - End" );
|
|
}
|
|
|
|
void CQueuedLoader::AddResourceToTable( const char *pFilename )
|
|
{
|
|
const char *pExt = V_GetFileExtension( pFilename );
|
|
if ( !pExt )
|
|
{
|
|
// unknown
|
|
// all resources are identified by their extension
|
|
return;
|
|
}
|
|
|
|
if ( pFilename[0] == '*' )
|
|
{
|
|
// this is a job exclusion, add to exclusion list
|
|
FileNameHandle_t hFilename = m_Filenames.FindOrAddFileName( pFilename + 1 );
|
|
m_ExcludeResourceNames.InsertNoSort( hFilename );
|
|
return;
|
|
}
|
|
|
|
const char *pTypeDir = NULL;
|
|
const char *pName = pFilename;
|
|
ResourcePreload_t type = RESOURCEPRELOAD_UNKNOWN;
|
|
|
|
if ( !V_stricmp( pExt, "wav" ) )
|
|
{
|
|
type = RESOURCEPRELOAD_SOUND;
|
|
pTypeDir = "sound\\";
|
|
}
|
|
else if ( !V_stricmp( pExt, "vmt" ) )
|
|
{
|
|
type = RESOURCEPRELOAD_MATERIAL;
|
|
pTypeDir = "materials\\";
|
|
}
|
|
else if ( !V_stricmp( pExt, "vtf" ) )
|
|
{
|
|
if ( V_stristr( pFilename, "maps\\" ) )
|
|
{
|
|
// only want cubemap textures
|
|
if ( !m_bLoadForHDR && V_stristr( pFilename, ".hdr." ) )
|
|
{
|
|
return;
|
|
}
|
|
else if ( m_bLoadForHDR && !V_stristr( pFilename, ".hdr." ) )
|
|
{
|
|
return;
|
|
}
|
|
type = RESOURCEPRELOAD_CUBEMAP;
|
|
pTypeDir = "materials\\";
|
|
}
|
|
else
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
else if ( !V_stricmp( pExt, "mdl" ) )
|
|
{
|
|
type = RESOURCEPRELOAD_MODEL;
|
|
pTypeDir = "models\\";
|
|
}
|
|
else if ( !V_stricmp( pExt, "vhv" ) )
|
|
{
|
|
// want static props only
|
|
pName = V_stristr( pFilename, "sp_" );
|
|
if ( !pName )
|
|
{
|
|
return;
|
|
}
|
|
|
|
if ( !m_bLoadForHDR && V_stristr( pFilename, "_hdr_" ) )
|
|
{
|
|
return;
|
|
}
|
|
else if ( m_bLoadForHDR && !V_stristr( pFilename, "_hdr_" ) )
|
|
{
|
|
return;
|
|
}
|
|
type = RESOURCEPRELOAD_STATICPROPLIGHTING;
|
|
}
|
|
else
|
|
{
|
|
// unknown, ignored
|
|
return;
|
|
}
|
|
|
|
if ( pTypeDir )
|
|
{
|
|
// want object name only
|
|
// skip past game/type directory prefixing
|
|
const char *pDir = V_stristr( pName, pTypeDir );
|
|
if ( pDir )
|
|
{
|
|
pName = pDir + strlen( pTypeDir );
|
|
}
|
|
}
|
|
|
|
FileNameHandle_t hFilename = m_Filenames.FindOrAddFileName( pName );
|
|
m_ResourceNames[type].InsertNoSort( hFilename );
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Parse the raw resource list into resource dictionaries
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::ParseResourceList( CUtlBuffer &resourceList )
|
|
{
|
|
// parse resource list into known types
|
|
characterset_t breakSet;
|
|
CharacterSetBuild( &breakSet, "" );
|
|
char szToken[MAX_PATH];
|
|
int nCount = 0;
|
|
for ( ;; )
|
|
{
|
|
int nTokenSize = resourceList.ParseToken( &breakSet, szToken, sizeof( szToken ) );
|
|
if ( nTokenSize <= 0 )
|
|
{
|
|
break;
|
|
}
|
|
AddResourceToTable( szToken );
|
|
nCount++;
|
|
// Keep the spinner going!
|
|
if ( ( nCount % 1200 ) == 0 )
|
|
{
|
|
m_pProgress->UpdateProgress( PROGRESS_GOTRESLIST );
|
|
}
|
|
}
|
|
|
|
// add any additional resources
|
|
// duplicates don't need to be culled, loaders are supposed to handle resources that already exist
|
|
for ( int i = 0; i < m_AdditionalResources.GetNumStrings(); i++ )
|
|
{
|
|
if ( g_QueuedLoader.GetSpewDetail() )
|
|
{
|
|
Msg( "QueuedLoader: Appending: %s\n", m_AdditionalResources.String( i ) );
|
|
}
|
|
AddResourceToTable( m_AdditionalResources.String( i ) );
|
|
}
|
|
|
|
// all done with adds
|
|
m_ExcludeResourceNames.RedoSort();
|
|
|
|
if ( g_QueuedLoader.GetSpewDetail() )
|
|
{
|
|
for ( int i = RESOURCEPRELOAD_UNKNOWN+1; i < RESOURCEPRELOAD_COUNT; i++ )
|
|
{
|
|
Msg( "QueuedLoader: %s: %d Entries\n", g_ResourceLoaderNames[i], m_ResourceNames[i].Count() );
|
|
}
|
|
|
|
Msg( "QueuedLoader: Excludes: %d Entries\n", m_ExcludeResourceNames.Count() );
|
|
}
|
|
|
|
m_pProgress->UpdateProgress( PROGRESS_PARSEDRESLIST );
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Mark the start of the queued loading process.
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::BeginMapLoading( const char *pMapName, bool bLoadForHDR, bool bOptimizeMapReload, void (*pfnBeginMapLoadingCallback)( int nStage ) )
|
|
{
|
|
if ( IsPC() )
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if ( CommandLine()->FindParm( "-noqueuedload" ) ||
|
|
CommandLine()->FindParm( "-noqueuedloader" ) )
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if ( m_bStarted )
|
|
{
|
|
// already started, shouldn't be started more than once
|
|
Assert( 0 );
|
|
return true;
|
|
}
|
|
|
|
COM_TimestampedLog( "CQueuedLoader::BeginMapLoading" );
|
|
|
|
GetDVDLayout();
|
|
|
|
// set the IO throttle markers based on available memory
|
|
// these safety watermarks throttle the i/o from flooding memory, when the cores cannot keep up
|
|
// the delta must be larger than any single operation, otherwise deadlock
|
|
// markers that are too close will cause excessive suspension
|
|
if ( loader_throttle_io.GetInt() )
|
|
{
|
|
size_t usedMemory, freeMemory;
|
|
g_pMemAlloc->GlobalMemoryStatus( &usedMemory, &freeMemory );
|
|
if ( freeMemory >= 64*1024*1024 )
|
|
{
|
|
// lots of available memory, can afford to have let the i/o get ahead
|
|
g_nHighIOSuspensionMark = 10*1024*1024;
|
|
g_nLowIOSuspensionMark = 2*1024*1024;
|
|
}
|
|
else
|
|
{
|
|
// low memory, suspend the i/o more frequently
|
|
g_nHighIOSuspensionMark = 5*1024*1024;
|
|
g_nLowIOSuspensionMark = 1*1024*1024;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// no IO throttling
|
|
g_nHighIOSuspensionMark = 512*1024*1024;
|
|
g_nLowIOSuspensionMark = 512*1024*1024;
|
|
}
|
|
|
|
// default disabled
|
|
// can force an artificial i/o suspension, for debugging only
|
|
g_nForceSuspendIO = 0;
|
|
|
|
if ( GetSpewDetail() )
|
|
{
|
|
Msg( "QueuedLoader: Suspend I/O at [%.2f,%.2f] MB\n", (float)g_nLowIOSuspensionMark/(1024.0f*1024.0f), (float)g_nHighIOSuspensionMark/(1024.0f*1024.0f) );
|
|
}
|
|
|
|
m_bStarted = true;
|
|
m_bLoadForHDR = bLoadForHDR;
|
|
|
|
// map pak will be accessed asynchronously throughout loading and into game frame
|
|
g_pFullFileSystem->BeginMapAccess();
|
|
|
|
// remove any prior stale entries
|
|
CleanQueue();
|
|
Assert( m_SubmittedJobs.Count() == 0 && g_nActiveJobs == 0 && g_nQueuedJobs == 0 );
|
|
|
|
m_bActive = true;
|
|
m_nSubmitCount = 0;
|
|
m_StartTime = Plat_MSTime();
|
|
m_EndTime = 0;
|
|
m_bCanBatch = false;
|
|
m_bBatching = false;
|
|
m_bDoProgress = false;
|
|
|
|
g_nIOMemory = 0;
|
|
g_nAnonymousIOMemory = 0;
|
|
g_nIOMemoryPeak = 0;
|
|
g_nAnonymousIOMemoryPeak = 0;
|
|
|
|
m_bSameMap = bOptimizeMapReload && ( V_stricmp( pMapName, m_szMapNameToCompareSame ) == 0 );
|
|
if ( m_bSameMap )
|
|
{
|
|
// Data will persist (so reloading a map is fast)
|
|
}
|
|
else
|
|
{
|
|
// Full load of the new map's data
|
|
V_strncpy( m_szMapNameToCompareSame, pMapName, sizeof( m_szMapNameToCompareSame ) );
|
|
}
|
|
|
|
m_pProgress->BeginProgress();
|
|
m_pProgress->UpdateProgress( PROGRESS_START );
|
|
|
|
// load this map's resource list before any other i/o
|
|
char szBaseName[MAX_PATH];
|
|
char szFilename[MAX_PATH];
|
|
V_FileBase( pMapName, szBaseName, sizeof( szBaseName ) );
|
|
V_snprintf( szFilename, sizeof( szFilename ), "reslists_xbox/%s%s.lst", szBaseName, GetPlatformExt() );
|
|
|
|
CUtlBuffer resListBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER );
|
|
if ( !g_pFullFileSystem->ReadFile( szFilename, "GAME", resListBuffer, 0, 0 ) )
|
|
{
|
|
// very bad, a valid reslist is critical
|
|
DevWarning( "QueuedLoader: Failed to get reslist '%s', Non-Optimal Loading.\n", szFilename );
|
|
if ( IsGameConsole() )
|
|
{
|
|
// We can't ship with this; it's fatal.
|
|
Msg( "\n\n\n########\n######## BAD IMAGE: cannot load map reslist (%s)!\n########\n\n\n\n", szFilename );
|
|
DebuggerBreakIfDebugging();
|
|
}
|
|
m_bActive = false;
|
|
return false;
|
|
}
|
|
|
|
if ( XBX_IsLocalized() )
|
|
{
|
|
// find optional localized reslist fixup
|
|
V_snprintf( szFilename, sizeof( szFilename ), "reslists_xbox/%s%s.lst", XBX_GetLanguageString(), GetPlatformExt() );
|
|
CUtlBuffer localizedBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER );
|
|
if ( g_pFullFileSystem->ReadFile( szFilename, "GAME", localizedBuffer, 0, 0 ) )
|
|
{
|
|
// append it
|
|
resListBuffer.EnsureCapacity( resListBuffer.TellPut() + localizedBuffer.TellPut() );
|
|
resListBuffer.Put( localizedBuffer.PeekGet(), localizedBuffer.TellPut() );
|
|
}
|
|
}
|
|
|
|
m_pProgress->UpdateProgress( PROGRESS_GOTRESLIST );
|
|
|
|
// due to its size, the bsp load is a lengthy i/o operation
|
|
// this causes a non-batched async i/o operation to commence immediately
|
|
m_pLoaders[RESOURCEPRELOAD_MODEL]->PrepareForCreate( m_bSameMap );
|
|
if ( !m_pLoaders[RESOURCEPRELOAD_MODEL]->CreateResource( pMapName ) )
|
|
{
|
|
// very bad, a valid bsp is critical
|
|
DevWarning( "QueuedLoader: Failed to mount BSP '%s', Non-Optimal Loading.\n", pMapName );
|
|
m_bActive = false;
|
|
return false;
|
|
}
|
|
|
|
// parse the raw resource list into loader specific dictionaries
|
|
ParseResourceList( resListBuffer );
|
|
|
|
// run the distributed precache loaders, generating a batch of i/o requests
|
|
GetJobRequests();
|
|
|
|
// event each loader to discard dead resources
|
|
PurgeUnreferencedResources();
|
|
|
|
if ( pfnBeginMapLoadingCallback )
|
|
{
|
|
(*pfnBeginMapLoadingCallback)( 1 );
|
|
}
|
|
|
|
// sort and start async fulfilling the i/o requests
|
|
// waits for all "must complete" jobs to finish
|
|
SubmitBatchedJobsAndWait();
|
|
|
|
// progress is only relevant during preload
|
|
// normal load process takes over any progress bar
|
|
// disable progress tracking to prevent any late queued operation from updating
|
|
m_pProgress->EndProgress();
|
|
|
|
return m_bActive;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Signal the end of the queued loading process, i/o will still be in progress.
|
|
//-----------------------------------------------------------------------------
|
|
void CQueuedLoader::EndMapLoading( bool bAbort )
|
|
{
|
|
if ( !m_bStarted )
|
|
{
|
|
// already stopped or never started
|
|
return;
|
|
}
|
|
|
|
/////////////////////////////////////////////////////
|
|
// TBD: Cannot abort!!!! feature has not been done //
|
|
/////////////////////////////////////////////////////
|
|
bAbort = false;
|
|
|
|
bool bShutdownAppRequest = false;
|
|
#ifdef _PS3
|
|
bShutdownAppRequest = GetTLSGlobals()->bNormalQuitRequested;
|
|
#endif
|
|
|
|
// To satisfy system shutdown request, purge all pending jobs
|
|
// that haven't been submitted yet
|
|
if ( bShutdownAppRequest )
|
|
{
|
|
m_PendingJobs.Purge();
|
|
}
|
|
|
|
if ( m_bActive )
|
|
{
|
|
if ( bAbort )
|
|
{
|
|
PurgeQueue();
|
|
}
|
|
else
|
|
{
|
|
if ( GetSpewDetail() )
|
|
{
|
|
// Under normal HDD circumstances this would be 0, the HDD can deliver fast enough
|
|
// that the jobs are done before the legacy load finishes. Under DVD maybe a few.
|
|
// When the debugging loader_defer_non_critical_jobs convar is active this will
|
|
// be a substantial portion due to it deferring all those jobs to now, to stress test
|
|
// this final phase.
|
|
Msg( "QueuedLoader: Finishing %d Jobs\n", (int)g_nJobsToFinishBeforePlay );
|
|
}
|
|
|
|
// We have unexplained GPU hangs, all of them die inside the CLoaderMemAlloc
|
|
// as we are trying to drive the loading progres below.
|
|
m_pProgress->PauseNonInteractiveProgress( true );
|
|
|
|
// clear any possible suspension and prevent any possible suspension from occurring
|
|
g_nForceSuspendIO = 0;
|
|
AdjustAsyncIOSpeed();
|
|
|
|
// finish all outstanding priority jobs
|
|
SubmitPendingJobs();
|
|
|
|
// Upon shutdown we need to abort all jobs that have already been
|
|
// submitted since they can take a very long time to finish
|
|
if ( bShutdownAppRequest )
|
|
{
|
|
// we need to flush IO before aborting thread pool, because IO jobs are constantly spawning compute jobs in g_pThreadPool
|
|
// in case we quit during loading a level
|
|
g_pFullFileSystem->AsyncFlush();
|
|
|
|
g_pThreadPool->AbortAll();
|
|
g_pFullFileSystem->AsyncFlush();
|
|
g_pFullFileSystem->RemoveAllSearchPaths();
|
|
m_pProgress->UpdateProgress( PROGRESS_END, true );
|
|
}
|
|
else
|
|
{
|
|
while ( g_nHighPriorityJobs != 0 || g_nJobsToFinishBeforePlay != 0 )
|
|
{
|
|
// yield some time
|
|
g_pThreadPool->Yield( MAIN_THREAD_YIELD_TIME );
|
|
m_pProgress->UpdateProgress( PROGRESS_END, true );
|
|
}
|
|
}
|
|
|
|
// Restore
|
|
m_pProgress->PauseNonInteractiveProgress( false );
|
|
}
|
|
|
|
m_EndTime = Plat_MSTime();
|
|
m_bActive = false;
|
|
|
|
// transmit the end map event
|
|
for ( int i = RESOURCEPRELOAD_UNKNOWN+1; i < RESOURCEPRELOAD_COUNT; i++ )
|
|
{
|
|
if ( m_pLoaders[i] )
|
|
{
|
|
m_pLoaders[i]->OnEndMapLoading( bAbort );
|
|
}
|
|
}
|
|
|
|
// free any unclaimed anonymous buffers
|
|
int iIndex = m_AnonymousJobs.First();
|
|
while ( iIndex != m_AnonymousJobs.InvalidIndex() )
|
|
{
|
|
FileJob_t *pFileJob = m_AnonymousJobs[iIndex];
|
|
if ( pFileJob->m_bFreeTargetAfterIO && pFileJob->m_pTargetData )
|
|
{
|
|
g_pFullFileSystem->FreeOptimalReadBuffer( pFileJob->m_pTargetData );
|
|
pFileJob->m_pTargetData = NULL;
|
|
}
|
|
g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead;
|
|
iIndex = m_AnonymousJobs.Next( iIndex );
|
|
}
|
|
m_AnonymousJobs.Purge();
|
|
|
|
if ( g_nIOMemory || g_nAnonymousIOMemory )
|
|
{
|
|
// expected to be zero, otherwise logic flaw
|
|
DevWarning( "QueuedLoader: Unclaimed I/O memory: total:%d anonymous:%d\n", ( int ) g_nIOMemory, ( int ) g_nAnonymousIOMemory );
|
|
g_nIOMemory = 0;
|
|
g_nAnonymousIOMemory = 0;
|
|
}
|
|
|
|
// no longer needed
|
|
m_AdditionalResources.RemoveAll();
|
|
}
|
|
|
|
g_pFullFileSystem->EndMapAccess();
|
|
m_bStarted = false;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Returns true if loader is accepting queue requests.
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::IsMapLoading() const
|
|
{
|
|
return m_bActive;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Returns true if loader is working on same map as last load
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::IsSameMapLoading() const
|
|
{
|
|
return m_bActive && m_bSameMap;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Returns true if the loader is idle, indicates all i/o and work has completed.
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::IsFinished() const
|
|
{
|
|
return ( m_bActive == false && g_nActiveJobs == 0 && g_nQueuedJobs == 0 );
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Returns true if loader is batching
|
|
//-----------------------------------------------------------------------------
|
|
bool CQueuedLoader::IsBatching() const
|
|
{
|
|
return m_bBatching;
|
|
}
|
|
|
|
int CQueuedLoader::GetSpewDetail() const
|
|
{
|
|
int spewDetail = loader_spew_info.GetInt();
|
|
if ( spewDetail <= 0 )
|
|
{
|
|
return spewDetail;
|
|
}
|
|
|
|
return 1 << ( spewDetail - 1 );
|
|
}
|
|
|
|
unsigned int CQueuedLoader::GetStartTime()
|
|
{
|
|
return m_StartTime;
|
|
}
|