expvintl/source-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
29985681a1
source-engine/gcsdk/gcjob.cpp
FluorescentCIAAfricanAmerican 3bf9df6b27 1
2020-04-22 12:56:21 -04:00

1591 lines
59 KiB
C++
Raw Blame History

//========= Copyright Valve Corporation, All rights reserved. ============//
#include "stdafx.h"
#include "msgprotobuf.h"
#include "smartptr.h"
#include "rtime.h"
#include "gcsdk/gcreportprinter.h"
#include <winsock.h>
// memdbgon must be the last include file in a .cpp file!!!
#include "tier0/memdbgon.h"
namespace GCSDK
{
GCConVar cv_webapi_result_size_limit( "webapi_result_size_limit", "50000000" );
GCConVar cv_webapi_serialize_threads( "webapi_serialize_threads", "1", 0, "Switch for serializing webAPI responses on threads" );
static GCConVar webapi_account_tracking( "webapi_account_tracking", "1", "Controls whether or not account tracking stats are collected for web api usage" );
static GCConVar webapi_kill_switch( "webapi_kill_switch", "0", "When set to zero this will block no web api calls, when set to 1 this will block all web api except those sent by steam. To block steam, use the webapi_enable_steam_<priority> controls" );
static GCConVar webapi_kill_switch_error_response( "webapi_kill_switch_error_response", "1", "Determines if a response should be sent when the kill switch kills a web api request" );
static GCConVar webapi_rate_limit_calls_per_min( "webapi_rate_limit_calls_per_min", "600", "Determines how many messages can be sent from an account via the web api per minute. <0 disables this limiting" );
static GCConVar webapi_rate_limit_mb_per_min( "webapi_rate_limit_mb_per_min", "20", "Determines how many megabytes of data can be sent to an account via the web api per minute. <0 disables this limiting" );
static GCConVar webapi_elevated_rate_limit_calls_per_min( "webapi_elevated_rate_limit_calls_per_min", "-1", "Determines how many messages can be sent from an account via the web api per minute for elevated accounts. <0 disables this limiting" );
static GCConVar webapi_elevated_rate_limit_mb_per_min( "webapi_elevated_rate_limit_mb_per_min", "-1", "Determines how many megabytes of data can be sent to an account via the web api per minute for elevated accounts. <0 disables this limiting" );
static GCConVar webapi_ip_rate_limit( "webapi_ip_rate_limit", "1", "Controls whether or not we rate limit based upon IPs or just accounts" );
//-----------------------------------------------------------------------------
// CWebAPIAccountTracker
//
// Utility that tracks web api calls based upon accesses made by various users
//-----------------------------------------------------------------------------
class CWebAPIAccountTracker
{
public:
//called when a web api request is made to track the call. A return value of true indicates that it should be allowed,
//false indicates it should be blocked
bool TrackUser( AccountID_t nID, uint32 nIP );
//called once the size of the response is known and will track bandwidth and caller attributed to the provided function
void TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize );
//called to reset all permissions to default
void ResetAccountPermissions();
//called to associate a permission level with an account
void SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel );
//completely resets accumulated stats
void ResetStats();
//resets just the profile stats
void ResetProfileStats();
//different caller report filters that can be used
enum EDumpCaller
{
eDumpCaller_All,
eDumpCaller_Blocked,
eDumpCaller_Status,
eDumpCaller_Calls,
};
//different reports that can be provided
void DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const;
void DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const;
void DumpCaller( AccountID_t nID ) const;
void DumpIP( uint32 nIP ) const;
void DumpFunctions() const;
void DumpProfile( bool bAllTime ) const;
void DumpSteamServers() const;
//given a steam server name, this will return the identifier of that server
uint32 GetSteamServerID( const char* pszServer );
//for steam level requests, we have a priority provided, and to track stats separately we break them up to different account IDs instead of just zero
static const uint32 k_nSteamIP_High = 2;
static const uint32 k_nSteamIP_Normal = 1;
static const uint32 k_nSteamIP_Low = 0;
private:
//called to get the starting time of this rate interval
RTime32 GetRateIntervalStart( AccountID_t nCaller ) const;
struct SCallerStats
{
SCallerStats() :
m_nBlockedCalls( 0 ),
m_eLevel( eWebAPIAccountLevel_RateLimited )
{
ResetRateInterval( 0 );
}
void ResetRateInterval( RTime32 nStart )
{
m_nRateIntervalStartTime = nStart;
m_nRateIntervalCalls = 0;
m_nRateIntervalBytes = 0;
}
//the most recent rate interval that we received a message from the user (used to expire old counts)
RTime32 m_nRateIntervalStartTime;
//how many messages have been sent within this rate interval (used for rate limiting)
uint32 m_nRateIntervalCalls;
//how many bytes have been sent for this account during this interval
uint32 m_nRateIntervalBytes;
//total number of blocked calls
uint32 m_nBlockedCalls;
//flags associated with this caller, used to block/whitelist, etc
EWebAPIAccountLevel m_eLevel;
};
struct SFunctionStats
{
//track the number of calls and bandwidth. The profile versions are separate and used for displaying profiles over a window
uint32 m_nTotalCalls;
uint32 m_nProfileCalls;
uint32 m_nMaxBytes;
uint32 m_nProfileMaxBytes;
uint64 m_nTotalBytes;
uint64 m_nProfileBytes;
//a map of who has called us, which consists of the caller account and IP as the key
struct SCaller
{
bool operator==( const SCaller& rhs ) const { return m_nAccountID == rhs.m_nAccountID && m_nIP == rhs.m_nIP; }
AccountID_t m_nAccountID;
uint32 m_nIP;
};
struct SCalls
{
uint32 m_nCalls;
uint64 m_nBytes;
};
CUtlHashMapLarge< SCaller, SCalls > m_Callers;
};
//a structure used to simplify reporting so a vector can just be built of these, and then provided to the report function which will handle sorting it and displaying
struct SReportRow
{
SReportRow( const char* pszFunction, uint32 nCalls, uint64 nSize ) :
m_pszFunction( pszFunction ),
m_nCalls( nCalls ),
m_nSize( nSize )
{}
const char* m_pszFunction;
uint32 m_nCalls;
uint64 m_nSize;
};
struct SSteamServer
{
CUtlString m_sName;
uint32 m_nID;
};
//called to find an existing user, or create one if not in the list already
SCallerStats* CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart );
SCallerStats* CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart );
//called to print a report of the provided report rows as either an ID list or a function list. This will re-sort the provided vector
static void PrintReport( const CUtlVector< SReportRow >& vec );
//how many seconds are in a rate interval
static const uint32 knRateIntervalTimeS = 60;
CUtlHashMapLarge< AccountID_t, SCallerStats > m_AccountCallers;
CUtlHashMapLarge< uint32, SCallerStats > m_IPCallers;
CUtlHashMapLarge< uintp, SFunctionStats* > m_Functions;
CUtlHashMapLarge< const char*, SSteamServer*, CaseSensitiveStrEquals, MurmurHash3ConstCharPtr > m_SteamServers;
CJobTime m_ProfileTime;
};
//our global profiler
static CWebAPIAccountTracker g_WebAPIAccountTracker;
void WebAPIAccount_ResetAllPermissions()
{
g_WebAPIAccountTracker.ResetAccountPermissions();
}
void WebAPIAccount_SetPermission( AccountID_t nID, EWebAPIAccountLevel eLevel )
{
g_WebAPIAccountTracker.SetAccountPermission( nID, eLevel );
}
bool WebAPIAccount_BTrackUserAndValidate( AccountID_t nID, uint32 unIP )
{
return g_WebAPIAccountTracker.TrackUser( nID, unIP );
}
RTime32 CWebAPIAccountTracker::GetRateIntervalStart( AccountID_t nCaller ) const
{
//we shift the time by the account ID so that all users don't wrap at the same time which can cause a temporary surge in web API requests
RTime32 curTime = CRTime::RTime32TimeCur() + ( nCaller % knRateIntervalTimeS );
return curTime - ( curTime % knRateIntervalTimeS );
}
CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart )
{
int nIndex = m_AccountCallers.Find( nID );
if( nIndex == m_AccountCallers.InvalidIndex() )
{
nIndex = m_AccountCallers.Insert( nID );
SCallerStats& caller = m_AccountCallers[ nIndex ];
caller.m_nRateIntervalStartTime = nRateIntervalStart;
//account ID is always unrestricted!
if( nID == 0 )
caller.m_eLevel = eWebAPIAccountLevel_Unlimited;
}
return &m_AccountCallers[ nIndex ];
}
CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart )
{
int nIndex = m_IPCallers.Find( nIP );
if( nIndex == m_IPCallers.InvalidIndex() )
{
nIndex = m_IPCallers.Insert( nIP );
SCallerStats& caller = m_IPCallers[ nIndex ];
caller.m_nRateIntervalStartTime = nRateIntervalStart;
}
return &m_IPCallers[ nIndex ];
}
uint32 CWebAPIAccountTracker::GetSteamServerID( const char* pszServer )
{
int nIndex = m_SteamServers.Find( pszServer );
if( nIndex == m_SteamServers.InvalidIndex() )
{
SSteamServer* pServer = new SSteamServer;
pServer->m_sName = pszServer;
pServer->m_nID = m_SteamServers.Count();
m_SteamServers.Insert( pServer->m_sName, pServer );
return pServer->m_nID;
}
return m_SteamServers[ nIndex ]->m_nID;
}
void CWebAPIAccountTracker::DumpSteamServers() const
{
CGCReportPrinter rp;
rp.AddStringColumn( "ID" );
rp.AddStringColumn( "Server" );
FOR_EACH_MAP_FAST( m_SteamServers, nServer )
{
const uint32 nID = m_SteamServers[ nServer ]->m_nID;
rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nID << 8 ) ) );
rp.StrValue( m_SteamServers[ nServer ]->m_sName );
rp.CommitRow();
}
rp.SortReport( "ID", false );
rp.PrintReport( SPEW_CONSOLE );
}
//determines what the resulting account level access should be based upon the access rights of the IP address and the account
static EWebAPIAccountLevel DetermineAccessLevel( EWebAPIAccountLevel eAccount, EWebAPIAccountLevel eIP )
{
//unrestricted users should always be allowed, regardless of the IP range that they are making requests from, same with unlimited IP addresses
if( ( eAccount == eWebAPIAccountLevel_Unlimited ) || ( eIP == eWebAPIAccountLevel_Unlimited ) )
return eWebAPIAccountLevel_Unlimited;
//otherwise, if either is blocked, then block
if( ( eAccount == eWebAPIAccountLevel_Blocked ) || ( eIP == eWebAPIAccountLevel_Blocked ) )
return eWebAPIAccountLevel_Blocked;
//now we are dealing with default case versus elevated. Elevated wins over default
if( ( eAccount == eWebAPIAccountLevel_Elevated ) || ( eIP == eWebAPIAccountLevel_Elevated ) )
return eWebAPIAccountLevel_Elevated;
//default
return eWebAPIAccountLevel_RateLimited;
}
bool CWebAPIAccountTracker::TrackUser( AccountID_t nID, uint32 nIP )
{
if( !webapi_account_tracking.GetBool() )
return true;
//first off update their aggregate caller stats
{
//what is our current time, and at what time did this rate interval start
const RTime32 rateIntervalStart = GetRateIntervalStart( nID );
//see if this account is completely blocked
SCallerStats* pAccountCaller = CreateAccountUser( nID, rateIntervalStart );
SCallerStats* pIPCaller = CreateIPUser( nIP, rateIntervalStart );
//determine what our policy should be based upon the access level of the IP and the user
EWebAPIAccountLevel eAccessLevel = DetermineAccessLevel( pAccountCaller->m_eLevel, pIPCaller->m_eLevel );
//if we are blocked, just bail now
if( eAccessLevel == eWebAPIAccountLevel_Blocked )
{
pAccountCaller->m_nBlockedCalls++;
pIPCaller->m_nBlockedCalls++;
return false;
}
//reset the rate interval tracking
if( pAccountCaller->m_nRateIntervalStartTime < rateIntervalStart )
pAccountCaller->ResetRateInterval( rateIntervalStart );
if( pIPCaller->m_nRateIntervalStartTime < rateIntervalStart )
pIPCaller->ResetRateInterval( rateIntervalStart );
//now handle rate limiting
if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) || ( eAccessLevel == eWebAPIAccountLevel_Elevated ) )
{
//determine the rate we want to limit
int32 nCallsPerMin = ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_calls_per_min.GetInt() : webapi_elevated_rate_limit_calls_per_min.GetInt();
int32 nBytesPerMin = ( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_mb_per_min.GetInt() : webapi_elevated_rate_limit_mb_per_min.GetInt() ) * 1024 * 1024;
//see if this account is rate limited
if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) )
{
bool bAllow = true;
//see if we are being limited based upon call rate limiting (tracking based upon ip and account) Note that
//we don't return until we've dones stat tracking for both so the reports are accurate and capture it at both levels
if( ( nCallsPerMin >= 0 && pAccountCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) ||
( nBytesPerMin >= 0 && pAccountCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) )
{
pAccountCaller->m_nBlockedCalls++;
bAllow = false;
}
if( webapi_ip_rate_limit.GetBool() )
{
if( ( nCallsPerMin >= 0 && pIPCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) ||
( nBytesPerMin >= 0 && pIPCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) )
{
pIPCaller->m_nBlockedCalls++;
bAllow = false;
}
}
if( !bAllow )
return false;
}
}
}
return true;
}
void CWebAPIAccountTracker::TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize )
{
if( !webapi_account_tracking.GetBool() )
return;
//update the bytes for that user
{
int nCallerIndex = m_AccountCallers.Find( nID );
if( nCallerIndex != m_AccountCallers.InvalidIndex() )
{
SCallerStats& caller = m_AccountCallers[ nCallerIndex ];
caller.m_nRateIntervalBytes += nResponseSize;
caller.m_nRateIntervalCalls++;
}
}
//update the bytes for that user and for their IP
{
int nCallerIndex = m_IPCallers.Find( nIP );
if( nCallerIndex != m_IPCallers.InvalidIndex() )
{
SCallerStats& caller = m_IPCallers[ nCallerIndex ];
caller.m_nRateIntervalBytes += nResponseSize;
caller.m_nRateIntervalCalls++;
}
}
//now update the function specific stats
{
int nFunctionIndex = m_Functions.Find( ( uintp )pszFunction );
if( nFunctionIndex == m_Functions.InvalidIndex() )
{
SFunctionStats* pNewStats = new SFunctionStats;
pNewStats->m_nTotalCalls = 0;
pNewStats->m_nProfileCalls = 0;
pNewStats->m_nTotalBytes = 0;
pNewStats->m_nProfileBytes = 0;
pNewStats->m_nMaxBytes = 0;
pNewStats->m_nProfileMaxBytes = 0;
nFunctionIndex = m_Functions.Insert( ( uintp )pszFunction, pNewStats );
}
//update our stats
SFunctionStats& function = *m_Functions[ nFunctionIndex ];
function.m_nTotalCalls++;
function.m_nProfileCalls++;
function.m_nTotalBytes += nResponseSize;
function.m_nProfileBytes += nResponseSize;
function.m_nMaxBytes = MAX( function.m_nMaxBytes, nResponseSize );
function.m_nProfileMaxBytes = MAX( function.m_nProfileMaxBytes, nResponseSize );
//update caller stats
{
struct SFunctionStats::SCaller caller;
caller.m_nAccountID = nID;
caller.m_nIP = nIP;
int nCallerIndex = function.m_Callers.Find( caller );
if( nCallerIndex == function.m_Callers.InvalidIndex() )
{
nCallerIndex = function.m_Callers.Insert( caller );
function.m_Callers[ nCallerIndex ].m_nCalls = 1;
function.m_Callers[ nCallerIndex ].m_nBytes = nResponseSize;
}
else
{
function.m_Callers[ nCallerIndex ].m_nCalls++;
function.m_Callers[ nCallerIndex ].m_nBytes += nResponseSize;
}
}
}
}
void CWebAPIAccountTracker::SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel )
{
SCallerStats* pCaller = CreateAccountUser( nID, GetRateIntervalStart( nID ) );
pCaller->m_eLevel = eLevel;
}
void CWebAPIAccountTracker::ResetAccountPermissions()
{
FOR_EACH_MAP_FAST( m_AccountCallers, nCaller )
{
m_AccountCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited;
}
FOR_EACH_MAP_FAST( m_IPCallers, nCaller )
{
m_IPCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited;
}
}
void CWebAPIAccountTracker::ResetStats()
{
FOR_EACH_MAP_FAST( m_AccountCallers, nCaller )
{
m_AccountCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_AccountCallers.Key( nCaller ) ) );
m_AccountCallers[ nCaller ].m_nBlockedCalls = 0;
}
FOR_EACH_MAP_FAST( m_IPCallers, nCaller )
{
m_IPCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_IPCallers.Key( nCaller ) ) );
m_IPCallers[ nCaller ].m_nBlockedCalls = 0;
}
m_Functions.PurgeAndDeleteElements();
}
void CWebAPIAccountTracker::ResetProfileStats()
{
FOR_EACH_MAP_FAST( m_Functions, nFunction )
{
m_Functions[ nFunction ]->m_nProfileCalls = 0;
m_Functions[ nFunction ]->m_nProfileBytes = 0;
m_Functions[ nFunction ]->m_nProfileMaxBytes = 0;
}
m_ProfileTime.SetToJobTime();
}
static const int k_cSteamIDRenderedMaxLen = 36;
//-----------------------------------------------------------------------------
// Purpose: Renders the steam ID to a buffer with an admin console link. NOTE: for convenience of
// calling code, this code returns a pointer to a static buffer and is NOT thread-safe.
// Output: buffer with rendered Steam ID
//-----------------------------------------------------------------------------
static const char * CSteamID_RenderLink( const CSteamID & steamID )
{
// longest length of returned string is k_cBufLen
// <link cmd="steamid64 %llu"></link> => 30 + 20 == 50
// 50 + k_cSteamIDRenderedMaxLen + 1
const int k_cBufLen = 50 + k_cSteamIDRenderedMaxLen + 1;
const int k_cBufs = 4; // # of static bufs to use (so people can compose output with multiple calls to RenderLink() )
static char rgchBuf[k_cBufs][k_cBufLen];
static int nBuf = 0;
char * pchBuf = rgchBuf[nBuf]; // get pointer to current static buf
nBuf++; // use next buffer for next call to this method
nBuf %= k_cBufs;
Q_snprintf( pchBuf, k_cBufLen, "<link cmd=\"steamid64 %llu\">%s</link>", steamID.ConvertToUint64(), steamID.Render() );
return pchBuf;
}
//-----------------------------------------------------------------------------
// Purpose: Renders the passed-in steam ID to a buffer with admin console link. NOTE: for convenience
// of calling code, this code returns a pointer to a static buffer and is NOT thread-safe.
// Input: 64-bit representation of Steam ID to render
// Output: buffer with rendered Steam ID link
//-----------------------------------------------------------------------------
static const char * CSteamID_RenderLink( uint64 ulSteamID )
{
CSteamID steamID( ulSteamID );
return CSteamID_RenderLink( steamID );
}
void CWebAPIAccountTracker::DumpCaller( AccountID_t nID ) const
{
const CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( nID );
//cache the account name here so we don't yield while we have indices
CUtlString sPersona = GGCBase()->YieldingGetPersonaName( steamID, "[Unknown]" );
//dump high level user stats
int nCallerIndex = m_AccountCallers.Find( nID );
if( nCallerIndex == m_AccountCallers.InvalidIndex() )
{
EG_MSG( SPEW_CONSOLE, "User %u not found in any web api calls\n", nID );
return;
}
//a map of IP addresses that have been used by this account
CUtlHashMapLarge< uint32, SFunctionStats::SCalls > ipCalls;
//now each function they called
uint64 nTotalBytes = 0;
uint32 nTotalCalls = 0;
CUtlVector< SReportRow > vFuncs;
FOR_EACH_MAP_FAST( m_Functions, nFunc )
{
//add up how many calls they made to this function across all IPs
uint64 nFnBytes = 0;
uint32 nFnCalls = 0;
FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller )
{
const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller );
if( caller.m_nAccountID == nID )
{
const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ];
nFnBytes += calls.m_nBytes;
nFnCalls += calls.m_nCalls;
int nIPIndex = ipCalls.Find( caller.m_nIP );
if( nIPIndex == ipCalls.InvalidIndex() )
{
SFunctionStats::SCalls toAdd;
toAdd.m_nBytes = calls.m_nBytes;
toAdd.m_nCalls = calls.m_nCalls;
ipCalls.Insert( caller.m_nIP, toAdd );
}
else
{
ipCalls[ nIPIndex ].m_nBytes += calls.m_nBytes;
ipCalls[ nIPIndex ].m_nBytes += calls.m_nCalls;
}
}
}
if( nFnCalls > 0 )
{
vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) );
}
nTotalBytes += nFnBytes;
nTotalCalls += nFnCalls;
}
const SCallerStats& caller = m_AccountCallers[ nCallerIndex ];
EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" );
EG_MSG( SPEW_CONSOLE, "User %s: \"%s\"\n", CSteamID_RenderLink( steamID ), sPersona.String() );
double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 );
double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 );
double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 );
EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour );
//don't let someone accidentally change Steam's access!
if( nID != 0 )
{
if( caller.m_eLevel == eWebAPIAccountLevel_RateLimited )
{
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Block Account]</link>", nID, eWebAPIAccountLevel_Blocked );
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Elevate Account]</link>\n", nID, eWebAPIAccountLevel_Elevated );
}
else if( caller.m_eLevel == eWebAPIAccountLevel_Blocked )
{
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Unblock Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited );
}
else if( caller.m_eLevel == eWebAPIAccountLevel_Elevated )
{
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Demote Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited );
}
}
//print a report of the IP addresses that they are calling from
{
CGCReportPrinter rp;
rp.AddStringColumn( "IP" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
FOR_EACH_MAP_FAST( ipCalls, nIP )
{
rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( ipCalls.Key( nIP ) ) ), CFmtStr( "webapi_account_dump_ip %u", ipCalls.Key( nIP ) ) );
rp.IntValue( ipCalls[ nIP ].m_nCalls );
rp.IntValue( ipCalls[ nIP ].m_nBytes );
rp.CommitRow();
}
rp.SortReport( "MB" );
rp.PrintReport( SPEW_CONSOLE );
}
//and print a report of all the functions that they've called
PrintReport( vFuncs );
}
void CWebAPIAccountTracker::DumpIP( uint32 nIP ) const
{
//dump high level user stats
int nCallerIndex = m_IPCallers.Find( nIP );
if( nCallerIndex == m_IPCallers.InvalidIndex() )
{
EG_MSG( SPEW_CONSOLE, "IP %u not found in any web api calls\n", nIP );
return;
}
//a map of IP addresses that have been used by this account
CUtlHashMapLarge< AccountID_t, SFunctionStats::SCalls > accountCalls;
//now each function they called
uint64 nTotalBytes = 0;
uint32 nTotalCalls = 0;
CUtlVector< SReportRow > vFuncs;
FOR_EACH_MAP_FAST( m_Functions, nFunc )
{
//add up how many calls they made to this function across all IPs
uint64 nFnBytes = 0;
uint32 nFnCalls = 0;
FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller )
{
const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller );
if( caller.m_nIP == nIP )
{
const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ];
nFnBytes += calls.m_nBytes;
nFnCalls += calls.m_nCalls;
int nAccountIndex = accountCalls.Find( caller.m_nAccountID );
if( nAccountIndex == accountCalls.InvalidIndex() )
{
SFunctionStats::SCalls toAdd;
toAdd.m_nBytes = calls.m_nBytes;
toAdd.m_nCalls = calls.m_nCalls;
accountCalls.Insert( caller.m_nAccountID, toAdd );
GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( caller.m_nAccountID ) );
}
else
{
accountCalls[ nAccountIndex ].m_nBytes += calls.m_nBytes;
accountCalls[ nAccountIndex ].m_nBytes += calls.m_nCalls;
}
}
}
if( nFnCalls > 0 )
{
vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) );
}
nTotalBytes += nFnBytes;
nTotalCalls += nFnCalls;
}
const SCallerStats& caller = m_IPCallers[ nCallerIndex ];
EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" );
EG_MSG( SPEW_CONSOLE, "IP %u.%u.%u.%u\n", iptod( nIP ) );
double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 );
double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 );
double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 );
EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour );
//print a report of the accounts that they are calling from
{
CGCReportPrinter rp;
rp.AddSteamIDColumn( "Account" );
rp.AddStringColumn( "Persona" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
FOR_EACH_MAP_FAST( accountCalls, nAccount )
{
CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( accountCalls.Key( nAccount ) );
rp.SteamIDValue( steamID, CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) );
rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ), CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) );
rp.IntValue( accountCalls[ nAccount ].m_nCalls );
rp.IntValue( accountCalls[ nAccount ].m_nBytes );
rp.CommitRow();
}
rp.SortReport( "MB" );
rp.PrintReport( SPEW_CONSOLE );
}
//and print a report of all the functions that they've called
PrintReport( vFuncs );
}
struct SCallerReportStats
{
uint32 m_nFunctions;
uint32 m_nCalls;
uint64 m_nBytes;
};
void CWebAPIAccountTracker::DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter ) const
{
//accumulate stats for each unique caller
CUtlHashMapLarge< AccountID_t, SCallerReportStats > mapCallers;
FOR_EACH_MAP_FAST( m_Functions, nCurrFunction )
{
//handle filtering out functions we don't care about
const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction );
if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) )
continue;
const SFunctionStats& function = *m_Functions[ nCurrFunction ];
FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller )
{
const AccountID_t key = function.m_Callers.Key( nCurrCaller ).m_nAccountID;
//add this account
int nStatIndex = mapCallers.Find( key );
if( nStatIndex == mapCallers.InvalidIndex() )
{
SCallerReportStats stats;
stats.m_nFunctions = 0;
stats.m_nCalls = 0;
stats.m_nBytes = 0;
nStatIndex = mapCallers.Insert( key, stats );
GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( key ) );
}
mapCallers[ nStatIndex ].m_nFunctions += 1;
mapCallers[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls;
mapCallers[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes;
}
}
CGCReportPrinter rp;
rp.AddStringColumn( "Account" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None );
rp.AddStringColumn( "UserName" );
const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 );
//now show the report
FOR_EACH_MAP_FAST( m_AccountCallers, nCurrCaller )
{
//apply filters to our results
if( ( eFilter == eDumpCaller_Blocked ) && ( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) )
continue;
if( ( eFilter == eDumpCaller_Status ) && ( m_AccountCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) )
continue;
const AccountID_t accountID = m_AccountCallers.Key( nCurrCaller );
const SCallerReportStats* pStats = NULL;
int nCollectedStats = mapCallers.Find( accountID );
if( nCollectedStats != mapCallers.InvalidIndex() )
{
pStats = &mapCallers[ nCollectedStats ];
}
//filter out users that didn't make any calls if appropriate
if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) )
continue;
const CSteamID steamID( GGCInterface()->ConstructSteamIDForClient( accountID ) );
rp.StrValue( steamID.Render() , CFmtStr( "webapi_account_dump_caller %u", accountID ) );
rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 );
rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 );
rp.IntValue( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls );
rp.IntValue( m_AccountCallers[ nCurrCaller ].m_eLevel );
rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 );
rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ) );
rp.CommitRow();
}
const char* pszSort = "MB/h";
if( eFilter == eDumpCaller_Blocked )
pszSort = "Blocked";
else if( eFilter == eDumpCaller_Status )
pszSort = "Access";
rp.SortReport( pszSort );
rp.PrintReport( SPEW_CONSOLE );
}
void CWebAPIAccountTracker::DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter ) const
{
//accumulate stats for each unique caller
CUtlHashMapLarge< uint32, SCallerReportStats > mapIPs;
FOR_EACH_MAP_FAST( m_Functions, nCurrFunction )
{
//handle filtering out functions we don't care about
const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction );
if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) )
continue;
const SFunctionStats& function = *m_Functions[ nCurrFunction ];
FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller )
{
const uint32 key = function.m_Callers.Key( nCurrCaller ).m_nIP;
//add this account
int nStatIndex = mapIPs.Find( key );
if( nStatIndex == mapIPs.InvalidIndex() )
{
SCallerReportStats stats;
stats.m_nFunctions = 0;
stats.m_nCalls = 0;
stats.m_nBytes = 0;
nStatIndex = mapIPs.Insert( key, stats );
}
mapIPs[ nStatIndex ].m_nFunctions += 1;
mapIPs[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls;
mapIPs[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes;
}
}
CGCReportPrinter rp;
rp.AddStringColumn( "IP" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None );
const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 );
//now show the report
FOR_EACH_MAP_FAST( m_IPCallers, nCurrCaller )
{
//apply filters to our results
if( ( eFilter == eDumpCaller_Blocked ) && ( m_IPCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) )
continue;
if( ( eFilter == eDumpCaller_Status ) && ( m_IPCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) )
continue;
const uint32 nIP = m_IPCallers.Key( nCurrCaller );
const SCallerReportStats* pStats = NULL;
int nCollectedStats = mapIPs.Find( nIP );
if( nCollectedStats != mapIPs.InvalidIndex() )
{
pStats = &mapIPs[ nCollectedStats ];
}
//filter out users that didn't make any calls if appropriate
if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) )
continue;
rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nIP ) ), CFmtStr( "webapi_account_dump_ip %u", nIP ) );
rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 );
rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 );
rp.IntValue( m_IPCallers[ nCurrCaller ].m_nBlockedCalls );
rp.IntValue( m_IPCallers[ nCurrCaller ].m_eLevel );
rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 );
rp.CommitRow();
}
const char* pszSort = "MB/h";
if( eFilter == eDumpCaller_Blocked )
pszSort = "Blocked";
else if( eFilter == eDumpCaller_Status )
pszSort = "Access";
rp.SortReport( pszSort );
rp.PrintReport( SPEW_CONSOLE );
}
void CWebAPIAccountTracker::DumpFunctions() const
{
CUtlVector< SReportRow > vFuncs;
vFuncs.EnsureCapacity( m_Functions.Count() );
FOR_EACH_MAP_FAST( m_Functions, i )
{
vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( i ), m_Functions[ i ]->m_nTotalCalls, m_Functions[ i ]->m_nTotalBytes ) );
}
PrintReport( vFuncs );
}
void CWebAPIAccountTracker::DumpProfile( bool bAllTime ) const
{
//accumulate totals so we can do percentage
uint32 nTotalCalls = 0;
uint64 nTotalBytes = 0;
FOR_EACH_MAP_FAST( m_Functions, nFunction )
{
if( bAllTime )
{
nTotalCalls += m_Functions[ nFunction ]->m_nTotalCalls;
nTotalBytes += m_Functions[ nFunction ]->m_nTotalBytes;
}
else
{
nTotalCalls += m_Functions[ nFunction ]->m_nProfileCalls;
nTotalBytes += m_Functions[ nFunction ]->m_nProfileBytes;
}
}
//determine how much time we are covering, and come up with a scale to normalize the values
uint64 nSampleMicroS = ( bAllTime ) ? ( uint64 )GGCBase()->GetGCUpTime() * 1000000 : m_ProfileTime.CServerMicroSecsPassed();
double fToPS = ( nSampleMicroS > 0 ) ? 1000000.0 / ( double )nSampleMicroS : 1.0;
EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Web API Profile: Sampled %.2f seconds\n", nSampleMicroS / ( 1000.0 * 1000.0 ) );
CGCReportPrinter rp;
rp.AddStringColumn( "Web API Name" );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 );
rp.AddFloatColumn( "KBPS", CGCReportPrinter::eSummary_Total, 1 );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 );
rp.AddFloatColumn( "CallsPS", CGCReportPrinter::eSummary_Total, 1 );
rp.AddIntColumn( "MaxKB", CGCReportPrinter::eSummary_Max );
FOR_EACH_MAP_FAST( m_Functions, nFunction )
{
const SFunctionStats* pFunc = m_Functions[ nFunction ];
uint32 nCalls = ( bAllTime ) ? pFunc->m_nTotalCalls : pFunc->m_nProfileCalls;
uint32 nMax = ( bAllTime ) ? pFunc->m_nMaxBytes : pFunc->m_nProfileMaxBytes;
uint64 nBytes = ( bAllTime ) ? pFunc->m_nTotalBytes : pFunc->m_nProfileBytes;
rp.StrValue( ( const char* )m_Functions.Key( nFunction ) );
rp.IntValue( nBytes );
rp.FloatValue( ( 100.0 * nBytes ) / nTotalBytes );
rp.FloatValue( ( nBytes / 1024.0 ) * fToPS );
rp.IntValue( nCalls );
rp.FloatValue( ( 100.0 * nCalls ) / nTotalCalls );
rp.FloatValue( nCalls * fToPS );
rp.IntValue( nMax / 1024 );
rp.CommitRow();
}
rp.SortReport( "KBPS" );
rp.PrintReport( SPEW_CONSOLE );
}
void CWebAPIAccountTracker::PrintReport( const CUtlVector< SReportRow >& vec )
{
CGCReportPrinter rp;
//now print it out based upon the type
rp.AddStringColumn( "Function" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddFloatColumn( "Calls/h", CGCReportPrinter::eSummary_Total, 0 );
rp.AddFloatColumn( "MB/h", CGCReportPrinter::eSummary_Total );
const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 );
for( int i = 0; i < vec.Count(); i++ )
{
rp.StrValue( vec[ i ].m_pszFunction, CFmtStr( "webapi_account_dump_function_callers %s", vec[ i ].m_pszFunction ) );
rp.IntValue( vec[ i ].m_nCalls );
rp.IntValue( vec[ i ].m_nSize );
rp.FloatValue( vec[ i ].m_nCalls / fUpTimeHrs );
rp.FloatValue( ( vec[ i ].m_nSize / ( 1024.0 * 1024.0 ) ) / fUpTimeHrs );
rp.CommitRow();
}
rp.SortReport( "MB/h" );
rp.PrintReport( SPEW_CONSOLE );
}
GC_CON_COMMAND( webapi_account_dump_steam_servers, "Dumps the ID listings of the various steam servers encoded in the IP address of Steam requests" )
{
g_WebAPIAccountTracker.DumpSteamServers();
}
GC_CON_COMMAND( webapi_account_dump_callers, "Dumps the most frequent callers of web api's for the current run of the GC" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls );
}
GC_CON_COMMAND( webapi_account_dump_ips, "Dumps the most frequent ip callers of web api's for the current run of the GC" )
{
g_WebAPIAccountTracker.DumpTotalIPs( CWebAPIAccountTracker::eDumpCaller_Calls );
}
GC_CON_COMMAND( webapi_account_dump_blocked_callers, "Dumps the callers that have been blocked and how many calls have been blocked" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Blocked );
}
GC_CON_COMMAND( webapi_account_dump_caller_access, "Dumps the access rights of any caller that is not the default rate limiting" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Status );
}
GC_CON_COMMAND( webapi_account_dump_functions, "Dumps the most frequently called web api functions" )
{
g_WebAPIAccountTracker.DumpFunctions();
}
GC_CON_COMMAND_PARAMS( webapi_account_dump_function_callers, 1, "<function name> - Dumps the most frequent callers of functions that match the provided substring" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls, args[ 1 ] );
}
GC_CON_COMMAND_PARAMS( webapi_account_dump_caller, 1, "<caller account> - Dumps the functions that the provided account ID has been calling the most" )
{
g_WebAPIAccountTracker.DumpCaller( ( AccountID_t )V_atoui64( args[ 1 ] ) );
}
GC_CON_COMMAND_PARAMS( webapi_account_dump_ip, 1, "<ip> - Dumps the functions that the provided ip has been calling the most" )
{
g_WebAPIAccountTracker.DumpIP( ( AccountID_t )V_atoui64( args[ 1 ] ) );
}
GC_CON_COMMAND( webapi_account_reset_stats, "Forces a reset of all stats collected for web api account stats" )
{
g_WebAPIAccountTracker.ResetStats();
}
//utility class for dumping out the profile results after time has expired
static void DumpWebAPIProfile()
{
g_WebAPIAccountTracker.DumpProfile( false );
}
GC_CON_COMMAND_PARAMS( webapi_profile, 1, "<seconds to profile> Turns on web api profiling for N seconds and dumps the results" )
{
float fSeconds = MAX( 1.0f, atof( args[ 1 ] ) );
g_WebAPIAccountTracker.ResetProfileStats();
static CGlobalScheduledFunction s_DumpProfile;
s_DumpProfile.ScheduleMS( DumpWebAPIProfile, fSeconds * 1000.0f );
}
//console commands to control web API profiling
GC_CON_COMMAND( webapi_profile_reset, "Turns on web api profiling" )
{
g_WebAPIAccountTracker.ResetProfileStats();
}
GC_CON_COMMAND( webapi_profile_dump, "Displays stats collected while web api profiling was enabled" )
{
g_WebAPIAccountTracker.DumpProfile( false );
}
GC_CON_COMMAND( webapi_profile_dump_total, "Displays stats collected while web api profiling was enabled" )
{
g_WebAPIAccountTracker.DumpProfile( true );
}
//-----------------------------------------------------------------------------
// Purpose: Sends a message and waits for a response
// Input: steamIDTarget - The entity this message is going to
// msgOut - The message to send
// nTimeoutSec - Number of seconds to wait for a response
// pMsgIn - Pointer to the message that will contain the response
// eMsg - The type of message the response should be
// Returns: True is the response was received, false otherwise. The contents
// of pMsgIn will be valid only if the function returns true.
//-----------------------------------------------------------------------------
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, CGCMsgBase *pMsgIn, MsgType_t eMsg )
{
IMsgNetPacket *pNetPacket = NULL;
if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) )
return false;
pMsgIn->SetPacket( pNetPacket );
if ( pMsgIn->Hdr().m_eMsg != eMsg )
return false;
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Sends a message and waits for a response
// Input: steamIDTarget - The entity this message is going to
// msgOut - The message to send
// nTimeoutSec - Number of seconds to wait for a response
// ppNetPackets - Pointer to a IMsgNetPacket pointer which will contain
// the response
// Returns: True is the response was received, false otherwise. *ppNetPacket
// will point to a valid packet only if the function returns true.
//-----------------------------------------------------------------------------
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket )
{
msgOut.ExpectingReply( GetJobID() );
if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) )
return false;
SetJobTimeout( nTimeoutSec );
return BYieldingWaitForMsg( ppNetPacket );
}
//-----------------------------------------------------------------------------
// Purpose: BYldSendMessageAndGetReply, ProtoBuf edition
//-----------------------------------------------------------------------------
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg )
{
IMsgNetPacket *pNetPacket = NULL;
if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) )
return false;
pMsgIn->InitFromPacket( pNetPacket );
if ( pMsgIn->GetEMsg() != eMsg )
return false;
return true;
}
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket )
{
msgOut.ExpectingReply( GetJobID() );
if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) )
return false;
SetJobTimeout( nTimeoutSec );
return BYieldingWaitForMsg( ppNetPacket );
}
//-----------------------------------------------------------------------------
// Purpose: Constructor
//-----------------------------------------------------------------------------
CGCWGJob::CGCWGJob( CGCBase *pGCBase )
: CGCJob( pGCBase ),
m_steamID( k_steamIDNil ),
m_pWebApiFunc( NULL )
{
}
//-----------------------------------------------------------------------------
// Purpose: Destructor
//-----------------------------------------------------------------------------
CGCWGJob::~CGCWGJob()
{
}
//-----------------------------------------------------------------------------
// Purpose: Receive a k_EMsgWGRequest and manage keyvalues serialization/deserialization
//-----------------------------------------------------------------------------
bool CGCWGJob::BYieldingRunGCJob( IMsgNetPacket * pNetPacket )
{
CGCMsg<MsgGCWGRequest_t> msg( pNetPacket );
CUtlString strRequestName;
KeyValuesAD pkvRequest( "request" );
KeyValuesAD pkvResponse( "response" );
m_steamID = CSteamID( msg.Body().m_ulSteamID );
msg.BReadStr( &strRequestName );
//deserialize KV
m_bufRequest.Clear();
m_bufRequest.Put( msg.PubReadCur(), msg.Body().m_cubKeyValues );
KVPacker packer;
if ( !packer.ReadAsBinary( pkvRequest, m_bufRequest ) )
{
AssertMsg( false, "Failed to deserialize key values from WG request" );
CGCWGJobMgr::SendErrorMessage( msg, "Failed to deserialize key values from WG request", k_EResultInvalidParam );
return false;
}
if( !BVerifyParams( msg, pkvRequest, m_pWebApiFunc ) )
return false;
bool bRet = BYieldingRunJobFromRequest( pkvRequest, pkvResponse );
// request failed, set success for wg
if ( pkvResponse->IsEmpty( "success" ) )
{
pkvResponse->SetInt( "success", bRet ? k_EResultOK : k_EResultFail );
}
// send response msg
CGCWGJobMgr::SendResponse( msg, pkvResponse, bRet );
return true;
}
bool CGCWGJob::BVerifyParams( const CGCMsg<MsgGCWGRequest_t> & msg, KeyValues *pkvRequest, const WebApiFunc_t * pWebApiFunc )
{
// we've found the function; now validate the call
for ( int i = 0; i < Q_ARRAYSIZE( pWebApiFunc->m_rgParams ); i++ )
{
if ( !pWebApiFunc->m_rgParams[i].m_pchParam )
break;
// just simple validation for now; make sure the key exists
if ( !pWebApiFunc->m_rgParams[i].m_bOptional && !pkvRequest->FindKey( pWebApiFunc->m_rgParams[i].m_pchParam ) )
{
CGCWGJobMgr::SendErrorMessage( msg,
CFmtStr( "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName ),
k_EResultInvalidParam );
EmitWarning( SPEW_GC, 2, "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName );
return false;
}
}
return true;
}
void CGCWGJob::SetErrorMessage( KeyValues *pkvErr, const char *pchErrorMsg, int32 nResult )
{
CGCWGJobMgr::SetErrorMessage( pkvErr, pchErrorMsg, nResult );
}
//-----------------------------------------------------------------------------
// CGCJobVerifySession - A job that asks steam if a given user is still connected
// and cleans up the session if the user is gone
//-----------------------------------------------------------------------------
bool CGCJobVerifySession::BYieldingRunGCJob()
{
if ( !m_pGC->BYieldingLockSteamID( m_steamID, __FILE__, __LINE__ ) )
return false;
m_pGC->YieldingRequestSession( m_steamID );
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Constructor
//-----------------------------------------------------------------------------
CWebAPIJob::CWebAPIJob( CGCBase *pGC, EWebAPIOutputFormat eDefaultOutputFormat )
: CGCJob( pGC ), m_eDefaultOutputFormat( eDefaultOutputFormat )
{
}
//-----------------------------------------------------------------------------
// Purpose: Destructor
//-----------------------------------------------------------------------------
CWebAPIJob::~CWebAPIJob()
{
}
//-----------------------------------------------------------------------------
// Called to handle converting a web api response to a serialized result and free the object as well on a background thread
class CEmitWebAPIData
{
public:
CEmitWebAPIData( CMsgHttpRequest* pRequest) : m_bResult( false ), m_Request( pRequest ) {}
//inputs
CHTTPRequest m_Request;
//outputs
CHTTPResponse m_Response;
std::string m_sSerializedResponse;
bool m_bResult;
};
//the worker thread function that is responsible for serializing the response from web api values to a text buffer, freeing the web api value tree, and serializing the message to a protobuf for
//direct sending. If this function fails (a false response value) the message will not be serialized
static void ThreadedEmitFormattedOutputWrapperAndFreeResponse( CWebAPIResponse *pResponse, CEmitWebAPIData* pEmitData, EWebAPIOutputFormat eDefaultFormat, size_t unMaxResultSize )
{
// parse the output type that we want the result to be in
EWebAPIOutputFormat eOutputFormat = eDefaultFormat;
const char *pszParamOutput = pEmitData->m_Request.GetGETParamString( "format", NULL );
if ( !pszParamOutput )
pszParamOutput = pEmitData->m_Request.GetPOSTParamString( "format", NULL );
if ( pszParamOutput )
{
if ( Q_stricmp( pszParamOutput, "xml" ) == 0 )
eOutputFormat = k_EWebAPIOutputFormat_XML;
else if ( Q_stricmp( pszParamOutput, "vdf" ) == 0 )
eOutputFormat = k_EWebAPIOutputFormat_VDF;
else if ( Q_stricmp( pszParamOutput, "json" ) == 0 )
eOutputFormat = k_EWebAPIOutputFormat_JSON;
}
pEmitData->m_bResult = pResponse->BEmitFormattedOutput( eOutputFormat, *( pEmitData->m_Response.GetBodyBuffer() ), unMaxResultSize );
delete pResponse;
//update the response code on the output (can probably be done elsewhere), but must be done before we pack the message below
switch( eOutputFormat )
{
case k_EWebAPIOutputFormat_JSON:
pEmitData->m_Response.SetResponseHeaderValue( "content-type", "application/json; charset=UTF-8" );
break;
case k_EWebAPIOutputFormat_XML:
pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/xml; charset=UTF-8" );
break;
case k_EWebAPIOutputFormat_VDF:
pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/vdf; charset=UTF-8" );
break;
default:
break;
}
//if successful, we can go ahead and convert this all the way into a completely serialized form for sending over the wire
if( pEmitData->m_bResult )
{
CMsgHttpResponse msgResponse;
pEmitData->m_Response.SerializeIntoProtoBuf( msgResponse );
msgResponse.SerializeToString( &pEmitData->m_sSerializedResponse );
}
}
//called to respond to a web api request with the specified response value
static void WebAPIRespondToRequest( const char* pszName, uint32 nSenderIP, const CHTTPResponse& response, const CProtoBufMsg< CMsgWebAPIRequest >& msg )
{
VPROF_BUDGET( "WebAPI - sending result", VPROF_BUDGETGROUP_STEAM );
CProtoBufMsg<CMsgHttpResponse> msgResponse( k_EGCMsgWebAPIJobRequestHttpResponse, msg );
response.SerializeIntoProtoBuf( msgResponse.Body() );
GGCBase()->BReplyToMessage( msgResponse, msg );
//track this message in the web API response
g_WebAPIAccountTracker.TrackFunction( msg.Body().api_key().account_id(), nSenderIP, pszName, msgResponse.Body().body().size() );
}
//responses to the web api message with an error code
static void WebAPIRespondWithError( const char* pszName, uint32 nSenderIP, const CProtoBufMsg< CMsgWebAPIRequest >& msg, EHTTPStatusCode statusCode )
{
CHTTPResponse response;
response.SetStatusCode( statusCode );
WebAPIRespondToRequest( pszName, nSenderIP, response, msg );
}
//parses an IP address out of the provided string. This will be the last IP address in the list
static uint32 ParseIPAddrFromForwardHeader( const char* pszHeader )
{
//find the last comma in the string, our IP address follows that
const char* pszStart = V_strrchr( pszHeader, ',' );
//if no match, then we just have the ip address, otherwise advance past the comma
if( !pszStart )
pszStart = pszHeader;
else
pszStart++;
//skip leading spaces
while( V_isspace( *pszStart ) )
pszStart++;
return ntohl( inet_addr( pszStart ) );
}
//-----------------------------------------------------------------------------
// Purpose: Receive a k_EMsgWebAPIJobRequest and manage serialization/deserialization to
// web request/response objects
//-----------------------------------------------------------------------------
bool CWebAPIJob::BYieldingRunJobFromMsg( IMsgNetPacket * pNetPacket )
{
VPROF_BUDGET( "WebAPI", VPROF_BUDGETGROUP_STEAM );
CProtoBufMsg<CMsgWebAPIRequest> msg( pNetPacket );
//make sure all the required parameters were present
bool bMsgParsedOK = msg.Body().has_api_key()
&& msg.Body().has_interface_name()
&& msg.Body().has_method_name()
&& msg.Body().has_request()
&& msg.Body().has_version();
if( !bMsgParsedOK )
{
WebAPIRespondWithError( GetName(), 0, msg, k_EHTTPStatusCode400BadRequest );
return true;
}
//determine the account that sent this request
const AccountID_t nSenderAccountID = msg.Body().api_key().account_id();
uint32 nSenderIP = 0;
//if this isn't a system request, try and identify the IP address of the sender so we can rate limit accordingly
if( nSenderAccountID != 0 )
{
const int nNumHeaders = msg.Body().request().headers_size();
for( int nHeader = 0; nHeader < nNumHeaders; nHeader++ )
{
if( strcmp( msg.Body().request().headers( nHeader ).name().c_str(), "X-Forwarded-For" ) != 0 )
continue;
//this is our IP address
nSenderIP = ParseIPAddrFromForwardHeader( msg.Body().request().headers( nHeader ).value().c_str() );
}
//see if we have the kill switch turned on
if( webapi_kill_switch.GetBool() )
{
if( webapi_kill_switch_error_response.GetBool() )
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode503ServiceUnavailable );
return true;
}
}
else
{
// !FIXME! DOTAMERGE
// //determine the priority of this steam request
// uint32 nPriority;
// nSenderIP = GetSteamRequestIPAddress( msg.Body().request(), nPriority );
// //and allow for a kill switch based upon this priority
// if( ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Low ) && !webapi_enable_steam_low.GetBool() ) ||
// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Normal ) && !webapi_enable_steam_normal.GetBool() ) ||
// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_High ) && !webapi_enable_steam_high.GetBool() ) )
// {
// if( webapi_kill_switch_error_response.GetBool() )
// WebAPIRespondWithError( GetName(), 0, msg );
// return true;
// }
}
//track stats for this account, and handle rate limiting
if( !g_WebAPIAccountTracker.TrackUser( nSenderAccountID, nSenderIP ) )
{
if( webapi_kill_switch_error_response.GetBool() )
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode429TooManyRequests );
return true;
}
//allocate the data that we'll use to fill out the request and send it to the background thread for work
CPlainAutoPtr< CEmitWebAPIData > pEmitData( new CEmitWebAPIData( const_cast< CMsgHttpRequest* >( &msg.Body().request() ) ) );
CHTTPRequest& request = pEmitData->m_Request;
CHTTPResponse& response = pEmitData->m_Response;
{
VPROF_BUDGET( "WebAPI - Prepare msg", VPROF_BUDGETGROUP_STEAM );
m_webAPIKey.DeserializeFromProtoBuf( msg.Body().api_key() );
}
CPlainAutoPtr< CWebAPIResponse > pwebAPIResponse( new CWebAPIResponse() );
{
VPROF_BUDGET( "WebAPI - Process msg", VPROF_BUDGETGROUP_STEAM );
if( !BYieldingRunJobFromAPIRequest( msg.Body().interface_name().c_str(), msg.Body().method_name().c_str(), msg.Body().version(), &request, &response, pwebAPIResponse.Get() ) )
{
//error executing our job
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError );
return false;
}
}
// !FIXME! DOTAMERGE
// //see if they want to re-route this request
// if( m_nRerouteRequest >= 0 )
// {
// if( ( uint32 )m_nRerouteRequest == m_pGC->GetGCDirIndex() )
// {
// AssertMsg( false, "Error: WebAPI %s attempting to re-route a web api message to itself (%d)", GetName(), m_nRerouteRequest );
// }
// else
// {
// //route to the other GC and discard this message
// CProtoBufMsg< CMsgGCMsgWebAPIJobRequestForwardResponse > msgRoute( k_EGCMsgWebAPIJobRequestForwardResponse );
// msgRoute.Body().set_dir_index( m_nRerouteRequest );
// m_pGC->BReplyToMessage( msgRoute, msg );
// }
// return true;
// }
VPROF_BUDGET( "WebAPI - Emitting result", VPROF_BUDGETGROUP_STEAM );
response.SetStatusCode( pwebAPIResponse->GetStatusCode() );
response.SetExpirationHeaderDeltaFromNow( pwebAPIResponse->GetExpirationSeconds() );
if( pwebAPIResponse->GetLastModified() )
response.SetHeaderTimeValue( "last-modified", pwebAPIResponse->GetLastModified() );
// if we aren't allowed to have a message body on this, simply send the result back now
if( !CHTTPUtil::BStatusCodeAllowsBody( pwebAPIResponse->GetStatusCode() ) )
{
AssertMsg( pwebAPIResponse->GetRootValue() == NULL, "Response HTTP status code %d doesn't allow a body, but one was present", pwebAPIResponse->GetStatusCode() );
//since we didn't have a body to serialize, we need to handle sending just the response
WebAPIRespondToRequest( GetName(), nSenderIP, response, msg );
return true;
}
//let the job convert the formatting and free our response for us (quite costly). Note that we detach the web API response since this job will free it
bool bThreadFuncSucceeded = BYieldingWaitForThreadFunc( CreateFunctor( ThreadedEmitFormattedOutputWrapperAndFreeResponse, pwebAPIResponse.Detach(), pEmitData.Get(), m_eDefaultOutputFormat, (size_t)cv_webapi_result_size_limit.GetInt() ) );
//if we called the function successfully and had a valid result, just send back the preserialized body
if( bThreadFuncSucceeded && pEmitData->m_bResult )
{
m_pGC->BReplyToMessageWithPreSerializedBody( k_EGCMsgWebAPIJobRequestHttpResponse, msg, ( const byte* )pEmitData->m_sSerializedResponse.c_str(), pEmitData->m_sSerializedResponse.size() );
g_WebAPIAccountTracker.TrackFunction( nSenderAccountID, nSenderIP, GetName(), pEmitData->m_sSerializedResponse.size() );
}
else
{
//we failed to generate a response, see if we ran out of space
if( response.GetBodyBuffer()->TellMaxPut() > cv_webapi_result_size_limit.GetInt() )
{
// !FIXME! DOTAMERGE
//CGCAlertInfo alert( "WebAPIResponseSize", "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() );
//
//switch( request.GetEHTTPMethod() )
//{
//case k_EHTTPMethodGET:
// {
// const uint32 nNumParams = request.GetGETParamCount();
// for( uint32 nParam = 0; nParam < nNumParams; nParam++ )
// {
// alert.AddExtendedInfoLine( "%s=%s", request.GetGETParamName( nParam ), request.GetGETParamValue( nParam ) );
// }
// }
// break;
//case k_EHTTPMethodPOST:
// {
// const uint32 nNumParams = request.GetPOSTParamCount();
// for( uint32 nParam = 0; nParam < nNumParams; nParam++ )
// {
// alert.AddExtendedInfoLine( "%s=%s", request.GetPOSTParamName( nParam ), request.GetPOSTParamValue( nParam ) );
// }
// }
// break;
//}
//
//GGCBase()->PostAlert( alert );
GGCBase()->PostAlert( k_EAlertTypeInfo, false, CFmtStr( "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() ) );
}
else
{
EG_WARNING( SPEW_GC, "WebAPI %s - request %s failed to emit for unknown reason\n", GetName(), request.GetURL() );
}
//make sure that they get an error code back
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError );
}
return true;
}
void CWebAPIJob::AddLocalizedString( CWebAPIValues *pOutDefn, const char *pchFieldName, const char *pchKeyName, ELanguage eLang, bool bReturnTokenIfNotFound )
{
// NULL keys we just skip
if( !pchKeyName )
return;
const char *pchValue;
if( eLang == k_Lang_None )
{
pchValue = pchKeyName;
}
else
{
pchValue = GGCBase()->LocalizeToken( pchKeyName, eLang, bReturnTokenIfNotFound );
}
if( pchValue )
{
pOutDefn->CreateChildObject( pchFieldName )->SetStringValue( pchValue );
}
}
//-----------------------------------------------------------------------------
// Purpose: A wrapper to call BEmitFormattedOutput and pass out a return value
// since functors don't make return values available.
//-----------------------------------------------------------------------------
/*static*/ void CWebAPIJob::ThreadedEmitFormattedOutputWrapper( CWebAPIResponse *pResponse, EWebAPIOutputFormat eFormat, CUtlBuffer *poutputBuffer, size_t unMaxResultSize, bool *pbResult )
{
*pbResult = pResponse->BEmitFormattedOutput( eFormat, *poutputBuffer, unMaxResultSize );
}
} // namespace GCSDK
Reference in New Issue View Git Blame Copy Permalink