diff --git a/app/ApiInterface.php b/app/ApiInterface.php index 987e5ef..d8884e1 100644 --- a/app/ApiInterface.php +++ b/app/ApiInterface.php @@ -92,6 +92,28 @@ public function setWaypoint(int $systemId, string $accessToken, array $options = */ public function getCorporationRoles(int $corporationId, string $accessToken): array; + /** + * @return array + */ + public function getRegions(): array; + + /** + * @param int $regionId + * @return array + */ + public function getRegionData(int $regionId): array; + + /** + * @return array + */ + public function getConstellations(): array; + + /** + * @param int $constellationId + * @return array + */ + public function getConstellationData(int $constellationId): array; + /** * @param array $universeIds * @param array $additionalOptions @@ -109,6 +131,13 @@ public function getUniverseJumps(): array; */ public function getUniverseKills(): array; + /** + * @param int $typeId + * @param array $additionalOptions + * @return array + */ + public function getUniverseTypesData(int $typeId, array $additionalOptions = []): array; + /** * @param int $targetId * @param string $accessToken diff --git a/app/Config/ESIConf.php b/app/Config/ESIConf.php index 7e2516f..1dfec90 100644 --- a/app/Config/ESIConf.php +++ b/app/Config/ESIConf.php @@ -51,6 +51,21 @@ class ESIConf extends \Prefab { ], 'system_kills' => [ 'GET' => ' /v1/universe/system_kills/' + ], + 'regions' => [ + 'GET' => '/v1/universe/regions/{x}/', + 'list' => [ + 'GET' => '/v1/universe/regions/' + ] + ], + 'constellations' => [ + 'GET' => '/v1/universe/constellations/{x}/', + 'list' => [ + 'GET' => '/v1/universe/constellations/' + ] + ], + 'types' => [ + 'GET' => '/v3/universe/types/{x}/' ] ], 'ui' => [ diff --git a/app/ESI.php b/app/ESI.php index 590af0f..a5c165d 100644 --- a/app/ESI.php +++ b/app/ESI.php @@ -27,6 +27,12 @@ class ESI implements ApiInterface { */ private $esiUrl, $esiUserAgent, $esiDatasource, $endpointVersion = ''; + /** + * debugLevel + * @var int + */ + private $debugLevel = 0; + /** * ESI constructor. */ @@ -54,6 +60,13 @@ public function setDatasource(string $datasource){ $this->esiDatasource = $datasource; } + /** + * @param int $debug + */ + public function setDebugLevel(int $debug){ + $this->debugLevel = $debug; + } + /** * @param string $version */ @@ -82,6 +95,13 @@ public function getDatasource(): string{ return $this->esiDatasource; } + /** + * @return int + */ + public function getDebugLevel(): int { + return $this->debugLevel; + } + /** * @return string */ @@ -287,6 +307,68 @@ public function getCorporationRoles(int $corporationId, string $accessToken): ar return $rolesData; } + /** + * @return array + */ + public function getRegions(): array{ + $url = $this->getEndpointURL(['universe', 'regions', 'list', 'GET']); + $regionData = []; + $response = $this->request($url, 'GET'); + + if( !empty($response) ){ + $regionData = array_unique( array_map('intval', $response) ); + } + + return $regionData; + } + + /** + * @param int $regionId + * @return array + */ + public function getRegionData(int $regionId): array{ + $url = $this->getEndpointURL(['universe', 'regions', 'GET'], [$regionId]); + $regionData = []; + $response = $this->request($url, 'GET'); + + if( !empty($response) ){ + $regionData = (new namespace\Mapper\Region($response))->getData(); + } + + return $regionData; + } + + /** + * @return array + */ + public function getConstellations(): array{ + $url = $this->getEndpointURL(['universe', 'constellations', 'list', 'GET']); + $constellationData = []; + $response = $this->request($url, 'GET'); + + if( !empty($response) ){ + $constellationData = array_unique( array_map('intval', $response) ); + } + + return $constellationData; + } + + /** + * @param int $constellationId + * @return array + */ + public function getConstellationData(int $constellationId): array{ + $url = $this->getEndpointURL(['universe', 'constellations', 'GET'], [$constellationId]); + $constellationData = []; + $response = $this->request($url, 'GET'); + + if( !empty($response) ){ + $constellationData = (new namespace\Mapper\Constellation($response))->getData(); + } + + return $constellationData; + } + /** * @param array $universeIds * @param array $additionalOptions @@ -362,6 +444,23 @@ public function getUniverseKills(): array{ return $systemKills; } + /** + * @param int $typeId + * @param array $additionalOptions + * @return array + */ + public function getUniverseTypesData(int $typeId, array $additionalOptions = []): array { + $url = $this->getEndpointURL(['universe', 'types', 'GET'], [$typeId]); + $typesData = []; + $response = $this->request($url, 'GET', '', $additionalOptions); + + if( !empty($response) ){ + $typesData = (new namespace\Mapper\Universe\Type($response))->getData(); + } + + return $typesData; + } + /** * @param int $targetId * @param string $accessToken @@ -452,35 +551,38 @@ protected function request(string $url, string $method = 'GET', string $accessTo $responseBody = null; $method = strtoupper($method); - $webClient = namespace\Lib\WebClient::instance(); + $webClient = namespace\Lib\WebClient::instance($this->getDebugLevel()); if( \Audit::instance()->url($url) ){ - if( $webClient->checkRequestMethod($method) ){ - $requestOptions = [ - 'timeout' => self::ESI_TIMEOUT, - 'method' => $method, - 'user_agent' => $this->getUserAgent(), - 'header' => [ - 'Accept: application/json' - ] - ]; - - // add auth token if available (required for some endpoints) - if( !empty($accessToken) ){ - $requestOptions['header'][] = 'Authorization: Bearer ' . $accessToken; + // check if url is blocked (error limit exceeded) + if(!$webClient->isBlockedUrl($url)){ + if( $webClient->checkRequestMethod($method) ){ + $requestOptions = [ + 'timeout' => self::ESI_TIMEOUT, + 'method' => $method, + 'user_agent' => $this->getUserAgent(), + 'header' => [ + 'Accept: application/json' + ] + ]; + + // add auth token if available (required for some endpoints) + if( !empty($accessToken) ){ + $requestOptions['header'][] = 'Authorization: Bearer ' . $accessToken; + } + + if( !empty($additionalOptions['content']) ){ + // "Content-Type" Header is required for POST requests + $requestOptions['header'][] = 'Content-Type: application/json'; + + $requestOptions['content'] = json_encode($additionalOptions['content'], JSON_UNESCAPED_SLASHES); + unset($additionalOptions['content']); + } + + $responseBody = $webClient->request($url, $requestOptions, $additionalOptions); + }else{ + $webClient->getLogger('err_server')->write(sprintf(self::ERROR_ESI_METHOD, $method, $url)); } - - if( !empty($additionalOptions['content']) ){ - // "Content-Type" Header is required for POST requests - $requestOptions['header'][] = 'Content-Type: application/json'; - - $requestOptions['content'] = json_encode($additionalOptions['content'], JSON_UNESCAPED_SLASHES); - unset($additionalOptions['content']); - } - - $responseBody = $webClient->request($url, $requestOptions, $additionalOptions); - }else{ - $webClient->getLogger('err_server')->write(sprintf(self::ERROR_ESI_METHOD, $method, $url)); } }else{ $webClient->getLogger('err_server')->write(sprintf(self::ERROR_ESI_URL, $url)); diff --git a/app/Lib/WebClient.php b/app/Lib/WebClient.php index 8c8735f..4fc44b7 100644 --- a/app/Lib/WebClient.php +++ b/app/Lib/WebClient.php @@ -11,39 +11,62 @@ class WebClient extends \Web { + const CACHE_KEY_ERROR_LIMIT = 'CACHED_ERROR_LIMIT'; + const ERROR_STATUS_LOG = 'HTTP %s: \'%s\' | url: %s \'%s\'%s'; - const ERROR_RESOURCE_LEGACY = 'Resource: %s has been marked as legacy.'; - const ERROR_RESOURCE_DEPRECATED = 'Resource: %s has been marked as deprecated.'; + const ERROR_RESOURCE_LEGACY = 'Resource: %s has been marked as legacy. (%s)'; + const ERROR_RESOURCE_DEPRECATED = 'Resource: %s has been marked as deprecated. (%s)'; + const ERROR_LIMIT_CRITICAL = 'Error rate reached critical amount. url: %s | errorCount: %s | errorRemainCount: %s'; + const ERROR_LIMIT_EXCEEDED = 'Error rate limit exceeded! We are blocked for (%s seconds)'; + const DEBUG_URI_BLOCKED = 'Debug request blocked. Error limit exceeded. url: %s blocked for %2ss'; const REQUEST_METHODS = ['GET', 'POST', 'PUT', 'DELETE']; + // log error when this error count is reached for a single API endpoint in the current error window + const ERROR_COUNT_MAX_URL = 30; + + // log error if less then this errors remain in current error window (all endpoints) + const ERROR_COUNT_REMAIN_TOTAL = 10; + + // max number of CREST curls for a single endpoint until giving up... + // ->this is because CREST is not very stable + const RETRY_COUNT_MAX = 2; + /** - * max number of CREST curls for a single endpoint until giving up... - * this is because CREST is not very stable + * debugLevel used for internal error/warning logging + * @var int */ - const RETRY_COUNT_MAX = 3; + protected $debugLevel = 0; + + public function __construct(int $debugLevel = 0){ + $this->debugLevel = $debugLevel; + } /** - * end of line - * @var string + * parse array with HTTP header data + * @param array $headers + * @return array */ - private $eol = "\r\n"; + protected function parseHeaders(array $headers = []): array { + $parsedHeaders = []; + foreach($headers as $header){ + $parts = explode(':', $header, 2); + $parsedHeaders[strtolower(trim($parts[0]))] = isset($parts[1]) ? trim($parts[1]) : ''; + } + return $parsedHeaders; + } /** * @param array $headers * @return int */ - protected function getStatusCodeFromHeaders($headers = []){ + protected function getStatusCodeFromHeaders(array $headers = []): int { $statusCode = 0; - - if( - preg_match( - '/HTTP\/1\.\d (\d{3}?)/', - implode($this->eol, (array)$headers), - $matches - ) - ){ - $statusCode = (int)$matches[1]; + foreach($headers as $key => $value){ + if(preg_match('/http\/1\.\d (\d{3}?)/i', $key, $matches)){ + $statusCode = (int)$matches[1]; + break; + } } return $statusCode; } @@ -100,19 +123,19 @@ protected function getErrorMessageFromJsonResponse(int $code, string $method, st public function getLogger(string $statusType): \Log{ switch($statusType){ case 'err_server': - $logfile = 'esi.error.server'; + $logfile = 'esi_error_server'; break; case 'err_client': - $logfile = 'esi.error.client'; + $logfile = 'esi_error_client'; break; case 'resource_legacy': - $logfile = 'esi.resource.legacy'; + $logfile = 'esi_resource_legacy'; break; case 'resource_deprecated': - $logfile = 'esi.resource.deprecated'; + $logfile = 'esi_resource_deprecated'; break; default: - $logfile = 'esi.error.unknown'; + $logfile = 'esi_error_unknown'; } return new \Log($logfile . '.log'); } @@ -123,16 +146,90 @@ public function getLogger(string $statusType): \Log{ * @param string $url */ protected function checkResponseHeaders(array $headers, string $url){ - $headers = (array)$headers; + $statusCode = $this->getStatusCodeFromHeaders($headers); - if( preg_grep('/^Warning: 199/i', $headers) ){ - $this->getLogger('resource_legacy')->write(sprintf(self::ERROR_RESOURCE_LEGACY, $url)); + // check ESI warnings ----------------------------------------------------------------------------------------- + // extract ESI related headers + $warningHeaders = array_filter($headers, function($key){ + return preg_match('/^warning/i', $key); + }, ARRAY_FILTER_USE_KEY); + foreach($warningHeaders as $key => $value){ + if( preg_match('/^199/i', $value) ){ + $this->getLogger('resource_legacy')->write(sprintf(self::ERROR_RESOURCE_LEGACY, $url, $value)); + } + if( preg_match('/^299/i', $value) ){ + $this->getLogger('resource_deprecated')->write(sprintf(self::ERROR_RESOURCE_DEPRECATED, $url, $value)); + } } - if( preg_grep('/^Warning: 299/i', $headers) ){ - $this->getLogger('resource_deprecated')->write(sprintf(self::ERROR_RESOURCE_DEPRECATED, $url)); + + // check ESI error limits ------------------------------------------------------------------------------------- + if($statusCode >= 400 && $statusCode <= 599){ + // extract ESI related headers + $esiHeaders = array_filter($headers, function($key){ + return preg_match('/^x-esi-/i', $key); + }, ARRAY_FILTER_USE_KEY); + + if(array_key_exists('x-esi-error-limit-reset', $esiHeaders)){ + // time in seconds until current error limit "windows" reset + $esiErrorLimitReset = (int)$esiHeaders['x-esi-error-limit-reset']; + + // block further api calls for this URL until error limit is reset/clear + $blockUrl = false; + + // get "normalized" url path without params/placeholders + $urlPath = $this->getNormalizedUrlPath($url); + + $f3 = \Base::instance(); + if(!$f3->exists(self::CACHE_KEY_ERROR_LIMIT, $esiErrorRate)){ + $esiErrorRate = []; + } + // increase error count for this $url + $errorCount = (int)$esiErrorRate[$urlPath]['count'] + 1; + $esiErrorRate[$urlPath]['count'] = $errorCount; + + // sort by error count desc + uasort($esiErrorRate, function($a, $b) { + return $b['count'] <=> $a['count']; + }); + + if(array_key_exists('x-esi-error-limited', $esiHeaders)){ + // we are blocked until new error limit window opens this should never happen + $blockUrl = true; + $this->getLogger('err_server')->write(sprintf(self::ERROR_LIMIT_EXCEEDED, $esiErrorLimitReset)); + } + + if(array_key_exists('x-esi-error-limit-remain', $esiHeaders)){ + // remaining errors left until reset/clear + $esiErrorLimitRemain = (int)$esiHeaders['x-esi-error-limit-remain']; + + if( + $errorCount > self::ERROR_COUNT_MAX_URL || + $esiErrorLimitRemain < self::ERROR_COUNT_REMAIN_TOTAL + ){ + $blockUrl = true; + $this->getLogger('err_server')->write(sprintf(self::ERROR_LIMIT_CRITICAL, $urlPath, $errorCount, $esiErrorLimitRemain)); + } + } + + if($blockUrl){ + // to many error, block uri until error limit reset + $esiErrorRate[$urlPath]['blocked'] = true; + } + + $f3->set(self::CACHE_KEY_ERROR_LIMIT, $esiErrorRate, $esiErrorLimitReset); + } } } + /** + * get URL path from $url, removes path IDs, parameters, scheme and domain + * @param $url + * @return string + */ + protected function getNormalizedUrlPath($url): string { + return parse_url(strtok(preg_replace('/\/(\d+)\//', '/{x}/', $url), '?'), PHP_URL_PATH); + } + /** * check whether a HTTP request method is valid/given * @param $method @@ -146,6 +243,37 @@ public function checkRequestMethod($method): bool { return $valid; } + /** + * check API url against blocked API endpoints blacklist + * @param string $url + * @return bool + */ + public function isBlockedUrl(string $url): bool { + $isBlocked = false; + $f3 = \Base::instance(); + if($ttlData = $f3->exists(self::CACHE_KEY_ERROR_LIMIT, $esiErrorRate)){ + // check url path if blocked + $urlPath = $this->getNormalizedUrlPath($url); + $esiErrorData = array_filter($esiErrorRate, function($value, $key) use (&$urlPath){ + return ($key === $urlPath && $value['blocked']); + }, ARRAY_FILTER_USE_BOTH); + + if(!empty($esiErrorData)){ + $isBlocked = true; + if($this->debugLevel === 3){ + // log debug information + $this->getLogger('err_server')->write(sprintf( + self::DEBUG_URI_BLOCKED, + $urlPath, + round($ttlData[0] + $ttlData[1] - time()) + )); + } + } + } + + return $isBlocked; + } + /** * @param string $url * @param array|null $options @@ -172,8 +300,9 @@ public function request( $url, array $options = null, $additionalOptions = [], $ } if( !empty($responseHeaders)){ + $parsedResponseHeaders = $this->parseHeaders($responseHeaders); // check response headers - $this->checkResponseHeaders($responseHeaders, $url); + $this->checkResponseHeaders($parsedResponseHeaders, $url); $statusCode = $this->getStatusCodeFromHeaders($responseHeaders); $statusType = $this->getStatusType($statusCode); diff --git a/app/Mapper/Constellation.php b/app/Mapper/Constellation.php new file mode 100644 index 0000000..33005a3 --- /dev/null +++ b/app/Mapper/Constellation.php @@ -0,0 +1,25 @@ + 'id', + 'name' => 'name', + 'region_id' => 'regionId', + 'position' => 'position', + 'x' => 'x', + 'y' => 'y', + 'z' => 'z', + 'systems' => 'systems' + ]; +} \ No newline at end of file diff --git a/app/Mapper/Region.php b/app/Mapper/Region.php new file mode 100644 index 0000000..79c55cb --- /dev/null +++ b/app/Mapper/Region.php @@ -0,0 +1,21 @@ + 'id', + 'name' => 'name', + 'description' => 'description', + 'constellations' => 'constellationIds' + ]; +} \ No newline at end of file diff --git a/app/Mapper/Universe/Type.php b/app/Mapper/Universe/Type.php new file mode 100644 index 0000000..722eb84 --- /dev/null +++ b/app/Mapper/Universe/Type.php @@ -0,0 +1,30 @@ + 'id', + 'name' => 'name', + 'description' => 'description', + 'published' => 'published', + 'group_id' => 'groupId', + 'market_group_id' => 'marketGroupId', + 'radius' => 'radius', + 'volume' => 'volume', + 'packaged_volume' => 'packagedVolume', + 'capacity' => 'capacity', + 'portion_size' => 'portionSize', + 'mass' => 'mass', + 'graphic_id' => 'graphicId' + ]; +} \ No newline at end of file