From c5621b82cfeddee23b81871a53035fde747f73a9 Mon Sep 17 00:00:00 2001 From: Igor Scheller Date: Tue, 18 Dec 2018 02:23:44 +0100 Subject: Implemented /metrics endpoint and reimplemented /stats closes #418 (/metrics endpoint) Usage: ```yaml scrape_configs: - job_name: 'engelsystem' static_configs: - targets: ['engelsystem.example.com:80'] ``` --- config/routes.php | 5 + includes/pages/guest_stats.php | 47 ------ src/Controllers/Metrics/Controller.php | 131 ++++++++++++++++ src/Controllers/Metrics/MetricsEngine.php | 137 +++++++++++++++++ src/Controllers/Metrics/Stats.php | 144 ++++++++++++++++++ src/Middleware/LegacyMiddleware.php | 5 - tests/Unit/Controllers/Metrics/ControllerTest.php | 165 +++++++++++++++++++++ .../Unit/Controllers/Metrics/MetricsEngineTest.php | 69 +++++++++ tests/Unit/Controllers/Metrics/StatsTest.php | 74 +++++++++ 9 files changed, 725 insertions(+), 52 deletions(-) delete mode 100644 includes/pages/guest_stats.php create mode 100644 src/Controllers/Metrics/Controller.php create mode 100644 src/Controllers/Metrics/MetricsEngine.php create mode 100644 src/Controllers/Metrics/Stats.php create mode 100644 tests/Unit/Controllers/Metrics/ControllerTest.php create mode 100644 tests/Unit/Controllers/Metrics/MetricsEngineTest.php create mode 100644 tests/Unit/Controllers/Metrics/StatsTest.php diff --git a/config/routes.php b/config/routes.php index 2267bc88..8322cb2f 100644 --- a/config/routes.php +++ b/config/routes.php @@ -4,4 +4,9 @@ use FastRoute\RouteCollector; /** @var RouteCollector $route */ +// Pages $route->get('/credits', 'CreditsController@index'); + +// Stats +$route->get('/metrics', 'Metrics\\Controller@metrics'); +$route->get('/stats', 'Metrics\\Controller@stats'); diff --git a/includes/pages/guest_stats.php b/includes/pages/guest_stats.php deleted file mode 100644 index d9012748..00000000 --- a/includes/pages/guest_stats.php +++ /dev/null @@ -1,47 +0,0 @@ -has('api_key')) { - if (!empty($apiKey) && $request->input('api_key') == $apiKey) { - $stats = []; - - $stats['user_count'] = User::all()->count(); - $stats['arrived_user_count'] = State::whereArrived(true)->count(); - - $done_shifts_seconds = DB::selectOne(' - SELECT SUM(`Shifts`.`end` - `Shifts`.`start`) - FROM `ShiftEntry` - JOIN `Shifts` USING (`SID`) - WHERE `Shifts`.`end` < UNIX_TIMESTAMP() - '); - $done_shifts_seconds = (int)array_shift($done_shifts_seconds); - $stats['done_work_hours'] = round($done_shifts_seconds / (60 * 60), 0); - - $users_in_action = DB::select(' - SELECT `Shifts`.`start`, `Shifts`.`end` - FROM `ShiftEntry` - JOIN `Shifts` ON `Shifts`.`SID`=`ShiftEntry`.`SID` - WHERE UNIX_TIMESTAMP() BETWEEN `Shifts`.`start` AND `Shifts`.`end` - '); - $stats['users_in_action'] = count($users_in_action); - - header('Content-Type: application/json'); - raw_output(json_encode($stats)); - return; - } - raw_output(json_encode([ - 'error' => 'Wrong api_key.' - ])); - } - raw_output(json_encode([ - 'error' => 'Missing parameter api_key.' - ])); -} diff --git a/src/Controllers/Metrics/Controller.php b/src/Controllers/Metrics/Controller.php new file mode 100644 index 00000000..01fe1d6a --- /dev/null +++ b/src/Controllers/Metrics/Controller.php @@ -0,0 +1,131 @@ +config = $config; + $this->engine = $engine; + $this->request = $request; + $this->response = $response; + $this->stats = $stats; + } + + /** + * @return Response + */ + public function metrics() + { + $now = microtime(true); + $this->checkAuth(); + + $data = [ + $this->config->get('app_name') . ' stats', + 'users' => [ + 'type' => 'gauge', + ['labels' => ['state' => 'incoming'], 'value' => $this->stats->newUsers()], + ['labels' => ['state' => 'arrived', 'working' => 'no'], 'value' => $this->stats->arrivedUsers(false)], + ['labels' => ['state' => 'arrived', 'working' => 'yes'], 'value' => $this->stats->arrivedUsers(true)], + ], + 'users_working' => [ + 'type' => 'gauge', + ['labels' => ['freeloader' => false], $this->stats->currentlyWorkingUsers(false)], + ['labels' => ['freeloader' => true], $this->stats->currentlyWorkingUsers(true)], + ], + 'work_seconds' => [ + 'type' => 'gauge', + ['labels' => ['state' => 'done'], 'value' => $this->stats->workSeconds(true, false)], + ['labels' => ['state' => 'planned'], 'value' => $this->stats->workSeconds(false, false)], + ['labels' => ['state' => 'freeloaded'], 'value' => $this->stats->workSeconds(null, true)], + ], + 'registration_enabled' => ['type' => 'gauge', $this->config->get('registration_enabled')], + ]; + + $data['scrape_duration_seconds'] = [ + 'type' => 'gauge', + 'help' => 'Duration of the current request', + microtime(true) - $this->request->server->get('REQUEST_TIME_FLOAT', $now) + ]; + + return $this->response + ->withHeader('Content-Type', 'text/plain; version=0.0.4') + ->withContent($this->engine->get('/metrics', $data)); + } + + /** + * @return Response + */ + public function stats() + { + $this->checkAuth(true); + + $data = [ + 'user_count' => $this->stats->newUsers() + $this->stats->arrivedUsers(), + 'arrived_user_count' => $this->stats->arrivedUsers(), + 'done_work_hours' => round($this->stats->workSeconds(true) / 60 / 60, 0), + 'users_in_action' => $this->stats->currentlyWorkingUsers(), + ]; + + return $this->response + ->withHeader('Content-Type', 'application/json') + ->withContent(json_encode($data)); + } + + /** + * Ensure that the if the request is authorized + * + * @param bool $isJson + */ + protected function checkAuth($isJson = false) + { + $apiKey = $this->config->get('api_key'); + if (empty($apiKey) || $this->request->get('api_key') == $apiKey) { + return; + } + + $message = 'The api_key is invalid'; + $headers = []; + + if ($isJson) { + $message = json_encode(['error' => $message]); + $headers['Content-Type'] = 'application/json'; + } + + throw new HttpForbidden($message, $headers); + } +} diff --git a/src/Controllers/Metrics/MetricsEngine.php b/src/Controllers/Metrics/MetricsEngine.php new file mode 100644 index 00000000..eeb47d8a --- /dev/null +++ b/src/Controllers/Metrics/MetricsEngine.php @@ -0,0 +1,137 @@ + [['labels' => ['foo'=>'bar'], 'value'=>42]], 'bar'=>123] + * + * @param string $path + * @param mixed[] $data + * @return string + */ + public function get($path, $data = []): string + { + $return = []; + foreach ($data as $name => $list) { + if (is_int($name)) { + $return[] = '# ' . $this->escape($list); + continue; + } + + $list = is_array($list) ? $list : [$list]; + $name = 'engelsystem_' . $name; + + if (isset($list['help'])) { + $return[] = sprintf('# HELP %s %s', $name, $this->escape($list['help'])); + unset($list['help']); + } + + if (isset($list['type'])) { + $return[] = sprintf('# TYPE %s %s', $name, $list['type']); + unset($list['type']); + } + + $list = (!isset($list['value']) || !isset($list['labels'])) ? $list : [$list]; + foreach ($list as $row) { + $row = is_array($row) ? $row : [$row]; + + $return[] = $this->formatData($name, $row); + } + } + + return implode("\n", $return); + } + + /** + * @param string $path + * @return bool + */ + public function canRender($path): bool + { + return $path == '/metrics'; + } + + /** + * @param string $name + * @param array|mixed $row + * @see https://prometheus.io/docs/instrumenting/exposition_formats/ + * @return string + */ + protected function formatData($name, $row): string + { + return sprintf( + '%s%s %s', + $name, + $this->renderLabels($row), + $this->renderValue($row)); + } + + /** + * @param array|mixed $row + * @return mixed + */ + protected function renderLabels($row): string + { + $labels = []; + if (!is_array($row) || empty($row['labels'])) { + return ''; + } + + foreach ($row['labels'] as $type => $value) { + $labels[$type] = $type . '="' . $this->formatValue($value) . '"'; + } + + return '{' . implode(',', $labels) . '}'; + } + + /** + * @param array|mixed $row + * @return mixed + */ + protected function renderValue($row) + { + if (isset($row['value'])) { + return $this->formatValue($row['value']); + } + + return $this->formatValue(array_pop($row)); + } + + /** + * @param mixed $value + * @return mixed + */ + protected function formatValue($value) + { + if (is_bool($value)) { + return (int)$value; + } + + return $this->escape($value); + } + + /** + * @param mixed $value + * @return mixed + */ + protected function escape($value) + { + $replace = [ + '\\' => '\\\\', + '"' => '\\"', + "\n" => '\\n', + ]; + + return str_replace( + array_keys($replace), + array_values($replace), + $value + ); + } +} diff --git a/src/Controllers/Metrics/Stats.php b/src/Controllers/Metrics/Stats.php new file mode 100644 index 00000000..891f8c80 --- /dev/null +++ b/src/Controllers/Metrics/Stats.php @@ -0,0 +1,144 @@ +db = $db; + } + + /** + * The number of not arrived users + * + * @param null $working + * @return int + */ + public function arrivedUsers($working = null): int + { + $query = $this + ->getQuery('users') + ->join('users_state', 'user_id', '=', 'id') + ->where('arrived', '=', 1); + + if (!is_null($working)) { + // @codeCoverageIgnoreStart + $query + ->leftJoin('UserWorkLog', 'UserWorkLog.user_id', '=', 'users.id') + ->leftJoin('ShiftEntry', 'ShiftEntry.UID', '=', 'users.id') + ->groupBy('users.id'); + + $query->where(function ($query) use ($working) { + /** @var QueryBuilder $query */ + if ($working) { + $query + ->whereNotNull('ShiftEntry.SID') + ->orWhereNotNull('UserWorkLog.work_hours'); + + return; + } + $query + ->whereNull('ShiftEntry.SID') + ->whereNull('UserWorkLog.work_hours'); + }); + // @codeCoverageIgnoreEnd + } + + return $query + ->count(); + } + + /** + * The number of not arrived users + * + * @return int + */ + public function newUsers(): int + { + return $this + ->getQuery('users') + ->join('users_state', 'user_id', '=', 'id') + ->where('arrived', '=', 0) + ->count(); + } + + /** + * The number of currently working users + * + * @param null $freeloaded + * @return int + * @codeCoverageIgnore + */ + public function currentlyWorkingUsers($freeloaded = null): int + { + $query = $this + ->getQuery('users') + ->join('ShiftEntry', 'ShiftEntry.UID', '=', 'users.id') + ->join('Shifts', 'Shifts.SID', '=', 'ShiftEntry.SID') + ->where('Shifts.start', '<=', time()) + ->where('Shifts.end', '>', time()); + + if (!is_null($freeloaded)) { + $query->where('ShiftEntry.freeloaded', '=', $freeloaded); + } + + return $query->count(); + } + + /** + * The number of worked shifts + * + * @param bool|null $done + * @param bool|null $freeloaded + * @return int + * @codeCoverageIgnore + */ + public function workSeconds($done = null, $freeloaded = null): int + { + $query = $this + ->getQuery('ShiftEntry') + ->join('Shifts', 'Shifts.SID', '=', 'ShiftEntry.SID'); + + if (!is_null($freeloaded)) { + $query->where('freeloaded', '=', $freeloaded); + } + + if (!is_null($done)) { + $query->where('end', ($done == true ? '<' : '>='), time()); + } + + return $query->sum($this->raw('end - start')); + } + + /** + * @param string $table + * @return QueryBuilder + */ + protected function getQuery(string $table): QueryBuilder + { + return $this->db + ->getConnection() + ->table($table); + } + + /** + * @param mixed $value + * @return QueryExpression + * @codeCoverageIgnore + */ + protected function raw($value) + { + return $this->db->getConnection()->raw($value); + } +} diff --git a/src/Middleware/LegacyMiddleware.php b/src/Middleware/LegacyMiddleware.php index b1315fda..8524764f 100644 --- a/src/Middleware/LegacyMiddleware.php +++ b/src/Middleware/LegacyMiddleware.php @@ -26,7 +26,6 @@ class LegacyMiddleware implements MiddlewareInterface 'shift_entries', 'shifts', 'shifts_json_export', - 'stats', 'users', 'user_driver_licenses', 'user_password_recovery', @@ -122,10 +121,6 @@ class LegacyMiddleware implements MiddlewareInterface case 'shifts_json_export': require_once realpath(__DIR__ . '/../../includes/controller/shifts_controller.php'); shifts_json_export_controller(); - /** @noinspection PhpMissingBreakStatementInspection */ - case 'stats': - require_once realpath(__DIR__ . '/../../includes/pages/guest_stats.php'); - guest_stats(); case 'user_password_recovery': require_once realpath(__DIR__ . '/../../includes/controller/users_controller.php'); $title = user_password_recovery_title(); diff --git a/tests/Unit/Controllers/Metrics/ControllerTest.php b/tests/Unit/Controllers/Metrics/ControllerTest.php new file mode 100644 index 00000000..013a3352 --- /dev/null +++ b/tests/Unit/Controllers/Metrics/ControllerTest.php @@ -0,0 +1,165 @@ +getMocks(); + + $request->server = new ServerBag(); + $request->server->set('REQUEST_TIME_FLOAT', 0.0123456789); + + $engine->expects($this->once()) + ->method('get') + ->willReturnCallback(function ($path, $data) use ($response) { + $this->assertEquals('/metrics', $path); + $this->assertArrayHasKey('users', $data); + $this->assertArrayHasKey('users_working', $data); + $this->assertArrayHasKey('work_seconds', $data); + $this->assertArrayHasKey('registration_enabled', $data); + $this->assertArrayHasKey('scrape_duration_seconds', $data); + + return 'metrics return'; + }); + + $response->expects($this->once()) + ->method('withHeader') + ->with('Content-Type', 'text/plain; version=0.0.4') + ->willReturn($response); + $response->expects($this->once()) + ->method('withContent') + ->with('metrics return') + ->willReturn($response); + + $stats->expects($this->exactly(2)) + ->method('arrivedUsers') + ->withConsecutive([false], [true]) + ->willReturnOnConsecutiveCalls(7, 43); + $stats->expects($this->exactly(2)) + ->method('currentlyWorkingUsers') + ->withConsecutive([false], [true]) + ->willReturnOnConsecutiveCalls(10, 1); + $stats->expects($this->exactly(3)) + ->method('workSeconds') + ->withConsecutive([true, false], [false, false], [null, true]) + ->willReturnOnConsecutiveCalls(60 * 37, 60 * 251, 60 * 3); + $this->setExpects($stats, 'newUsers', null, 9); + + $config->set('registration_enabled', 1); + + $controller = new Controller($response, $engine, $config, $request, $stats); + $controller->metrics(); + } + + /** + * @covers \Engelsystem\Controllers\Metrics\Controller::stats + * @covers \Engelsystem\Controllers\Metrics\Controller::checkAuth + */ + public function testStats() + { + /** @var Response|MockObject $response */ + /** @var Request|MockObject $request */ + /** @var MetricsEngine|MockObject $engine */ + /** @var Stats|MockObject $stats */ + /** @var Config $config */ + list($response, $request, $engine, $stats, $config) = $this->getMocks(); + + $response->expects($this->once()) + ->method('withHeader') + ->with('Content-Type', 'application/json') + ->willReturn($response); + $response->expects($this->once()) + ->method('withContent') + ->with(json_encode([ + 'user_count' => 13, + 'arrived_user_count' => 10, + 'done_work_hours' => 99, + 'users_in_action' => 5 + ])) + ->willReturn($response); + + $request->expects($this->once()) + ->method('get') + ->with('api_key') + ->willReturn('ApiKey987'); + + $config->set('api_key', 'ApiKey987'); + + $stats->expects($this->once()) + ->method('workSeconds') + ->with(true) + ->willReturn(60 * 60 * 99.47); + $this->setExpects($stats, 'newUsers', null, 3); + $this->setExpects($stats, 'arrivedUsers', null, 10, $this->exactly(2)); + $this->setExpects($stats, 'currentlyWorkingUsers', null, 5); + + $controller = new Controller($response, $engine, $config, $request, $stats); + $controller->stats(); + } + + /** + * @covers \Engelsystem\Controllers\Metrics\Controller::checkAuth + */ + public function testCheckAuth() + { + /** @var Response|MockObject $response */ + /** @var Request|MockObject $request */ + /** @var MetricsEngine|MockObject $engine */ + /** @var Stats|MockObject $stats */ + /** @var Config $config */ + list($response, $request, $engine, $stats, $config) = $this->getMocks(); + + $request->expects($this->once()) + ->method('get') + ->with('api_key') + ->willReturn('LoremIpsum!'); + + $config->set('api_key', 'fooBar!'); + + $controller = new Controller($response, $engine, $config, $request, $stats); + + $this->expectException(HttpForbidden::class); + $this->expectExceptionMessage(json_encode(['error' => 'The api_key is invalid'])); + $controller->stats(); + } + + /** + * @return array + */ + protected function getMocks(): array + { + /** @var Response|MockObject $response */ + $response = $this->createMock(Response::class); + /** @var Request|MockObject $request */ + $request = $this->createMock(Request::class); + /** @var MetricsEngine|MockObject $engine */ + $engine = $this->createMock(MetricsEngine::class); + /** @var Stats|MockObject $stats */ + $stats = $this->createMock(Stats::class); + $config = new Config(); + + return array($response, $request, $engine, $stats, $config); + } +} diff --git a/tests/Unit/Controllers/Metrics/MetricsEngineTest.php b/tests/Unit/Controllers/Metrics/MetricsEngineTest.php new file mode 100644 index 00000000..b810b10a --- /dev/null +++ b/tests/Unit/Controllers/Metrics/MetricsEngineTest.php @@ -0,0 +1,69 @@ +assertEquals('', $engine->get('/metrics')); + + $this->assertEquals('engelsystem_users 13', $engine->get('/metrics', ['users' => 13])); + + $this->assertEquals('engelsystem_bool_val 0', $engine->get('/metrics', ['bool_val' => false])); + + $this->assertEquals('# Lorem \n Ipsum', $engine->get('/metrics', ["Lorem \n Ipsum"])); + + $this->assertEquals( + 'engelsystem_foo{lorem="ip\\\\sum"} \\"lorem\\n\\\\ipsum\\"', + $engine->get('/metrics', [ + 'foo' => ['labels' => ['lorem' => 'ip\\sum'], 'value' => "\"lorem\n\\ipsum\""] + ]) + ); + + $this->assertEquals( + 'engelsystem_foo_count{bar="14"} 42', + $engine->get('/metrics', ['foo_count' => ['labels' => ['bar' => 14], 'value' => 42],]) + ); + + $this->assertEquals( + 'engelsystem_lorem{test="123"} NaN' . "\n" . 'engelsystem_lorem{test="456"} 999.99', + $engine->get('/metrics', [ + 'lorem' => [ + ['labels' => ['test' => 123], 'value' => 'NaN'], + ['labels' => ['test' => 456], 'value' => 999.99], + ], + ]) + ); + + $this->assertEquals( + "# HELP engelsystem_test Some help\\n text\n# TYPE engelsystem_test counter\nengelsystem_test 99", + $engine->get('/metrics', ['test' => ['help' => "Some help\n text", 'type' => 'counter', 'value' => 99]]) + ); + } + + /** + * @covers \Engelsystem\Controllers\Metrics\MetricsEngine::canRender + */ + public function testCanRender() + { + $engine = new MetricsEngine(); + + $this->assertFalse($engine->canRender('/')); + $this->assertFalse($engine->canRender('/metrics.foo')); + $this->assertTrue($engine->canRender('/metrics')); + } +} diff --git a/tests/Unit/Controllers/Metrics/StatsTest.php b/tests/Unit/Controllers/Metrics/StatsTest.php new file mode 100644 index 00000000..1618b99b --- /dev/null +++ b/tests/Unit/Controllers/Metrics/StatsTest.php @@ -0,0 +1,74 @@ +initDatabase(); + $this->addUsers(); + + $stats = new Stats($this->database); + $this->assertEquals(2, $stats->newUsers()); + } + + /** + * @covers \Engelsystem\Controllers\Metrics\Stats::arrivedUsers + */ + public function testArrivedUsers() + { + $this->initDatabase(); + $this->addUsers(); + + $stats = new Stats($this->database); + $this->assertEquals(3, $stats->arrivedUsers()); + } + + /** + * Add some example users + */ + protected function addUsers() + { + $this->addUser(); + $this->addUser(); + $this->addUser(['arrived' => 1]); + $this->addUser(['arrived' => 1, 'active' => 1]); + $this->addUser(['arrived' => 1, 'active' => 1]); + } + + /** + * @param array $state + */ + protected function addUser(array $state = []) + { + $name = 'user_' . Str::random(5); + + $user = new User([ + 'name' => $name, + 'password' => '', + 'email' => $name . '@engel.example.com', + 'api_key' => '', + ]); + $user->save(); + + $state = new State($state); + $state->user() + ->associate($user) + ->save(); + } +} -- cgit v1.2.3-54-g00ecf