Skip to content

Commit 7ea8d76

Browse files
christyjacob4christyjacob4
committed
feat: aggregate stats
1 parent ff73939 commit 7ea8d76

File tree

4 files changed

+218
-0
lines changed

4 files changed

+218
-0
lines changed

‎Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ RUN chmod +x /usr/local/bin/doctor && \
314314
chmod +x /usr/local/bin/sdks && \
315315
chmod +x /usr/local/bin/specs && \
316316
chmod +x /usr/local/bin/ssl && \
317+
chmod +x /usr/local/bin/stat && \
317318
chmod +x /usr/local/bin/test && \
318319
chmod +x /usr/local/bin/vars && \
319320
chmod +x /usr/local/bin/worker-audits && \

‎bin/stat

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
3+
php /usr/src/code/app/cli.php stat $@

‎src/Appwrite/Platform/Services/Tasks.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
use Appwrite\Platform\Tasks\SDKs;
1313
use Appwrite\Platform\Tasks\Specs;
1414
use Appwrite\Platform\Tasks\SSL;
15+
use Appwrite\Platform\Tasks\Stat;
1516
use Appwrite\Platform\Tasks\Usage;
1617
use Appwrite\Platform\Tasks\Vars;
1718
use Appwrite\Platform\Tasks\Version;
@@ -27,6 +28,7 @@ public function __construct()
2728
->addAction(Usage::getName(), new Usage())
2829
->addAction(Vars::getName(), new Vars())
2930
->addAction(SSL::getName(), new SSL())
31+
->addAction(Stat::getName(), new Stat())
3032
->addAction(Doctor::getName(), new Doctor())
3133
->addAction(Install::getName(), new Install())
3234
->addAction(Maintenance::getName(), new Maintenance())

‎src/Appwrite/Platform/Tasks/Stat.php

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
<?php
2+
3+
namespace Appwrite\Platform\Tasks;
4+
5+
use Utopia\Platform\Action;
6+
use Exception;
7+
use PDO;
8+
use Utopia\App;
9+
use Utopia\Cache\Adapter\None;
10+
use Utopia\Cache\Cache;
11+
use Utopia\CLI\Console;
12+
use Utopia\Database\Adapter\MySQL;
13+
use Utopia\Database\Database;
14+
use Utopia\Database\Query;
15+
use Utopia\Database\Validator\Authorization;
16+
use Utopia\DSN\DSN;
17+
18+
class Stat extends Action
19+
{
20+
public static function getName(): string
21+
{
22+
return 'stat';
23+
}
24+
25+
public function __construct()
26+
{
27+
$this
28+
->desc('Get stats for project')
29+
->callback(fn () => $this->action());
30+
}
31+
32+
function getConnection(string $dsn): PDO
33+
{
34+
if (empty($dsn)) {
35+
throw new Exception("Missing value for DSN connection");
36+
}
37+
$dsn = new DSN($dsn);
38+
$dsnHost = $dsn->getHost();
39+
$dsnPort = $dsn->getPort();
40+
$dsnUser = $dsn->getUser();
41+
$dsnPass = $dsn->getPassword();
42+
$dsnScheme = $dsn->getScheme();
43+
$dsnDatabase = $dsn->getPath();
44+
45+
$connection = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
46+
PDO::ATTR_TIMEOUT => 3, // Seconds
47+
PDO::ATTR_PERSISTENT => true,
48+
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
49+
PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING,
50+
PDO::ATTR_EMULATE_PREPARES => true,
51+
PDO::ATTR_STRINGIFY_FETCHES => true
52+
));
53+
54+
return $connection;
55+
}
56+
57+
58+
function getStats(Database $dbForProject): array
59+
{
60+
$range = '90d';
61+
$periods = [
62+
'90d' => [
63+
'period' => '1d',
64+
'limit' => 90,
65+
],
66+
];
67+
68+
$metrics = [
69+
'files.$all.count.total',
70+
'buckets.$all.count.total',
71+
'databases.$all.count.total',
72+
'documents.$all.count.total',
73+
'collections.$all.count.total',
74+
'project.$all.storage.size',
75+
'project.$all.network.requests',
76+
'project.$all.network.bandwidth',
77+
'users.$all.count.total',
78+
'sessions.$all.requests.create',
79+
'executions.$all.compute.total',
80+
];
81+
82+
$stats = [];
83+
84+
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
85+
foreach ($metrics as $metric) {
86+
$limit = $periods[$range]['limit'];
87+
$period = $periods[$range]['period'];
88+
89+
$requestDocs = $dbForProject->find('stats', [
90+
Query::equal('period', [$period]),
91+
Query::equal('metric', [$metric]),
92+
Query::limit($limit),
93+
Query::orderDesc('time'),
94+
]);
95+
96+
$stats[$metric] = [];
97+
foreach ($requestDocs as $requestDoc) {
98+
$stats[$metric][] = [
99+
'value' => $requestDoc->getAttribute('value'),
100+
'date' => $requestDoc->getAttribute('time'),
101+
];
102+
}
103+
104+
$stats[$metric] = array_reverse($stats[$metric]);
105+
// Calculate aggregate of each metric
106+
$stats[$metric . '.sum'] = array_sum(array_column($stats[$metric], 'value'));
107+
}
108+
});
109+
110+
// return only the ahhggregate values
111+
return array_filter($stats, fn ($key) => strpos($key, '.sum') !== false, ARRAY_FILTER_USE_KEY);
112+
}
113+
114+
115+
public function action(): void
116+
{
117+
Console::success('Getting stats...');
118+
119+
$databases = [
120+
'console' => [
121+
'type' => 'database',
122+
'dsns' => '',
123+
'multiple' => false,
124+
'schemes' => ['mariadb', 'mysql'],
125+
],
126+
'projects' => [
127+
'type' => 'database',
128+
'dsns' => '',
129+
'multiple' => true,
130+
'schemes' => ['mariadb', 'mysql'],
131+
],
132+
];
133+
134+
$dsns = explode(',', $databases['projects']['dsns']);
135+
$projectdsns = [];
136+
foreach ($dsns as &$dsn) {
137+
$dsn = explode('=', $dsn);
138+
$name = 'database' . '_' . $dsn[0];
139+
$dsn = $dsn[1] ?? '';
140+
$projectdsns[$name] = $dsn;
141+
}
142+
143+
$cache = new Cache(new None());
144+
$consoledsn = explode('=', $databases['console']['dsns']);
145+
$consoledsn = $consoledsn[1] ?? '';
146+
$adapter = new MySQL($this->getConnection($consoledsn));
147+
$dbForConsole = new Database($adapter, $cache);
148+
$dbForConsole->setDefaultDatabase('appwrite');
149+
$dbForConsole->setNamespace('console');
150+
151+
$totalProjects = $dbForConsole->count('projects') + 1;
152+
Console::success("Iterating through : {$totalProjects} projects");
153+
154+
$app = new App('UTC');
155+
$console = $app->getResource('console');
156+
157+
$projects = [$console];
158+
$count = 0;
159+
$limit = 30;
160+
$sum = 30;
161+
$offset = 0;
162+
163+
$stats = [];
164+
165+
while (!empty($projects)) {
166+
foreach ($projects as $project) {
167+
/**
168+
* Skip user projects with id 'console'
169+
*/
170+
if ($project->getId() === 'console') {
171+
continue;
172+
}
173+
Console::info("Getting stats for {$project->getId()}");
174+
175+
try {
176+
// TODO: Iterate through all project DBs
177+
$db = $project->getAttribute('database');
178+
$dsn = $projectdsns[$db] ?? '';
179+
$cache = new Cache(new None());
180+
$adapter = new MySQL($this->getConnection($dsn));
181+
$dbForProject = new Database($adapter, $cache);
182+
$dbForProject->setDefaultDatabase('appwrite');
183+
$dbForProject->setNamespace('_' . $project->getInternalId());
184+
$statsPerProject = $this->getStats($dbForProject);
185+
186+
foreach ($statsPerProject as $key => $value) {
187+
$stats[$key] = isset($stats[$key]) ? $stats[$key] + $value : $value;
188+
}
189+
190+
} catch (\Throwable $th) {
191+
throw $th;
192+
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
193+
}
194+
195+
}
196+
197+
$sum = \count($projects);
198+
199+
$projects = $dbForConsole->find('projects', [
200+
Query::limit($limit),
201+
Query::offset($offset),
202+
]);
203+
204+
$offset = $offset + $limit;
205+
$count = $count + $sum;
206+
207+
Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
208+
}
209+
210+
var_dump($stats);
211+
}
212+
}

0 commit comments

Comments
 (0)