Skip to content

Commit 3e4ad61

Browse files
shimonewmanshimonewman
committed
usage migration
1 parent 46f07fa commit 3e4ad61

File tree

6 files changed

+211
-23
lines changed

6 files changed

+211
-23
lines changed

‎Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,8 @@ RUN chmod +x /usr/local/bin/doctor && \
138138
chmod +x /usr/local/bin/worker-builds && \
139139
chmod +x /usr/local/bin/worker-mails && \
140140
chmod +x /usr/local/bin/worker-messaging && \
141-
chmod +x /usr/local/bin/worker-webhooks
141+
chmod +x /usr/local/bin/worker-webhooks && \
142+
chmod +x /usr/local/bin/migrate-usage
142143

143144
# Letsencrypt Permissions
144145
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

‎app/config/collections.php

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3104,6 +3104,17 @@
31043104
'array' => false,
31053105
'filters' => [],
31063106
],
3107+
[
3108+
'$id' => ID::custom('metricV2'),
3109+
'type' => Database::VAR_STRING,
3110+
'format' => '',
3111+
'size' => 255,
3112+
'signed' => true,
3113+
'required' => false,
3114+
'default' => null,
3115+
'array' => false,
3116+
'filters' => [],
3117+
],
31073118
[
31083119
'$id' => ID::custom('region'),
31093120
'type' => Database::VAR_STRING,

‎app/console

Submodule console updated 326 files

‎composer.lock

Lines changed: 23 additions & 20 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/Appwrite/Platform/Services/Tasks.php

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
namespace Appwrite\Platform\Services;
44

5+
use Appwrite\Platform\Tasks\MigrateUsage;
56
use Utopia\Platform\Service;
67
use Appwrite\Platform\Tasks\Doctor;
78
use Appwrite\Platform\Tasks\Install;
@@ -37,6 +38,7 @@ public function __construct()
3738
->addAction(Migrate::getName(), new Migrate())
3839
->addAction(SDKs::getName(), new SDKs())
3940
->addAction(VolumeSync::getName(), new VolumeSync())
40-
->addAction(Specs::getName(), new Specs());
41+
->addAction(Specs::getName(), new Specs())
42+
->addAction(MigrateUsage::getName(), new MigrateUsage());
4143
}
4244
}

‎src/Appwrite/Platform/Tasks/MigrateUsage.php

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
<?php
2+
3+
namespace Appwrite\Platform\Tasks;
4+
5+
use Utopia\Database\Database;
6+
use Utopia\Database\Query;
7+
use Utopia\Platform\Action;
8+
use Utopia\CLI\Console;
9+
use Utopia\Database\Validator\Authorization;
10+
use Utopia\Validator\Boolean;
11+
use Utopia\Validator\Text;
12+
13+
class MigrateUsage extends Action
14+
{
15+
public static function getName(): string
16+
{
17+
return 'migrate-usage';
18+
}
19+
20+
public function __construct()
21+
{
22+
$this
23+
->desc('Migrate old usage metric pattern to new')
24+
->param('structure', false, new Boolean(), 'Manipulate usage cols structure', true)
25+
->inject('dbForConsole')
26+
->inject('getProjectDB')
27+
->callback(fn(bool $structure, Database $dbForConsole, callable $getProjectDB) => $this->action($structure, $dbForConsole, $getProjectDB));
28+
}
29+
30+
/**
31+
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
32+
*/
33+
public function action(bool $structure, Database $dbForConsole, callable $getProjectDB): void
34+
{
35+
Authorization::disable();
36+
Authorization::setDefaultStatus(false);
37+
38+
39+
$limit = 100;
40+
$projectCursor = null;
41+
while (true) {
42+
$projectsQueries = [Query::limit($limit)];
43+
if ($projectCursor !== null) {
44+
$projectsQueries[] = Query::cursorAfter($projectCursor);
45+
}
46+
$projects = $dbForConsole->find('projects', $projectsQueries);
47+
48+
if (count($projects) === 0) {
49+
break;
50+
}
51+
52+
foreach ($projects as $project) {
53+
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")");
54+
$dbForProject = $getProjectDB($project);
55+
56+
if ($structure) {
57+
try {
58+
/**
59+
* Update 'mimeType' attribute size (127->255)
60+
*/
61+
$dbForProject->renameAttribute('stats', 'metric', 'metricOld');
62+
$dbForProject->renameAttribute('stats', 'metricV2', 'metric');
63+
} catch (\Throwable $th) {
64+
Console::Error("Error while trying yo rename col: {$th->getMessage()}");
65+
}
66+
67+
Console::log("Renaming stats col metric to metricOld, metricV2 to metric");
68+
continue;
69+
}
70+
71+
72+
/**
73+
* Project level
74+
*/
75+
$metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound'];
76+
$metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound'];
77+
$metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests'];
78+
$metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions'];
79+
$metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute'];
80+
$metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage'];
81+
$metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files'];
82+
$metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets'];
83+
$metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections'];
84+
$metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases'];
85+
$metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users'];
86+
$metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents'];
87+
$metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents'];
88+
89+
foreach ($metrics as $metric) {
90+
$stats = $dbForProject->find('stats', [
91+
Query::equal('metric', [$metric['from']]),
92+
Query::greaterThan('value', 0)
93+
]);
94+
foreach ($stats as $stat) {
95+
$stat->setAttribute('metricV2', $metric['to']);
96+
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
97+
}
98+
}
99+
100+
/**
101+
* Databases, collections, documents
102+
*/
103+
$databases = $dbForProject->find('databases', []);
104+
foreach ($databases as $database) {
105+
$metrics = [];
106+
$metrics[] = [
107+
'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'),
108+
'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')];
109+
$metrics[] = [
110+
'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'),
111+
'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')];
112+
foreach ($metrics as $metric) {
113+
$stats = $dbForProject->find('stats', [
114+
Query::equal('metric', [$metric['from']]),
115+
Query::greaterThan('value', 0)
116+
]);
117+
118+
foreach ($stats as $stat) {
119+
$stat->setAttribute('metricV2', $metric['to']);
120+
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
121+
}
122+
123+
$collections = $dbForProject->find('database_' . $database->getInternalId(), []);
124+
125+
foreach ($collections as $collection) {
126+
$__metric = [
127+
'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'),
128+
'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')];
129+
130+
$__stats = $dbForProject->find('stats', [
131+
Query::equal('metric', [$__metric['from']]),
132+
Query::greaterThan('value', 0)
133+
]);
134+
135+
foreach ($__stats as $__stat) {
136+
$__stat->setAttribute('metricV2', $__metric['to']);
137+
$dbForProject->updateDocument('stats', $__stat->getId(), $__stat);
138+
}
139+
}
140+
}
141+
}
142+
143+
/**
144+
* executions
145+
*/
146+
$functions = $dbForProject->find('functions', []);
147+
foreach ($functions as $function) {
148+
$metrics = [];
149+
$metrics[] = [
150+
'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'),
151+
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute')];
152+
$metrics[] = [
153+
'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'),
154+
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions')];
155+
foreach ($metrics as $metric) {
156+
$stats = $dbForProject->find('stats', [
157+
Query::equal('metric', [$metric['from']]),
158+
Query::greaterThan('value', 0)
159+
]);
160+
161+
foreach ($stats as $stat) {
162+
$stat->setAttribute('metricV2', $metric['to']);
163+
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
164+
}
165+
}
166+
}
167+
$projectCursor = $projects[array_key_last($projects)];
168+
}
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)