Feature/storage/monitoring tool

.gitignore

@@ -13,6 +13,7 @@ var/
 var_*/
 certs/
 backups/
+logs/
 
 src/static/output.css
 src/static/output.js

docker-compose.yml

@@ -30,6 +30,7 @@ services:
       - .:/app:delegated
       - /tmp/codalab-v2/django:/codalab_tmp
       - ./backups:/app/backups
+      - ./var/logs:/app/logs
     restart: unless-stopped
     ports:
       - 8000:8000

src/apps/analytics/migrations/0001_initial.py

@@ -0,0 +1,62 @@
+# Generated by Django 2.2.17 on 2023-09-14 13:19
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('competitions', '0035_auto_20230914_1319'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AdminStorageDataPoint',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('backups_total', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('at_date', models.DateTimeField()),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='StorageUsageHistory',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('bucket_name', models.CharField(max_length=255)),
+                ('total_usage', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('competitions_usage', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('users_usage', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('admin_usage', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('orphaned_file_usage', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('at_date', models.DateTimeField()),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='UserStorageDataPoint',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('datasets_total', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('submissions_total', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('at_date', models.DateTimeField()),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='CompetitionStorageDataPoint',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('datasets_total', models.DecimalField(blank=True, decimal_places=2, max_digits=14, null=True)),
+                ('at_date', models.DateTimeField()),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('competition', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='competitions.Competition')),
+            ],
+        ),
+    ]

src/apps/analytics/models.py

@@ -0,0 +1,55 @@
+from django.db import models
+from django.conf import settings
+from competitions.models import Competition
+
+
+class StorageUsageHistory(models.Model):
+    bucket_name = models.CharField(max_length=255)
+    total_usage = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )  # in KiB up to ~ 930 TiB
+    competitions_usage = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    users_usage = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    admin_usage = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    orphaned_file_usage = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    at_date = models.DateTimeField()
+    created_at = models.DateTimeField(auto_now_add=True)
+
+
+class CompetitionStorageDataPoint(models.Model):
+    competition = models.ForeignKey(
+        "competitions.competition", null=True, on_delete=models.SET_NULL
+    )
+    datasets_total = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    at_date = models.DateTimeField()
+    created_at = models.DateTimeField(auto_now_add=True)
+
+
+class UserStorageDataPoint(models.Model):
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL)
+    datasets_total = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    submissions_total = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    at_date = models.DateTimeField()
+    created_at = models.DateTimeField(auto_now_add=True)
+
+
+class AdminStorageDataPoint(models.Model):
+    backups_total = models.DecimalField(
+        max_digits=14, decimal_places=2, null=True, blank=True
+    )
+    at_date = models.DateTimeField()
+    created_at = models.DateTimeField(auto_now_add=True)