From f9648324005e3dca3a1c07a70f2e9921e7d4bab1 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 15 May 2023 23:43:19 -0400 Subject: [PATCH 1/2] feat: adjust models to properly support MySQL This codebase was originally developed and tested using only SQLite. In this commit, we're adding proper support for MySQL. In particular: * The size of ``key`` fields and titles were reduced from 1000 to 500. This is to accommodate MySQL's index size limit (3072 bytes which translates to 768 unicode code points in 4-byte encoding). Saving a little headroom for future compound indexes. * fields.py now has helper classes to support multiple collations so that we can specify utf8mb4 for the charset in MySQL. * fields.py now has helper functions that allow us to normalize case sensitivity across databases. By default, fields would otherwise be case sensitive in SQLite and case insensitive in MySQL. This is important for correctness, since ``key`` fields are meant to be be case sensitive for the purposes of uniqueness constraints. --- .../components/migrations/0001_initial.py | 149 ++----- openedx_learning/core/components/models.py | 10 +- .../core/contents/migrations/0001_initial.py | 112 ++--- openedx_learning/core/contents/models.py | 34 +- .../publishing/migrations/0001_initial.py | 383 ++++-------------- .../0002_add_publish_log_constraints.py | 24 -- openedx_learning/core/publishing/models.py | 9 +- openedx_learning/lib/collations.py | 116 ++++++ openedx_learning/lib/fields.py | 137 +++++-- setup.py | 10 +- 10 files changed, 425 insertions(+), 559 deletions(-) delete mode 100644 openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py create mode 100644 openedx_learning/lib/collations.py diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index ac379c67d..1d3707476 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,153 +1,80 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import uuid class Migration(migrations.Migration): + initial = True dependencies = [ - ("oel_publishing", "0001_initial"), - ("oel_contents", "0001_initial"), + ('oel_publishing', '0001_initial'), + ('oel_contents', '0001_initial'), ] operations = [ migrations.CreateModel( - name="Component", + name='Component', fields=[ - ( - "publishable_entity", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), - ("namespace", models.CharField(max_length=100)), - ("type", models.CharField(blank=True, max_length=100)), - ("local_key", models.CharField(max_length=255)), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('publishable_entity', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), + ('namespace', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100)), + ('type', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100)), + ('local_key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Component", - "verbose_name_plural": "Components", + 'verbose_name': 'Component', + 'verbose_name_plural': 'Components', }, ), migrations.CreateModel( - name="ComponentVersion", + name='ComponentVersion', fields=[ - ( - "publishable_entity_version", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentityversion", - ), - ), - ( - "component", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="versions", - to="oel_components.component", - ), - ), + ('publishable_entity_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentityversion')), + ('component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='oel_components.component')), ], options={ - "verbose_name": "Component Version", - "verbose_name_plural": "Component Versions", + 'verbose_name': 'Component Version', + 'verbose_name_plural': 'Component Versions', }, ), migrations.CreateModel( - name="ComponentVersionRawContent", + name='ComponentVersionRawContent', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ("learner_downloadable", models.BooleanField(default=False)), - ( - "component_version", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_components.componentversion", - ), - ), - ( - "raw_content", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_contents.rawcontent", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('learner_downloadable', models.BooleanField(default=False)), + ('component_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_components.componentversion')), + ('raw_content', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_contents.rawcontent')), ], ), migrations.AddField( - model_name="componentversion", - name="raw_contents", - field=models.ManyToManyField( - related_name="component_versions", - through="oel_components.ComponentVersionRawContent", - to="oel_contents.RawContent", - ), + model_name='componentversion', + name='raw_contents', + field=models.ManyToManyField(related_name='component_versions', through='oel_components.ComponentVersionRawContent', to='oel_contents.RawContent'), ), migrations.AddIndex( - model_name="componentversionrawcontent", - index=models.Index( - fields=["raw_content", "component_version"], - name="oel_cvrawcontent_c_cv", - ), + model_name='componentversionrawcontent', + index=models.Index(fields=['raw_content', 'component_version'], name='oel_cvrawcontent_c_cv'), ), migrations.AddIndex( - model_name="componentversionrawcontent", - index=models.Index( - fields=["component_version", "raw_content"], - name="oel_cvrawcontent_cv_d", - ), + model_name='componentversionrawcontent', + index=models.Index(fields=['component_version', 'raw_content'], name='oel_cvrawcontent_cv_d'), ), migrations.AddConstraint( - model_name="componentversionrawcontent", - constraint=models.UniqueConstraint( - fields=("component_version", "key"), name="oel_cvrawcontent_uniq_cv_key" - ), + model_name='componentversionrawcontent', + constraint=models.UniqueConstraint(fields=('component_version', 'key'), name='oel_cvrawcontent_uniq_cv_key'), ), migrations.AddIndex( - model_name="component", - index=models.Index( - fields=["learning_package", "namespace", "type", "local_key"], - name="oel_component_idx_lc_ns_t_lk", - ), + model_name='component', + index=models.Index(fields=['learning_package', 'namespace', 'type', 'local_key'], name='oel_component_idx_lc_ns_t_lk'), ), migrations.AddConstraint( - model_name="component", - constraint=models.UniqueConstraint( - fields=("learning_package", "namespace", "type", "local_key"), - name="oel_component_uniq_lc_ns_t_lk", - ), + model_name='component', + constraint=models.UniqueConstraint(fields=('learning_package', 'namespace', 'type', 'local_key'), name='oel_component_uniq_lc_ns_t_lk'), ), ] diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index b3a848f78..2d2f21391 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -18,7 +18,11 @@ """ from django.db import models -from openedx_learning.lib.fields import key_field, immutable_uuid_field +from openedx_learning.lib.fields import ( + case_sensitive_char_field, + immutable_uuid_field, + key_field, +) from ..publishing.models import LearningPackage from ..publishing.model_mixins import ( PublishableEntityMixin, @@ -77,13 +81,13 @@ class Component(PublishableEntityMixin): # namespace and type work together to help figure out what Component needs # to handle this data. A namespace is *required*. The namespace for XBlocks # is "xblock.v1" (to match the setup.py entrypoint naming scheme). - namespace = models.CharField(max_length=100, null=False, blank=False) + namespace = case_sensitive_char_field(max_length=100, blank=False) # type is a way to help sub-divide namespace if that's convenient. This # field cannot be null, but it can be blank if it's not necessary. For an # XBlock, type corresponds to tag, e.g. "video". It's also the block_type in # the UsageKey. - type = models.CharField(max_length=100, null=False, blank=True) + type = case_sensitive_char_field(max_length=100, blank=True) # local_key is an identifier that is local to the (namespace, type). The # publishable.key should be calculated as a combination of (namespace, type, diff --git a/openedx_learning/core/contents/migrations/0001_initial.py b/openedx_learning/core/contents/migrations/0001_initial.py index f106553e7..768730389 100644 --- a/openedx_learning/core/contents/migrations/0001_initial.py +++ b/openedx_learning/core/contents/migrations/0001_initial.py @@ -1,103 +1,75 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import openedx_learning.lib.validators +def use_compressed_table_format(apps, schema_editor): + """ + Use the COMPRESSED row format for TextContent if we're using MySQL. + + This table will hold a lot of OLX, which compresses very well using MySQL's + built-in zlib compression. This is especially important because we're + keeping so much version history. + """ + if schema_editor.connection.vendor == 'mysql': + table_name = apps.get_model("oel_contents", "TextContent")._meta.db_table + sql = f"ALTER TABLE {table_name} ROW_FORMAT=COMPRESSED;" + schema_editor.execute(sql) + + class Migration(migrations.Migration): + initial = True dependencies = [ - ("oel_publishing", "0001_initial"), + ('oel_publishing', '0001_initial'), ] operations = [ migrations.CreateModel( - name="RawContent", + name='RawContent', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("hash_digest", models.CharField(editable=False, max_length=40)), - ("mime_type", models.CharField(max_length=255)), - ( - "size", - models.PositiveBigIntegerField( - validators=[django.core.validators.MaxValueValidator(50000000)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ("file", models.FileField(null=True, upload_to="")), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hash_digest', models.CharField(editable=False, max_length=40)), + ('mime_type', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, max_length=255)), + ('size', models.PositiveBigIntegerField(validators=[django.core.validators.MaxValueValidator(50000000)])), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('file', models.FileField(null=True, upload_to='')), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Raw Content", - "verbose_name_plural": "Raw Contents", + 'verbose_name': 'Raw Content', + 'verbose_name_plural': 'Raw Contents', }, ), migrations.CreateModel( - name="TextContent", + name='TextContent', fields=[ - ( - "raw_content", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - related_name="text_content", - serialize=False, - to="oel_contents.rawcontent", - ), - ), - ("text", models.TextField(blank=True, max_length=100000)), - ("length", models.PositiveIntegerField()), + ('raw_content', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='text_content', serialize=False, to='oel_contents.rawcontent')), + ('text', openedx_learning.lib.fields.MultiCollationTextField(blank=True, db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100000)), + ('length', models.PositiveIntegerField()), ], ), + # Call out to custom code here to change row format for TextContent + migrations.RunPython(use_compressed_table_format, reverse_code=migrations.RunPython.noop, atomic=False), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "mime_type"], - name="oel_content_idx_lp_mime_type", - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', 'mime_type'], name='oel_content_idx_lp_mime_type'), ), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-size"], name="oel_content_idx_lp_rsize" - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', '-size'], name='oel_content_idx_lp_rsize'), ), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-created"], - name="oel_content_idx_lp_rcreated", - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', '-created'], name='oel_content_idx_lp_rcreated'), ), migrations.AddConstraint( - model_name="rawcontent", - constraint=models.UniqueConstraint( - fields=("learning_package", "mime_type", "hash_digest"), - name="oel_content_uniq_lc_mime_type_hash_digest", - ), + model_name='rawcontent', + constraint=models.UniqueConstraint(fields=('learning_package', 'mime_type', 'hash_digest'), name='oel_content_uniq_lc_mime_type_hash_digest'), ), ] diff --git a/openedx_learning/core/contents/models.py b/openedx_learning/core/contents/models.py index e19f151cb..6794f1b23 100644 --- a/openedx_learning/core/contents/models.py +++ b/openedx_learning/core/contents/models.py @@ -5,9 +5,15 @@ """ from django.db import models from django.conf import settings +from django.core.files.storage import default_storage from django.core.validators import MaxValueValidator -from openedx_learning.lib.fields import hash_field, manual_date_time_field +from openedx_learning.lib.fields import ( + case_insensitive_char_field, + hash_field, + manual_date_time_field, + MultiCollationTextField, +) from ..publishing.models import LearningPackage @@ -73,11 +79,13 @@ class RawContent(models.Model): # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type # and sub-type may each be 127 chars, making a max of 255 (including the "/" - # in between). + # in between). Also, while MIME types are almost always written in lowercase + # as a matter of convention, by spec they are NOT case sensitive. # # DO NOT STORE parameters here, e.g. "charset=". We can make a new field if - # that becomes necessary. - mime_type = models.CharField(max_length=255, blank=False, null=False) + # that becomes necessary. If we do decide to store parameters and values + # later, note that those *may be* case sensitive. + mime_type = case_insensitive_char_field(max_length=255, blank=False) # This is the size of the raw data file in bytes. This can be different than # the character length, since UTF-8 encoding can use anywhere between 1-4 @@ -96,10 +104,7 @@ class RawContent(models.Model): # models that offer better latency guarantees. file = models.FileField( null=True, - storage=settings.OPENEDX_LEARNING.get( - "STORAGE", - settings.DEFAULT_FILE_STORAGE, - ), + storage=settings.OPENEDX_LEARNING.get("STORAGE", default_storage), ) class Meta: @@ -172,5 +177,16 @@ class TextContent(models.Model): primary_key=True, related_name="text_content", ) - text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) + text = MultiCollationTextField( + blank=True, + max_length=MAX_TEXT_LENGTH, + + # We don't really expect to ever sort by the text column. This is here + # primarily to force the column to be created as utf8mb4 on MySQL. I'm + # using the binary collation because it's a little cheaper/faster. + db_collations={ + "sqlite": "BINARY", + "mysql": "utf8mb4_bin", + } + ) length = models.PositiveIntegerField(null=False) diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 76e02b316..3f41cd45a 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,14 +1,16 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): + initial = True dependencies = [ @@ -17,360 +19,145 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name="LearningPackage", + name='LearningPackage', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ("title", models.CharField(max_length=1000)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "updated", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('title', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, max_length=500)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), ], options={ - "verbose_name": "Learning Package", - "verbose_name_plural": "Learning Packages", + 'verbose_name': 'Learning Package', + 'verbose_name_plural': 'Learning Packages', }, ), migrations.CreateModel( - name="PublishableEntity", + name='PublishableEntity', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "created_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Publishable Entity", - "verbose_name_plural": "Publishable Entities", + 'verbose_name': 'Publishable Entity', + 'verbose_name_plural': 'Publishable Entities', }, ), migrations.CreateModel( - name="PublishableEntityVersion", + name='PublishableEntityVersion', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("title", models.CharField(blank=True, default="", max_length=1000)), - ( - "version_num", - models.PositiveBigIntegerField( - validators=[django.core.validators.MinValueValidator(1)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "created_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "entity", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="versions", - to="oel_publishing.publishableentity", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('title', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, default='', max_length=500)), + ('version_num', models.PositiveBigIntegerField(validators=[django.core.validators.MinValueValidator(1)])), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='oel_publishing.publishableentity')), ], options={ - "verbose_name": "Publishable Entity Version", - "verbose_name_plural": "Publishable Entity Versions", + 'verbose_name': 'Publishable Entity Version', + 'verbose_name_plural': 'Publishable Entity Versions', }, ), migrations.CreateModel( - name="PublishLog", + name='PublishLog', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("message", models.CharField(blank=True, default="", max_length=1000)), - ( - "published_at", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), - ( - "published_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('message', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, default='', max_length=500)), + ('published_at', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), + ('published_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), ], options={ - "verbose_name": "Publish Log", - "verbose_name_plural": "Publish Logs", + 'verbose_name': 'Publish Log', + 'verbose_name_plural': 'Publish Logs', }, ), migrations.CreateModel( - name="Draft", + name='Draft', fields=[ - ( - "entity", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), + ('entity', models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), ], ), migrations.CreateModel( - name="Published", + name='Published', fields=[ - ( - "entity", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), + ('entity', models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), ], options={ - "verbose_name": "Published Entity", - "verbose_name_plural": "Published Entities", + 'verbose_name': 'Published Entity', + 'verbose_name_plural': 'Published Entities', }, ), migrations.CreateModel( - name="PublishLogRecord", + name='PublishLogRecord', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "entity", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentity", - ), - ), - ( - "new_version", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), - ), - ( - "old_version", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - related_name="+", - to="oel_publishing.publishableentityversion", - ), - ), - ( - "publish_log", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.publishlog", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entity', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentity')), + ('new_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion')), + ('old_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, related_name='+', to='oel_publishing.publishableentityversion')), + ('publish_log', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishlog')), ], options={ - "verbose_name": "Publish Log Record", - "verbose_name_plural": "Publish Log Records", + 'verbose_name': 'Publish Log Record', + 'verbose_name_plural': 'Publish Log Records', }, ), migrations.AddConstraint( - model_name="learningpackage", - constraint=models.UniqueConstraint( - fields=("key",), name="oel_publishing_lp_uniq_key" - ), + model_name='learningpackage', + constraint=models.UniqueConstraint(fields=('key',), name='oel_publishing_lp_uniq_key'), + ), + migrations.AddIndex( + model_name='publishlogrecord', + index=models.Index(fields=['entity', '-publish_log'], name='oel_plr_idx_entity_rplr'), + ), + migrations.AddConstraint( + model_name='publishlogrecord', + constraint=models.UniqueConstraint(fields=('publish_log', 'entity'), name='oel_plr_uniq_pl_publishable'), ), migrations.AddField( - model_name="published", - name="publish_log_record", - field=models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishlogrecord", - ), + model_name='published', + name='publish_log_record', + field=models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishlogrecord'), ), migrations.AddField( - model_name="published", - name="version", - field=models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), + model_name='published', + name='version', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion'), ), migrations.AddIndex( - model_name="publishableentityversion", - index=models.Index( - fields=["entity", "-created"], name="oel_pv_idx_entity_rcreated" - ), + model_name='publishableentityversion', + index=models.Index(fields=['entity', '-created'], name='oel_pv_idx_entity_rcreated'), ), migrations.AddIndex( - model_name="publishableentityversion", - index=models.Index(fields=["title"], name="oel_pv_idx_title"), + model_name='publishableentityversion', + index=models.Index(fields=['title'], name='oel_pv_idx_title'), ), migrations.AddConstraint( - model_name="publishableentityversion", - constraint=models.UniqueConstraint( - fields=("entity", "version_num"), name="oel_pv_uniq_entity_version_num" - ), + model_name='publishableentityversion', + constraint=models.UniqueConstraint(fields=('entity', 'version_num'), name='oel_pv_uniq_entity_version_num'), ), migrations.AddIndex( - model_name="publishableentity", - index=models.Index(fields=["key"], name="oel_pub_ent_idx_key"), + model_name='publishableentity', + index=models.Index(fields=['key'], name='oel_pub_ent_idx_key'), ), migrations.AddIndex( - model_name="publishableentity", - index=models.Index( - fields=["learning_package", "-created"], - name="oel_pub_ent_idx_lp_rcreated", - ), + model_name='publishableentity', + index=models.Index(fields=['learning_package', '-created'], name='oel_pub_ent_idx_lp_rcreated'), ), migrations.AddConstraint( - model_name="publishableentity", - constraint=models.UniqueConstraint( - fields=("learning_package", "key"), name="oel_pub_ent_uniq_lp_key" - ), + model_name='publishableentity', + constraint=models.UniqueConstraint(fields=('learning_package', 'key'), name='oel_pub_ent_uniq_lp_key'), ), migrations.AddField( - model_name="draft", - name="version", - field=models.OneToOneField( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), + model_name='draft', + name='version', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion'), ), ] diff --git a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py b/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py deleted file mode 100644 index 986683c47..000000000 --- a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.2.19 on 2023-05-14 13:58 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("oel_publishing", "0001_initial"), - ] - - operations = [ - migrations.AddIndex( - model_name="publishlogrecord", - index=models.Index( - fields=["entity", "-publish_log"], name="oel_plr_idx_entity_rplr" - ), - ), - migrations.AddConstraint( - model_name="publishlogrecord", - constraint=models.UniqueConstraint( - fields=("publish_log", "entity"), name="oel_plr_uniq_pl_publishable" - ), - ), - ] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 0f9a50fdd..78f19b4ba 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -17,8 +17,9 @@ from django.core.validators import MinValueValidator from openedx_learning.lib.fields import ( - key_field, + case_insensitive_char_field, immutable_uuid_field, + key_field, manual_date_time_field, ) @@ -32,7 +33,7 @@ class LearningPackage(models.Model): uuid = immutable_uuid_field() key = key_field() - title = models.CharField(max_length=1000, null=False, blank=False) + title = case_insensitive_char_field(max_length=500, blank=False) created = manual_date_time_field() updated = manual_date_time_field() @@ -213,7 +214,7 @@ class PublishableEntityVersion(models.Model): # Most publishable things will have some sort of title, but blanks are # allowed for those that don't require one. - title = models.CharField(max_length=1000, default="", null=False, blank=True) + title = case_insensitive_char_field(max_length=500, blank=True, default="") # The version_num starts at 1 and increments by 1 with each new version for # a given PublishableEntity. Doing it this way makes it more convenient for @@ -339,7 +340,7 @@ class PublishLog(models.Model): uuid = immutable_uuid_field() learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) - message = models.CharField(max_length=1000, null=False, blank=True, default="") + message = case_insensitive_char_field(max_length=500, blank=True, default="") published_at = manual_date_time_field() published_by = models.ForeignKey( settings.AUTH_USER_MODEL, diff --git a/openedx_learning/lib/collations.py b/openedx_learning/lib/collations.py new file mode 100644 index 000000000..2afdcc377 --- /dev/null +++ b/openedx_learning/lib/collations.py @@ -0,0 +1,116 @@ +""" +This module has collation-related code to allow us to attach collation settings +to specific fields on a per-database-vendor basis. This used by the ``fields`` +module in order to specify field types have have normalized behavior between +SQLite and MySQL (see fields.py for more details). +""" +from django.db import models + + +class MultiCollationMixin: + """ + Mixin to enable multiple, database-vendor-specific collations. + + This should be mixed into new subclasses of CharField and TextField, since + they're the only Field types that store text data. + """ + + def __init__(self, *args, db_collations=None, db_collation=None, **kwargs): + """ + Init like any field but add db_collations and disallow db_collation + + The ``db_collations`` param should be a dict of vendor names to + collations, like:: + + { + 'msyql': 'utf8mb4_bin', + 'sqlite': 'BINARY' + } + + It is an error to pass in a CharField-style ``db_collation``. I + originally wanted to use this attribute name, but I needed to preserve + it for Django 3.2 compatibility (see the ``db_collation`` method + docstring for details). + """ + if db_collation is not None: + raise ValueError( + f"Cannot use db_collation with {self.__class__.__name__}. " + + "Please use a db_collations dict instead." + ) + + super().__init__(*args, **kwargs) + self.db_collations = db_collations or {} + + # This is part of a hack to get this to work for Django < 4.1. Please + # see comments in the db_collation method for details. + self._vendor = None + + @property + def db_collation(self): + """ + Return the db_collation, understanding that it varies by vendor. + + This method is a hack for Django 3.2 compatibility and should be removed + after we move to 4.2. + + Description of why this is hacky: + + In Django 4.2, the schema builder pulls the collation settings from the + field using the value returned from the ``db_parameters`` method, and + this does what we want it to do. In Django 3.2, field.db_parameters is + called, but any collation value sent back is ignored and the code grabs + the value of db_collation directly from the field: + + https://github.com/django/django/blob/stable/3.2.x/django/db/backends/base/schema.py#L214-L224 + + But this call to get the ``field.db_collation`` attribute happens almost + immediately after the ``field.db_parameters`` method call. So our + fragile hack is to set ``self._vendor`` in the ``db_parameters`` method, + using the value we get from the connection that is passed in there. We + can then use ``self._vendor`` to return the right value when Django + calls ``field.db_collation`` (which is this property method). + + This method, the corresponding setter, and all references to + ``self._vendor`` should be removed after we've cut over to Django 4.2. + """ + return self.db_collations.get(self._vendor) + + @db_collation.setter + def db_collation(self, value): + """ + Don't allow db_collation to be set manually (just ignore). + + This can be removed when we move to Django 4.2. + """ + pass + + def db_parameters(self, connection): + """ + Return database parameters for this field. This adds collation info. + + We examine this field's ``db_collations`` attribute and return the + collation that maps to ``connection.vendor``. This will typically be + 'mysql' or 'sqlite'. + """ + db_params = models.Field.db_parameters(self, connection) + + # Remove once we no longer need to support Django < 4.1 + self._vendor = connection.vendor + + # Now determine collation based on DB vendor (e.g. 'sqlite', 'mysql') + if connection.vendor in self.db_collations: + db_params["collation"] = self.db_collations[connection.vendor] + + return db_params + + def deconstruct(self): + """ + How to serialize our Field for the migration file. + + For our mixin fields, this is just doing what the field's superclass + would do and then tacking on our custom ``db_collations`` dict data. + """ + name, path, args, kwargs = super().deconstruct() + if self.db_collations: + kwargs["db_collations"] = self.db_collations + return name, path, args, kwargs diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index 71273afa0..0e231e3a6 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -1,51 +1,82 @@ """ Convenience functions to make consistent field conventions easier. -Field conventions: - -* Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a - primary key + separate UUID column. +Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a +primary key + separate UUID column. https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html -* The UUID fields are intended to be globally unique identifiers that other - services can store and rely on staying the same. -* The "key" fields can be more human-friendly strings, but these may only - be unique within a given scope. These values should be treated as mutable, - even if they rarely change in practice. - -TODO: -* Try making a CaseSensitiveCharField and CaseInsensitiveCharField -* Investigate more efficient UUID binary encoding + search in MySQL - -Other data thoughts: -* It would be good to make a data-dumping sort of script that exported part of - the data to SQLite3. -* identifiers support stable import/export with serialized formats -* UUIDs will import/export in the SQLite3 dumps, but are not there in other contexts + +We have helpers to make case sensitivity consistent across backends. MySQL is +case-insensitive by default, SQLite and Postgres are case-sensitive. + + """ import hashlib import uuid from django.db import models +from .collations import MultiCollationMixin from .validators import validate_utc_datetime -def key_field(): +def create_hash_digest(data_bytes): + return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() + + +def case_insensitive_char_field(**kwargs): """ - Externally created Identifier fields. + Return a case-insensitive ``MultiCollationCharField``. - These will often be local to a particular scope, like within a - LearningPackage. It's up to the application as to whether they're - semantically meaningful or look more machine-generated. + This means that entries will sort in a case-insensitive manner, and that + unique indexes will be case insensitive, e.g. you would not be able to + insert "abc" and "ABC" into the same table field if you put a unique index + on this field. - Other apps should *not* make references to these values directly, since - these values may in theory change. + You may override any argument that you would normally pass into + ``MultiCollationCharField`` (which is itself a subclass of ``CharField``). """ - return models.CharField( - max_length=255, - blank=False, - null=False, - ) + # Set our default arguments + final_kwargs = { + "null": False, + "db_collations": { + "sqlite": "NOCASE", + # We're using utf8mb4_unicode_ci to keep MariaDB compatibility, + # since their collation support diverges after this. MySQL is now on + # utf8mb4_0900_ai_ci based on Unicode 9, while MariaDB has + # uca1400_ai_ci based on Unicode 14. + "mysql": "utf8mb4_unicode_ci", + }, + } + # Override our defaults with whatever is passed in. + final_kwargs.update(kwargs) + + return MultiCollationCharField(**final_kwargs) + + +def case_sensitive_char_field(**kwargs): + """ + Return a case-sensitive ``MultiCollationCharField``. + + This means that entries will sort in a case-sensitive manner, and that + unique indexes will be case sensitive, e.g. "abc" and "ABC" would be + distinct and you would not get a unique constraint violation by adding them + both to the same table field. + + You may override any argument that you would normally pass into + ``MultiCollationCharField`` (which is itself a subclass of ``CharField``). + """ + # Set our default arguments + final_kwargs = { + "null": False, + "db_collations": { + "sqlite": "BINARY", + "mysql": "utf8mb4_bin", + }, + } + # Override our defaults with whatever is passed in. + final_kwargs.update(kwargs) + + return MultiCollationCharField(**final_kwargs) def immutable_uuid_field(): @@ -66,6 +97,20 @@ def immutable_uuid_field(): ) +def key_field(): + """ + Externally created Identifier fields. + + These will often be local to a particular scope, like within a + LearningPackage. It's up to the application as to whether they're + semantically meaningful or look more machine-generated. + + Other apps should *not* make references to these values directly, since + these values may in theory change (even if this is rare in practice). + """ + return case_sensitive_char_field(max_length=500, blank=False) + + def hash_field(): """ Holds a hash digest meant to identify a piece of content. @@ -85,10 +130,6 @@ def hash_field(): ) -def create_hash_digest(data_bytes): - return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() - - def manual_date_time_field(): """ DateTimeField that does not auto-generate values. @@ -117,3 +158,29 @@ def manual_date_time_field(): validate_utc_datetime, ], ) + + +class MultiCollationCharField(MultiCollationMixin, models.CharField): + """ + CharField subclass with per-database-vendor collation settings. + + Django's CharField already supports specifying the database collation, but + that only works with a single value. So there would be no way to say, "Use + utf8mb4_bin for MySQL, and BINARY if we're running SQLite." This is a + problem because we run tests in SQLite (and may potentially run more later). + It's also a problem if we ever want to support other database backends, like + PostgreSQL. Even MariaDB is starting to diverge from MySQL in terms of what + collations are supported. + """ + pass + + +class MultiCollationTextField(MultiCollationMixin, models.TextField): + """ + TextField subclass with per-database-vendor collation settings. + + We don't ever really want to _sort_ by a TextField, but setting a collation + forces the compatible charset to be set in MySQL, and that's the part that + matters for our purposes. So for example, if you set + """ + pass diff --git a/setup.py b/setup.py index 4fef021c4..ab642620a 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import re import sys -from setuptools import setup +from setuptools import find_packages, setup def get_version(*file_paths): @@ -72,10 +72,10 @@ def is_requirement(line): author='David Ormsbee', author_email='dave@tcril.org', url='https://github.com/openedx/openedx-learning', - packages=[ - 'openedx_learning', - 'openedx_tagging', - ], + packages=find_packages( + include=['openedx_learning*', 'openedx_tagging*'], + exclude=['*.test', '*.tests'] + ), include_package_data=True, install_requires=load_requirements('requirements/base.in'), python_requires=">=3.8", From 7aa277613e235009fe396f8451a7174b70bb8c88 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 20 Jun 2023 12:19:32 -0400 Subject: [PATCH 2/2] test: use MySQL when running unit tests in CI This will run all CI tests against MySQL 8.0. It changes the tox py38-django{32|42} envs to use a new MySQL settings file. Running pytest manually will continue to use the in-memory SQLite database. --- .github/workflows/ci.yml | 23 ++++++++++++++++++++++- mysql_test_settings.py | 26 ++++++++++++++++++++++++++ requirements/base.txt | 4 +++- requirements/ci.txt | 8 ++++---- requirements/dev.txt | 39 +++++++++++++++++++++------------------ requirements/doc.txt | 23 +++++++++++++---------- requirements/quality.txt | 33 ++++++++++++++++++--------------- requirements/test.in | 2 ++ requirements/test.txt | 16 ++++++++++------ tox.ini | 4 ++++ 10 files changed, 123 insertions(+), 55 deletions(-) create mode 100644 mysql_test_settings.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eff1b8e4..432111450 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,30 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] # Add macos-latest later? python-version: ['3.8'] toxenv: ["py38-django32", "py38-django42"] + # We're only testing against MySQL 8 right now because 5.7 is + # incompatible with Djagno 4.2. We'd have to make the tox.ini file more + # complicated than it's worth given the short expected shelf-life of + # MySQL 5.7 in our stack. + mysql-version: ["8"] + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "test_oel_db" + MYSQL_USER: "test_oel_user" + MYSQL_PASSWORD: "test_oel_pass" + MYSQL_RANDOM_ROOT_PASSWORD: true + # these options are blatantly copied from edx-platform's values + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: - uses: actions/checkout@v3 - name: setup python diff --git a/mysql_test_settings.py b/mysql_test_settings.py new file mode 100644 index 000000000..ae8e37d3c --- /dev/null +++ b/mysql_test_settings.py @@ -0,0 +1,26 @@ +""" +This is an extension of the default test_settings.py file that uses MySQL for +the backend. While the openedx-learning apps should run fine using SQLite, they +also do some MySQL-specific things around charset/collation settings and row +compression. + +The tox targets for py38-django32 and py38-django42 will use this settings file. +For the most part, you can use test_settings.py instead (that's the default if +you just run "pytest" with no arguments). +""" + +from test_settings import * + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "oel_db", + "USER": "test_oel_user", + "PASSWORD": "test_oel_pass", + "HOST": "127.0.0.1", + "PORT": "3306", + "OPTIONS": { + "charset": "utf8mb4" + } + } +} diff --git a/requirements/base.txt b/requirements/base.txt index d48934464..97ec03540 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via django django==3.2.19 # via @@ -19,3 +19,5 @@ pytz==2023.3 # djangorestframework sqlparse==0.4.4 # via django +typing-extensions==4.6.3 + # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index d851c07e2..6e1dd7f92 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ click==8.1.3 # via import-linter distlib==0.3.6 # via virtualenv -filelock==3.12.0 +filelock==3.12.2 # via # tox # virtualenv @@ -18,7 +18,7 @@ import-linter==1.9.0 # via -r requirements/ci.in packaging==23.1 # via tox -platformdirs==3.5.1 +platformdirs==3.6.0 # via virtualenv pluggy==1.0.0 # via tox @@ -34,9 +34,9 @@ tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # grimp # import-linter -virtualenv==20.23.0 +virtualenv==20.23.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 9f6bd5332..b2dcdaf2e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/quality.txt # django @@ -49,11 +49,11 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/quality.txt # pytest-cov -diff-cover==7.5.0 +diff-cover==7.6.0 # via -r requirements/dev.in dill==0.3.6 # via @@ -86,7 +86,7 @@ exceptiongroup==1.1.1 # via # -r requirements/quality.txt # pytest -filelock==3.12.0 +filelock==3.12.2 # via # -r requirements/ci.txt # tox @@ -104,7 +104,7 @@ import-linter==1.9.0 # via # -r requirements/ci.txt # -r requirements/quality.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # -r requirements/quality.txt # keyring @@ -130,7 +130,7 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.13.1 +keyring==24.0.0 # via # -r requirements/quality.txt # twine @@ -138,11 +138,11 @@ lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/quality.txt # jinja2 @@ -158,6 +158,8 @@ more-itertools==9.1.0 # via # -r requirements/quality.txt # jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/quality.txt packaging==23.1 # via # -r requirements/ci.txt @@ -178,7 +180,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -222,7 +224,7 @@ pylint-django==2.5.3 # via # -r requirements/quality.txt # edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # -r requirements/quality.txt # pylint-celery @@ -231,12 +233,12 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt @@ -254,11 +256,11 @@ pyyaml==6.0 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==37.3 +readme-renderer==40.0 # via # -r requirements/quality.txt # twine -requests==2.30.0 +requests==2.31.0 # via # -r requirements/quality.txt # requests-toolbelt @@ -271,7 +273,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.3.5 +rich==13.4.2 # via # -r requirements/quality.txt # twine @@ -324,21 +326,22 @@ tox-battery==0.6.1 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/ci.txt # -r requirements/quality.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # -r requirements/quality.txt # requests # twine -virtualenv==20.23.0 +virtualenv==20.23.1 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index aa92d8e59..d475be069 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -66,7 +66,7 @@ imagesize==1.4.1 # via sphinx import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via sphinx iniconfig==2.0.0 # via @@ -77,10 +77,12 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,12 +108,12 @@ pygments==2.15.1 # pydata-sphinx-theme # readme-renderer # sphinx -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -129,9 +131,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via -r requirements/doc.in -requests==2.30.0 +requests==2.31.0 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -183,13 +185,14 @@ tomli==2.0.1 # doc8 # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # grimp # import-linter # pydata-sphinx-theme -urllib3==2.0.2 +urllib3==2.0.3 # via requests webencodings==0.5.1 # via bleach diff --git a/requirements/quality.txt b/requirements/quality.txt index b36f07abd..ab8f3977d 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -60,7 +60,7 @@ idna==3.4 # via requests import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # keyring # twine @@ -80,13 +80,13 @@ jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.13.1 +keyring==24.0.0 # via twine lazy-object-proxy==1.9.0 # via astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 @@ -96,6 +96,8 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==9.1.0 # via jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,7 +108,7 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via pylint pluggy==1.0.0 # via @@ -130,16 +132,16 @@ pylint-celery==0.3 # via edx-lint pylint-django==2.5.3 # via edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -156,9 +158,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via twine -requests==2.30.0 +requests==2.31.0 # via # requests-toolbelt # twine @@ -166,7 +168,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.5 +rich==13.4.2 # via twine six==1.16.0 # via @@ -197,15 +199,16 @@ tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/quality.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # requests # twine diff --git a/requirements/test.in b/requirements/test.in index 65ac63e81..48446899b 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -3,6 +3,8 @@ -r base.txt # Core dependencies for this package +mysqlclient<3.0 # MySQL support + coverage # Code coverage reporting import-linter # Track our internal dependencies diff --git a/requirements/test.txt b/requirements/test.txt index ae4cfa040..8865d16b8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/base.txt # django @@ -14,7 +14,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.in # pytest-cov @@ -34,20 +34,22 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.in packaging==23.1 # via pytest pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.in # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in @@ -73,7 +75,9 @@ tomli==2.0.1 # coverage # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via + # -r requirements/base.txt + # asgiref # grimp # import-linter diff --git a/tox.ini b/tox.ini index 362718418..23e2650ad 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,10 @@ deps = django32: Django>=3.2,<3.3 django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt +setenv = + # Note that the django32/django42 targets use MySQL, but running pytest by + # itself (without tox) will run tests in SQLite for developer convenience. + DJANGO_SETTINGS_MODULE = mysql_test_settings commands = pytest {posargs}