diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eff1b8e4..432111450 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,30 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] # Add macos-latest later? python-version: ['3.8'] toxenv: ["py38-django32", "py38-django42"] + # We're only testing against MySQL 8 right now because 5.7 is + # incompatible with Djagno 4.2. We'd have to make the tox.ini file more + # complicated than it's worth given the short expected shelf-life of + # MySQL 5.7 in our stack. + mysql-version: ["8"] + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "test_oel_db" + MYSQL_USER: "test_oel_user" + MYSQL_PASSWORD: "test_oel_pass" + MYSQL_RANDOM_ROOT_PASSWORD: true + # these options are blatantly copied from edx-platform's values + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: - uses: actions/checkout@v3 - name: setup python diff --git a/mysql_test_settings.py b/mysql_test_settings.py new file mode 100644 index 000000000..ae8e37d3c --- /dev/null +++ b/mysql_test_settings.py @@ -0,0 +1,26 @@ +""" +This is an extension of the default test_settings.py file that uses MySQL for +the backend. While the openedx-learning apps should run fine using SQLite, they +also do some MySQL-specific things around charset/collation settings and row +compression. + +The tox targets for py38-django32 and py38-django42 will use this settings file. +For the most part, you can use test_settings.py instead (that's the default if +you just run "pytest" with no arguments). +""" + +from test_settings import * + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "oel_db", + "USER": "test_oel_user", + "PASSWORD": "test_oel_pass", + "HOST": "127.0.0.1", + "PORT": "3306", + "OPTIONS": { + "charset": "utf8mb4" + } + } +} diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index ac379c67d..1d3707476 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,153 +1,80 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import uuid class Migration(migrations.Migration): + initial = True dependencies = [ - ("oel_publishing", "0001_initial"), - ("oel_contents", "0001_initial"), + ('oel_publishing', '0001_initial'), + ('oel_contents', '0001_initial'), ] operations = [ migrations.CreateModel( - name="Component", + name='Component', fields=[ - ( - "publishable_entity", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), - ("namespace", models.CharField(max_length=100)), - ("type", models.CharField(blank=True, max_length=100)), - ("local_key", models.CharField(max_length=255)), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('publishable_entity', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), + ('namespace', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100)), + ('type', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100)), + ('local_key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Component", - "verbose_name_plural": "Components", + 'verbose_name': 'Component', + 'verbose_name_plural': 'Components', }, ), migrations.CreateModel( - name="ComponentVersion", + name='ComponentVersion', fields=[ - ( - "publishable_entity_version", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentityversion", - ), - ), - ( - "component", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="versions", - to="oel_components.component", - ), - ), + ('publishable_entity_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentityversion')), + ('component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='oel_components.component')), ], options={ - "verbose_name": "Component Version", - "verbose_name_plural": "Component Versions", + 'verbose_name': 'Component Version', + 'verbose_name_plural': 'Component Versions', }, ), migrations.CreateModel( - name="ComponentVersionRawContent", + name='ComponentVersionRawContent', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ("learner_downloadable", models.BooleanField(default=False)), - ( - "component_version", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_components.componentversion", - ), - ), - ( - "raw_content", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_contents.rawcontent", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('learner_downloadable', models.BooleanField(default=False)), + ('component_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_components.componentversion')), + ('raw_content', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_contents.rawcontent')), ], ), migrations.AddField( - model_name="componentversion", - name="raw_contents", - field=models.ManyToManyField( - related_name="component_versions", - through="oel_components.ComponentVersionRawContent", - to="oel_contents.RawContent", - ), + model_name='componentversion', + name='raw_contents', + field=models.ManyToManyField(related_name='component_versions', through='oel_components.ComponentVersionRawContent', to='oel_contents.RawContent'), ), migrations.AddIndex( - model_name="componentversionrawcontent", - index=models.Index( - fields=["raw_content", "component_version"], - name="oel_cvrawcontent_c_cv", - ), + model_name='componentversionrawcontent', + index=models.Index(fields=['raw_content', 'component_version'], name='oel_cvrawcontent_c_cv'), ), migrations.AddIndex( - model_name="componentversionrawcontent", - index=models.Index( - fields=["component_version", "raw_content"], - name="oel_cvrawcontent_cv_d", - ), + model_name='componentversionrawcontent', + index=models.Index(fields=['component_version', 'raw_content'], name='oel_cvrawcontent_cv_d'), ), migrations.AddConstraint( - model_name="componentversionrawcontent", - constraint=models.UniqueConstraint( - fields=("component_version", "key"), name="oel_cvrawcontent_uniq_cv_key" - ), + model_name='componentversionrawcontent', + constraint=models.UniqueConstraint(fields=('component_version', 'key'), name='oel_cvrawcontent_uniq_cv_key'), ), migrations.AddIndex( - model_name="component", - index=models.Index( - fields=["learning_package", "namespace", "type", "local_key"], - name="oel_component_idx_lc_ns_t_lk", - ), + model_name='component', + index=models.Index(fields=['learning_package', 'namespace', 'type', 'local_key'], name='oel_component_idx_lc_ns_t_lk'), ), migrations.AddConstraint( - model_name="component", - constraint=models.UniqueConstraint( - fields=("learning_package", "namespace", "type", "local_key"), - name="oel_component_uniq_lc_ns_t_lk", - ), + model_name='component', + constraint=models.UniqueConstraint(fields=('learning_package', 'namespace', 'type', 'local_key'), name='oel_component_uniq_lc_ns_t_lk'), ), ] diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index b3a848f78..2d2f21391 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -18,7 +18,11 @@ """ from django.db import models -from openedx_learning.lib.fields import key_field, immutable_uuid_field +from openedx_learning.lib.fields import ( + case_sensitive_char_field, + immutable_uuid_field, + key_field, +) from ..publishing.models import LearningPackage from ..publishing.model_mixins import ( PublishableEntityMixin, @@ -77,13 +81,13 @@ class Component(PublishableEntityMixin): # namespace and type work together to help figure out what Component needs # to handle this data. A namespace is *required*. The namespace for XBlocks # is "xblock.v1" (to match the setup.py entrypoint naming scheme). - namespace = models.CharField(max_length=100, null=False, blank=False) + namespace = case_sensitive_char_field(max_length=100, blank=False) # type is a way to help sub-divide namespace if that's convenient. This # field cannot be null, but it can be blank if it's not necessary. For an # XBlock, type corresponds to tag, e.g. "video". It's also the block_type in # the UsageKey. - type = models.CharField(max_length=100, null=False, blank=True) + type = case_sensitive_char_field(max_length=100, blank=True) # local_key is an identifier that is local to the (namespace, type). The # publishable.key should be calculated as a combination of (namespace, type, diff --git a/openedx_learning/core/contents/migrations/0001_initial.py b/openedx_learning/core/contents/migrations/0001_initial.py index f106553e7..768730389 100644 --- a/openedx_learning/core/contents/migrations/0001_initial.py +++ b/openedx_learning/core/contents/migrations/0001_initial.py @@ -1,103 +1,75 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import openedx_learning.lib.validators +def use_compressed_table_format(apps, schema_editor): + """ + Use the COMPRESSED row format for TextContent if we're using MySQL. + + This table will hold a lot of OLX, which compresses very well using MySQL's + built-in zlib compression. This is especially important because we're + keeping so much version history. + """ + if schema_editor.connection.vendor == 'mysql': + table_name = apps.get_model("oel_contents", "TextContent")._meta.db_table + sql = f"ALTER TABLE {table_name} ROW_FORMAT=COMPRESSED;" + schema_editor.execute(sql) + + class Migration(migrations.Migration): + initial = True dependencies = [ - ("oel_publishing", "0001_initial"), + ('oel_publishing', '0001_initial'), ] operations = [ migrations.CreateModel( - name="RawContent", + name='RawContent', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("hash_digest", models.CharField(editable=False, max_length=40)), - ("mime_type", models.CharField(max_length=255)), - ( - "size", - models.PositiveBigIntegerField( - validators=[django.core.validators.MaxValueValidator(50000000)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ("file", models.FileField(null=True, upload_to="")), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hash_digest', models.CharField(editable=False, max_length=40)), + ('mime_type', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, max_length=255)), + ('size', models.PositiveBigIntegerField(validators=[django.core.validators.MaxValueValidator(50000000)])), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('file', models.FileField(null=True, upload_to='')), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Raw Content", - "verbose_name_plural": "Raw Contents", + 'verbose_name': 'Raw Content', + 'verbose_name_plural': 'Raw Contents', }, ), migrations.CreateModel( - name="TextContent", + name='TextContent', fields=[ - ( - "raw_content", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - related_name="text_content", - serialize=False, - to="oel_contents.rawcontent", - ), - ), - ("text", models.TextField(blank=True, max_length=100000)), - ("length", models.PositiveIntegerField()), + ('raw_content', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='text_content', serialize=False, to='oel_contents.rawcontent')), + ('text', openedx_learning.lib.fields.MultiCollationTextField(blank=True, db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100000)), + ('length', models.PositiveIntegerField()), ], ), + # Call out to custom code here to change row format for TextContent + migrations.RunPython(use_compressed_table_format, reverse_code=migrations.RunPython.noop, atomic=False), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "mime_type"], - name="oel_content_idx_lp_mime_type", - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', 'mime_type'], name='oel_content_idx_lp_mime_type'), ), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-size"], name="oel_content_idx_lp_rsize" - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', '-size'], name='oel_content_idx_lp_rsize'), ), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-created"], - name="oel_content_idx_lp_rcreated", - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', '-created'], name='oel_content_idx_lp_rcreated'), ), migrations.AddConstraint( - model_name="rawcontent", - constraint=models.UniqueConstraint( - fields=("learning_package", "mime_type", "hash_digest"), - name="oel_content_uniq_lc_mime_type_hash_digest", - ), + model_name='rawcontent', + constraint=models.UniqueConstraint(fields=('learning_package', 'mime_type', 'hash_digest'), name='oel_content_uniq_lc_mime_type_hash_digest'), ), ] diff --git a/openedx_learning/core/contents/models.py b/openedx_learning/core/contents/models.py index e19f151cb..6794f1b23 100644 --- a/openedx_learning/core/contents/models.py +++ b/openedx_learning/core/contents/models.py @@ -5,9 +5,15 @@ """ from django.db import models from django.conf import settings +from django.core.files.storage import default_storage from django.core.validators import MaxValueValidator -from openedx_learning.lib.fields import hash_field, manual_date_time_field +from openedx_learning.lib.fields import ( + case_insensitive_char_field, + hash_field, + manual_date_time_field, + MultiCollationTextField, +) from ..publishing.models import LearningPackage @@ -73,11 +79,13 @@ class RawContent(models.Model): # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type # and sub-type may each be 127 chars, making a max of 255 (including the "/" - # in between). + # in between). Also, while MIME types are almost always written in lowercase + # as a matter of convention, by spec they are NOT case sensitive. # # DO NOT STORE parameters here, e.g. "charset=". We can make a new field if - # that becomes necessary. - mime_type = models.CharField(max_length=255, blank=False, null=False) + # that becomes necessary. If we do decide to store parameters and values + # later, note that those *may be* case sensitive. + mime_type = case_insensitive_char_field(max_length=255, blank=False) # This is the size of the raw data file in bytes. This can be different than # the character length, since UTF-8 encoding can use anywhere between 1-4 @@ -96,10 +104,7 @@ class RawContent(models.Model): # models that offer better latency guarantees. file = models.FileField( null=True, - storage=settings.OPENEDX_LEARNING.get( - "STORAGE", - settings.DEFAULT_FILE_STORAGE, - ), + storage=settings.OPENEDX_LEARNING.get("STORAGE", default_storage), ) class Meta: @@ -172,5 +177,16 @@ class TextContent(models.Model): primary_key=True, related_name="text_content", ) - text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) + text = MultiCollationTextField( + blank=True, + max_length=MAX_TEXT_LENGTH, + + # We don't really expect to ever sort by the text column. This is here + # primarily to force the column to be created as utf8mb4 on MySQL. I'm + # using the binary collation because it's a little cheaper/faster. + db_collations={ + "sqlite": "BINARY", + "mysql": "utf8mb4_bin", + } + ) length = models.PositiveIntegerField(null=False) diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 76e02b316..3f41cd45a 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,14 +1,16 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): + initial = True dependencies = [ @@ -17,360 +19,145 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name="LearningPackage", + name='LearningPackage', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ("title", models.CharField(max_length=1000)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "updated", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('title', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, max_length=500)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), ], options={ - "verbose_name": "Learning Package", - "verbose_name_plural": "Learning Packages", + 'verbose_name': 'Learning Package', + 'verbose_name_plural': 'Learning Packages', }, ), migrations.CreateModel( - name="PublishableEntity", + name='PublishableEntity', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "created_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Publishable Entity", - "verbose_name_plural": "Publishable Entities", + 'verbose_name': 'Publishable Entity', + 'verbose_name_plural': 'Publishable Entities', }, ), migrations.CreateModel( - name="PublishableEntityVersion", + name='PublishableEntityVersion', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("title", models.CharField(blank=True, default="", max_length=1000)), - ( - "version_num", - models.PositiveBigIntegerField( - validators=[django.core.validators.MinValueValidator(1)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "created_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "entity", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="versions", - to="oel_publishing.publishableentity", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('title', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, default='', max_length=500)), + ('version_num', models.PositiveBigIntegerField(validators=[django.core.validators.MinValueValidator(1)])), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='oel_publishing.publishableentity')), ], options={ - "verbose_name": "Publishable Entity Version", - "verbose_name_plural": "Publishable Entity Versions", + 'verbose_name': 'Publishable Entity Version', + 'verbose_name_plural': 'Publishable Entity Versions', }, ), migrations.CreateModel( - name="PublishLog", + name='PublishLog', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("message", models.CharField(blank=True, default="", max_length=1000)), - ( - "published_at", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), - ( - "published_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('message', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, default='', max_length=500)), + ('published_at', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), + ('published_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), ], options={ - "verbose_name": "Publish Log", - "verbose_name_plural": "Publish Logs", + 'verbose_name': 'Publish Log', + 'verbose_name_plural': 'Publish Logs', }, ), migrations.CreateModel( - name="Draft", + name='Draft', fields=[ - ( - "entity", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), + ('entity', models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), ], ), migrations.CreateModel( - name="Published", + name='Published', fields=[ - ( - "entity", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), + ('entity', models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), ], options={ - "verbose_name": "Published Entity", - "verbose_name_plural": "Published Entities", + 'verbose_name': 'Published Entity', + 'verbose_name_plural': 'Published Entities', }, ), migrations.CreateModel( - name="PublishLogRecord", + name='PublishLogRecord', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "entity", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentity", - ), - ), - ( - "new_version", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), - ), - ( - "old_version", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - related_name="+", - to="oel_publishing.publishableentityversion", - ), - ), - ( - "publish_log", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.publishlog", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entity', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentity')), + ('new_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion')), + ('old_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, related_name='+', to='oel_publishing.publishableentityversion')), + ('publish_log', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishlog')), ], options={ - "verbose_name": "Publish Log Record", - "verbose_name_plural": "Publish Log Records", + 'verbose_name': 'Publish Log Record', + 'verbose_name_plural': 'Publish Log Records', }, ), migrations.AddConstraint( - model_name="learningpackage", - constraint=models.UniqueConstraint( - fields=("key",), name="oel_publishing_lp_uniq_key" - ), + model_name='learningpackage', + constraint=models.UniqueConstraint(fields=('key',), name='oel_publishing_lp_uniq_key'), + ), + migrations.AddIndex( + model_name='publishlogrecord', + index=models.Index(fields=['entity', '-publish_log'], name='oel_plr_idx_entity_rplr'), + ), + migrations.AddConstraint( + model_name='publishlogrecord', + constraint=models.UniqueConstraint(fields=('publish_log', 'entity'), name='oel_plr_uniq_pl_publishable'), ), migrations.AddField( - model_name="published", - name="publish_log_record", - field=models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishlogrecord", - ), + model_name='published', + name='publish_log_record', + field=models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishlogrecord'), ), migrations.AddField( - model_name="published", - name="version", - field=models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), + model_name='published', + name='version', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion'), ), migrations.AddIndex( - model_name="publishableentityversion", - index=models.Index( - fields=["entity", "-created"], name="oel_pv_idx_entity_rcreated" - ), + model_name='publishableentityversion', + index=models.Index(fields=['entity', '-created'], name='oel_pv_idx_entity_rcreated'), ), migrations.AddIndex( - model_name="publishableentityversion", - index=models.Index(fields=["title"], name="oel_pv_idx_title"), + model_name='publishableentityversion', + index=models.Index(fields=['title'], name='oel_pv_idx_title'), ), migrations.AddConstraint( - model_name="publishableentityversion", - constraint=models.UniqueConstraint( - fields=("entity", "version_num"), name="oel_pv_uniq_entity_version_num" - ), + model_name='publishableentityversion', + constraint=models.UniqueConstraint(fields=('entity', 'version_num'), name='oel_pv_uniq_entity_version_num'), ), migrations.AddIndex( - model_name="publishableentity", - index=models.Index(fields=["key"], name="oel_pub_ent_idx_key"), + model_name='publishableentity', + index=models.Index(fields=['key'], name='oel_pub_ent_idx_key'), ), migrations.AddIndex( - model_name="publishableentity", - index=models.Index( - fields=["learning_package", "-created"], - name="oel_pub_ent_idx_lp_rcreated", - ), + model_name='publishableentity', + index=models.Index(fields=['learning_package', '-created'], name='oel_pub_ent_idx_lp_rcreated'), ), migrations.AddConstraint( - model_name="publishableentity", - constraint=models.UniqueConstraint( - fields=("learning_package", "key"), name="oel_pub_ent_uniq_lp_key" - ), + model_name='publishableentity', + constraint=models.UniqueConstraint(fields=('learning_package', 'key'), name='oel_pub_ent_uniq_lp_key'), ), migrations.AddField( - model_name="draft", - name="version", - field=models.OneToOneField( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), + model_name='draft', + name='version', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion'), ), ] diff --git a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py b/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py deleted file mode 100644 index 986683c47..000000000 --- a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.2.19 on 2023-05-14 13:58 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("oel_publishing", "0001_initial"), - ] - - operations = [ - migrations.AddIndex( - model_name="publishlogrecord", - index=models.Index( - fields=["entity", "-publish_log"], name="oel_plr_idx_entity_rplr" - ), - ), - migrations.AddConstraint( - model_name="publishlogrecord", - constraint=models.UniqueConstraint( - fields=("publish_log", "entity"), name="oel_plr_uniq_pl_publishable" - ), - ), - ] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 0f9a50fdd..78f19b4ba 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -17,8 +17,9 @@ from django.core.validators import MinValueValidator from openedx_learning.lib.fields import ( - key_field, + case_insensitive_char_field, immutable_uuid_field, + key_field, manual_date_time_field, ) @@ -32,7 +33,7 @@ class LearningPackage(models.Model): uuid = immutable_uuid_field() key = key_field() - title = models.CharField(max_length=1000, null=False, blank=False) + title = case_insensitive_char_field(max_length=500, blank=False) created = manual_date_time_field() updated = manual_date_time_field() @@ -213,7 +214,7 @@ class PublishableEntityVersion(models.Model): # Most publishable things will have some sort of title, but blanks are # allowed for those that don't require one. - title = models.CharField(max_length=1000, default="", null=False, blank=True) + title = case_insensitive_char_field(max_length=500, blank=True, default="") # The version_num starts at 1 and increments by 1 with each new version for # a given PublishableEntity. Doing it this way makes it more convenient for @@ -339,7 +340,7 @@ class PublishLog(models.Model): uuid = immutable_uuid_field() learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) - message = models.CharField(max_length=1000, null=False, blank=True, default="") + message = case_insensitive_char_field(max_length=500, blank=True, default="") published_at = manual_date_time_field() published_by = models.ForeignKey( settings.AUTH_USER_MODEL, diff --git a/openedx_learning/lib/collations.py b/openedx_learning/lib/collations.py new file mode 100644 index 000000000..2afdcc377 --- /dev/null +++ b/openedx_learning/lib/collations.py @@ -0,0 +1,116 @@ +""" +This module has collation-related code to allow us to attach collation settings +to specific fields on a per-database-vendor basis. This used by the ``fields`` +module in order to specify field types have have normalized behavior between +SQLite and MySQL (see fields.py for more details). +""" +from django.db import models + + +class MultiCollationMixin: + """ + Mixin to enable multiple, database-vendor-specific collations. + + This should be mixed into new subclasses of CharField and TextField, since + they're the only Field types that store text data. + """ + + def __init__(self, *args, db_collations=None, db_collation=None, **kwargs): + """ + Init like any field but add db_collations and disallow db_collation + + The ``db_collations`` param should be a dict of vendor names to + collations, like:: + + { + 'msyql': 'utf8mb4_bin', + 'sqlite': 'BINARY' + } + + It is an error to pass in a CharField-style ``db_collation``. I + originally wanted to use this attribute name, but I needed to preserve + it for Django 3.2 compatibility (see the ``db_collation`` method + docstring for details). + """ + if db_collation is not None: + raise ValueError( + f"Cannot use db_collation with {self.__class__.__name__}. " + + "Please use a db_collations dict instead." + ) + + super().__init__(*args, **kwargs) + self.db_collations = db_collations or {} + + # This is part of a hack to get this to work for Django < 4.1. Please + # see comments in the db_collation method for details. + self._vendor = None + + @property + def db_collation(self): + """ + Return the db_collation, understanding that it varies by vendor. + + This method is a hack for Django 3.2 compatibility and should be removed + after we move to 4.2. + + Description of why this is hacky: + + In Django 4.2, the schema builder pulls the collation settings from the + field using the value returned from the ``db_parameters`` method, and + this does what we want it to do. In Django 3.2, field.db_parameters is + called, but any collation value sent back is ignored and the code grabs + the value of db_collation directly from the field: + + https://github.com/django/django/blob/stable/3.2.x/django/db/backends/base/schema.py#L214-L224 + + But this call to get the ``field.db_collation`` attribute happens almost + immediately after the ``field.db_parameters`` method call. So our + fragile hack is to set ``self._vendor`` in the ``db_parameters`` method, + using the value we get from the connection that is passed in there. We + can then use ``self._vendor`` to return the right value when Django + calls ``field.db_collation`` (which is this property method). + + This method, the corresponding setter, and all references to + ``self._vendor`` should be removed after we've cut over to Django 4.2. + """ + return self.db_collations.get(self._vendor) + + @db_collation.setter + def db_collation(self, value): + """ + Don't allow db_collation to be set manually (just ignore). + + This can be removed when we move to Django 4.2. + """ + pass + + def db_parameters(self, connection): + """ + Return database parameters for this field. This adds collation info. + + We examine this field's ``db_collations`` attribute and return the + collation that maps to ``connection.vendor``. This will typically be + 'mysql' or 'sqlite'. + """ + db_params = models.Field.db_parameters(self, connection) + + # Remove once we no longer need to support Django < 4.1 + self._vendor = connection.vendor + + # Now determine collation based on DB vendor (e.g. 'sqlite', 'mysql') + if connection.vendor in self.db_collations: + db_params["collation"] = self.db_collations[connection.vendor] + + return db_params + + def deconstruct(self): + """ + How to serialize our Field for the migration file. + + For our mixin fields, this is just doing what the field's superclass + would do and then tacking on our custom ``db_collations`` dict data. + """ + name, path, args, kwargs = super().deconstruct() + if self.db_collations: + kwargs["db_collations"] = self.db_collations + return name, path, args, kwargs diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index 71273afa0..0e231e3a6 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -1,51 +1,82 @@ """ Convenience functions to make consistent field conventions easier. -Field conventions: - -* Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a - primary key + separate UUID column. +Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a +primary key + separate UUID column. https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html -* The UUID fields are intended to be globally unique identifiers that other - services can store and rely on staying the same. -* The "key" fields can be more human-friendly strings, but these may only - be unique within a given scope. These values should be treated as mutable, - even if they rarely change in practice. - -TODO: -* Try making a CaseSensitiveCharField and CaseInsensitiveCharField -* Investigate more efficient UUID binary encoding + search in MySQL - -Other data thoughts: -* It would be good to make a data-dumping sort of script that exported part of - the data to SQLite3. -* identifiers support stable import/export with serialized formats -* UUIDs will import/export in the SQLite3 dumps, but are not there in other contexts + +We have helpers to make case sensitivity consistent across backends. MySQL is +case-insensitive by default, SQLite and Postgres are case-sensitive. + + """ import hashlib import uuid from django.db import models +from .collations import MultiCollationMixin from .validators import validate_utc_datetime -def key_field(): +def create_hash_digest(data_bytes): + return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() + + +def case_insensitive_char_field(**kwargs): """ - Externally created Identifier fields. + Return a case-insensitive ``MultiCollationCharField``. - These will often be local to a particular scope, like within a - LearningPackage. It's up to the application as to whether they're - semantically meaningful or look more machine-generated. + This means that entries will sort in a case-insensitive manner, and that + unique indexes will be case insensitive, e.g. you would not be able to + insert "abc" and "ABC" into the same table field if you put a unique index + on this field. - Other apps should *not* make references to these values directly, since - these values may in theory change. + You may override any argument that you would normally pass into + ``MultiCollationCharField`` (which is itself a subclass of ``CharField``). """ - return models.CharField( - max_length=255, - blank=False, - null=False, - ) + # Set our default arguments + final_kwargs = { + "null": False, + "db_collations": { + "sqlite": "NOCASE", + # We're using utf8mb4_unicode_ci to keep MariaDB compatibility, + # since their collation support diverges after this. MySQL is now on + # utf8mb4_0900_ai_ci based on Unicode 9, while MariaDB has + # uca1400_ai_ci based on Unicode 14. + "mysql": "utf8mb4_unicode_ci", + }, + } + # Override our defaults with whatever is passed in. + final_kwargs.update(kwargs) + + return MultiCollationCharField(**final_kwargs) + + +def case_sensitive_char_field(**kwargs): + """ + Return a case-sensitive ``MultiCollationCharField``. + + This means that entries will sort in a case-sensitive manner, and that + unique indexes will be case sensitive, e.g. "abc" and "ABC" would be + distinct and you would not get a unique constraint violation by adding them + both to the same table field. + + You may override any argument that you would normally pass into + ``MultiCollationCharField`` (which is itself a subclass of ``CharField``). + """ + # Set our default arguments + final_kwargs = { + "null": False, + "db_collations": { + "sqlite": "BINARY", + "mysql": "utf8mb4_bin", + }, + } + # Override our defaults with whatever is passed in. + final_kwargs.update(kwargs) + + return MultiCollationCharField(**final_kwargs) def immutable_uuid_field(): @@ -66,6 +97,20 @@ def immutable_uuid_field(): ) +def key_field(): + """ + Externally created Identifier fields. + + These will often be local to a particular scope, like within a + LearningPackage. It's up to the application as to whether they're + semantically meaningful or look more machine-generated. + + Other apps should *not* make references to these values directly, since + these values may in theory change (even if this is rare in practice). + """ + return case_sensitive_char_field(max_length=500, blank=False) + + def hash_field(): """ Holds a hash digest meant to identify a piece of content. @@ -85,10 +130,6 @@ def hash_field(): ) -def create_hash_digest(data_bytes): - return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() - - def manual_date_time_field(): """ DateTimeField that does not auto-generate values. @@ -117,3 +158,29 @@ def manual_date_time_field(): validate_utc_datetime, ], ) + + +class MultiCollationCharField(MultiCollationMixin, models.CharField): + """ + CharField subclass with per-database-vendor collation settings. + + Django's CharField already supports specifying the database collation, but + that only works with a single value. So there would be no way to say, "Use + utf8mb4_bin for MySQL, and BINARY if we're running SQLite." This is a + problem because we run tests in SQLite (and may potentially run more later). + It's also a problem if we ever want to support other database backends, like + PostgreSQL. Even MariaDB is starting to diverge from MySQL in terms of what + collations are supported. + """ + pass + + +class MultiCollationTextField(MultiCollationMixin, models.TextField): + """ + TextField subclass with per-database-vendor collation settings. + + We don't ever really want to _sort_ by a TextField, but setting a collation + forces the compatible charset to be set in MySQL, and that's the part that + matters for our purposes. So for example, if you set + """ + pass diff --git a/requirements/base.txt b/requirements/base.txt index d48934464..97ec03540 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via django django==3.2.19 # via @@ -19,3 +19,5 @@ pytz==2023.3 # djangorestframework sqlparse==0.4.4 # via django +typing-extensions==4.6.3 + # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index d851c07e2..6e1dd7f92 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ click==8.1.3 # via import-linter distlib==0.3.6 # via virtualenv -filelock==3.12.0 +filelock==3.12.2 # via # tox # virtualenv @@ -18,7 +18,7 @@ import-linter==1.9.0 # via -r requirements/ci.in packaging==23.1 # via tox -platformdirs==3.5.1 +platformdirs==3.6.0 # via virtualenv pluggy==1.0.0 # via tox @@ -34,9 +34,9 @@ tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # grimp # import-linter -virtualenv==20.23.0 +virtualenv==20.23.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 9f6bd5332..b2dcdaf2e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/quality.txt # django @@ -49,11 +49,11 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/quality.txt # pytest-cov -diff-cover==7.5.0 +diff-cover==7.6.0 # via -r requirements/dev.in dill==0.3.6 # via @@ -86,7 +86,7 @@ exceptiongroup==1.1.1 # via # -r requirements/quality.txt # pytest -filelock==3.12.0 +filelock==3.12.2 # via # -r requirements/ci.txt # tox @@ -104,7 +104,7 @@ import-linter==1.9.0 # via # -r requirements/ci.txt # -r requirements/quality.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # -r requirements/quality.txt # keyring @@ -130,7 +130,7 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.13.1 +keyring==24.0.0 # via # -r requirements/quality.txt # twine @@ -138,11 +138,11 @@ lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/quality.txt # jinja2 @@ -158,6 +158,8 @@ more-itertools==9.1.0 # via # -r requirements/quality.txt # jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/quality.txt packaging==23.1 # via # -r requirements/ci.txt @@ -178,7 +180,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -222,7 +224,7 @@ pylint-django==2.5.3 # via # -r requirements/quality.txt # edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # -r requirements/quality.txt # pylint-celery @@ -231,12 +233,12 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt @@ -254,11 +256,11 @@ pyyaml==6.0 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==37.3 +readme-renderer==40.0 # via # -r requirements/quality.txt # twine -requests==2.30.0 +requests==2.31.0 # via # -r requirements/quality.txt # requests-toolbelt @@ -271,7 +273,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.3.5 +rich==13.4.2 # via # -r requirements/quality.txt # twine @@ -324,21 +326,22 @@ tox-battery==0.6.1 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/ci.txt # -r requirements/quality.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # -r requirements/quality.txt # requests # twine -virtualenv==20.23.0 +virtualenv==20.23.1 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index aa92d8e59..d475be069 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -66,7 +66,7 @@ imagesize==1.4.1 # via sphinx import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via sphinx iniconfig==2.0.0 # via @@ -77,10 +77,12 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,12 +108,12 @@ pygments==2.15.1 # pydata-sphinx-theme # readme-renderer # sphinx -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -129,9 +131,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via -r requirements/doc.in -requests==2.30.0 +requests==2.31.0 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -183,13 +185,14 @@ tomli==2.0.1 # doc8 # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # grimp # import-linter # pydata-sphinx-theme -urllib3==2.0.2 +urllib3==2.0.3 # via requests webencodings==0.5.1 # via bleach diff --git a/requirements/quality.txt b/requirements/quality.txt index b36f07abd..ab8f3977d 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -60,7 +60,7 @@ idna==3.4 # via requests import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # keyring # twine @@ -80,13 +80,13 @@ jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.13.1 +keyring==24.0.0 # via twine lazy-object-proxy==1.9.0 # via astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 @@ -96,6 +96,8 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==9.1.0 # via jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,7 +108,7 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via pylint pluggy==1.0.0 # via @@ -130,16 +132,16 @@ pylint-celery==0.3 # via edx-lint pylint-django==2.5.3 # via edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -156,9 +158,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via twine -requests==2.30.0 +requests==2.31.0 # via # requests-toolbelt # twine @@ -166,7 +168,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.5 +rich==13.4.2 # via twine six==1.16.0 # via @@ -197,15 +199,16 @@ tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/quality.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # requests # twine diff --git a/requirements/test.in b/requirements/test.in index 65ac63e81..48446899b 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -3,6 +3,8 @@ -r base.txt # Core dependencies for this package +mysqlclient<3.0 # MySQL support + coverage # Code coverage reporting import-linter # Track our internal dependencies diff --git a/requirements/test.txt b/requirements/test.txt index ae4cfa040..8865d16b8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/base.txt # django @@ -14,7 +14,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.in # pytest-cov @@ -34,20 +34,22 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.in packaging==23.1 # via pytest pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.in # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in @@ -73,7 +75,9 @@ tomli==2.0.1 # coverage # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via + # -r requirements/base.txt + # asgiref # grimp # import-linter diff --git a/setup.py b/setup.py index 4fef021c4..ab642620a 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import re import sys -from setuptools import setup +from setuptools import find_packages, setup def get_version(*file_paths): @@ -72,10 +72,10 @@ def is_requirement(line): author='David Ormsbee', author_email='dave@tcril.org', url='https://github.com/openedx/openedx-learning', - packages=[ - 'openedx_learning', - 'openedx_tagging', - ], + packages=find_packages( + include=['openedx_learning*', 'openedx_tagging*'], + exclude=['*.test', '*.tests'] + ), include_package_data=True, install_requires=load_requirements('requirements/base.in'), python_requires=">=3.8", diff --git a/tox.ini b/tox.ini index 362718418..23e2650ad 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,10 @@ deps = django32: Django>=3.2,<3.3 django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt +setenv = + # Note that the django32/django42 targets use MySQL, but running pytest by + # itself (without tox) will run tests in SQLite for developer convenience. + DJANGO_SETTINGS_MODULE = mysql_test_settings commands = pytest {posargs}