Merge branch '5732-make-slug-permanent' of github.com:GSA/datagov-harvester into develop

Author: cmhedrickREI

app/commands/testdata.py

@@ -2,33 +2,50 @@
 from flask import Blueprint
 
 from database.interface import HarvesterDBInterface
+from database.models import db
 
 testdata = Blueprint("testdata", __name__)
 
-db = HarvesterDBInterface()
+db_interface = HarvesterDBInterface()
 
 
 ## Load Test Data
 # TODO move this into its own file when you break up routes
 @testdata.cli.command("load_test_data")
-def fixtures():
-    """Load database fixtures from JSON."""
-
+@click.option(
+    "--reset",
+    is_flag=True,
+    default=False,
+    help="Drop and recreate the database before loading fixtures.",
+)
+def fixtures(reset: bool) -> None:
+    """
+    Load database fixtures from JSON.
+
+    Use --reset to wipe the database and recreate all tables before loading.
+    """
     from tests.generate_fixtures import generate_dynamic_fixtures
 
+    if reset:
+        db.drop_all()
+        db.create_all()
+        click.echo("Database reset: all tables dropped and recreated.")
+
     fixture = generate_dynamic_fixtures()
 
     for item in fixture["organization"]:
-        db.add_organization(item)
+        db_interface.add_organization(item)
     for item in fixture["source"]:
-        db.add_harvest_source(item)
+        db_interface.add_harvest_source(item)
     for item in fixture["job"]:
-        db.add_harvest_job(item)
+        db_interface.add_harvest_job(item)
     for item in fixture["job_error"]:
-        db.add_harvest_job_error(item)
+        db_interface.add_harvest_job_error(item)
     for item in fixture["record"]:
-        db.add_harvest_record(item)
+        db_interface.add_harvest_record(item)
     for item in fixture["record_error"]:
-        db.add_harvest_record_error(item)
+        db_interface.add_harvest_record_error(item)
+    for item in fixture["dataset"]:
+        db_interface.insert_dataset(item)
 
     click.echo("Done.")

app/forms.py

@@ -231,3 +231,46 @@ def validate_url(self, field):
         validators=[url_paste_validate],
     )
     submit = SubmitField("Validate")
+    
+class DatasetSlugForm(FlaskForm):
+    """
+    Form for editing the slug of a Dataset.
+
+    Validates that the new slug is valid and unique across all
+    datasets, excluding the dataset currently being edited.
+    """
+
+    slug = StringField(
+        "Slug",
+        description=(
+            "Use lowercase letters, digits, and hyphens. "
+            "For example: 'my-dataset-title'."
+        ),
+        validators=[
+            DataRequired(),
+            Length(max=200),
+            Regexp(
+                r"^[a-z0-9-]+$",
+                message=(
+                    "Slug can only contain lowercase letters, digits, and hyphens."
+                ),
+            ),
+        ],
+        filters=[strip_filter],
+    )
+
+    def __init__(self, *args, **kwargs):
+        self.dataset_id = kwargs.pop("dataset_id", None)
+        self.db_interface = kwargs.pop("db_interface", None)
+        super().__init__(*args, **kwargs)
+
+    def validate_slug(self, field):
+        from database.interface import HarvesterDBInterface
+
+        db_interface = self.db_interface or HarvesterDBInterface()
+        existing = db_interface.get_dataset_by_slug(field.data)
+
+        if existing and existing.id != self.dataset_id:
+            raise ValidationError(
+                f"The slug '{field.data}' is already in use by another dataset."
+            )

app/routes.py

@@ -58,6 +58,7 @@
 )
 from .auth import LoginRequiredAuth
 from .forms import (
+    DatasetSlugForm,
     HarvestSourceForm,
     HarvestTriggerForm,
     OrganizationForm,
@@ -540,11 +541,16 @@ def add_harvest_source():
 )
 @valid_id_required  # TODO: Use an HTML 404 page
 def view_harvest_source(source_id: str):
-    htmx_vars = {
+    jobs_htmx_vars = {
         "target_div": "#paginated__harvest-jobs",
         "endpoint_url": f"/harvest_source/{source_id}",
         "page_param": "page",
     }
+    datasets_htmx_vars = {
+        "target_div": "#paginated__datasets",
+        "endpoint_url": f"/harvest_source/{source_id}",
+        "page_param": "datasets_page",
+    }
     harvest_jobs_facets = (
         f"harvest_source_id eq {source_id},date_created le {get_datetime()}"
     )
@@ -565,17 +571,42 @@ def view_harvest_source(source_id: str):
     )
 
     if htmx:
-        data = {
-            "source": {"id": source_id},
-            "jobs": jobs,
-            "htmx_vars": htmx_vars,
-        }
-        return render_block(
-            "view_source_data.html",
-            "htmx_paginated",
-            data=data,
-            pagination=pagination.to_dict(),
-        )
+        htmx_target = request.headers.get("HX-Target", "")
+
+        if "paginated__datasets" in htmx_target:
+            datasets_pagination = Pagination(
+                count=db.get_datasets_by_source(source_id=source_id, count=True),
+                current=request.args.get("datasets_page", 1, type=convert_to_int),
+            )
+            datasets = db.get_datasets_by_source(
+                source_id=source_id,
+                page=datasets_pagination.db_current,
+                per_page=datasets_pagination.per_page,
+            )
+            data = {
+                "source": {"id": source_id},
+                "datasets": datasets,
+                "datasets_htmx_vars": datasets_htmx_vars,
+            }
+            return render_block(
+                "view_source_data.html",
+                "htmx_paginated_datasets",
+                data=data,
+                pagination=datasets_pagination.to_dict(),
+                datasets_pagination=datasets_pagination.to_dict(),
+            )
+        else:
+            data = {
+                "source": {"id": source_id},
+                "jobs": jobs,
+                "htmx_vars": jobs_htmx_vars,
+            }
+            return render_block(
+                "view_source_data.html",
+                "htmx_paginated",
+                data=data,
+                pagination=pagination.to_dict(),
+            )
     elif request.method == "POST":
         form = HarvestTriggerForm(request.form)
         if form.data["edit"]:
@@ -690,18 +721,32 @@ def view_harvest_source(source_id: str):
             ],
         }
         source = db.get_harvest_source(source_id)
+        datasets_page = request.args.get("datasets_page", 1, type=convert_to_int)
+        datasets_count = db.get_datasets_by_source(source_id=source_id, count=True)
+        datasets_pagination = Pagination(
+            count=datasets_count,
+            current=datasets_page,
+        )
+        datasets = db.get_datasets_by_source(
+            source_id=source_id,
+            page=datasets_pagination.db_current,
+            per_page=datasets_pagination.per_page,
+        )
         data = {
             "ckan_url": CKAN_URL,
             "source": source,
             "summary_data": summary_data,
             "jobs": jobs,
             "chart_data": chart_data,
-            "htmx_vars": htmx_vars,
+            "htmx_vars": jobs_htmx_vars,
+            "datasets": datasets,
+            "datasets_htmx_vars": datasets_htmx_vars,
         }
         return render_template(
             "view_source_data.html",
             form=form,
             pagination=pagination.to_dict(),
+            datasets_pagination=datasets_pagination.to_dict(),
             data=data,
         )
 
@@ -713,6 +758,66 @@ def harvest_source_list():
     return render_template("view_source_list.html", data=data)
 
 
+@main.route("/dataset/<string:dataset_slug>", methods=["POST"])
+@api.get("/dataset/<string:dataset_slug>")
+@api.doc(
+    responses={
+        200: {"description": "View dataset detail page"},
+        404: {"description": "Dataset not found"},
+    }
+)
+def view_dataset(dataset_slug: str):
+    """View a dataset detail page by slug, and handle slug edits via POST."""
+    dataset = db.get_dataset_by_slug(dataset_slug)
+
+    if request.method == "POST":
+        if not session.get("user"):
+            flash("You must be logged in to edit a dataset slug.")
+            return redirect(url_for("api.view_dataset", dataset_slug=dataset_slug))
+
+        form = DatasetSlugForm(
+            request.form,
+            dataset_id=dataset.id if dataset else None,
+            db_interface=db,
+        )
+        if dataset is None:
+            flash("Dataset not found.")
+            return redirect(url_for("main.harvest_source_list"))
+
+        if form.validate():
+            updated = db.update_dataset_slug(dataset.id, form.slug.data)
+            if updated:
+                flash(f"Slug updated successfully to '{updated.slug}'.")
+                return redirect(url_for("api.view_dataset", dataset_slug=updated.slug))
+            else:
+                flash("Failed to update slug. Please try again.")
+                return redirect(url_for("api.view_dataset", dataset_slug=dataset_slug))
+        else:
+            # Re-render the page with the form errors shown inline.
+            data = {"dataset": dataset}
+            return (
+                render_template(
+                    "view_dataset_data.html",
+                    data=data,
+                    form=form,
+                ),
+                422,
+            )
+
+    # GET
+    form = DatasetSlugForm(
+        dataset_id=dataset.id if dataset else None,
+        db_interface=db,
+        data={"slug": dataset.slug} if dataset else {},
+    )
+    data = {"dataset": dataset}
+    return render_template(
+        "view_dataset_data.html",
+        data=data,
+        form=form,
+    ), (200 if dataset is not None else 404)
+
+
 ### Edit Source
 @api.route("/harvest_source/edit/<source_id>", methods=["GET", "POST"])
 @api.doc(hide=True)  # don't list the authenticated API

app/static/_scss/_uswds-theme-custom-styles.scss

@@ -85,6 +85,10 @@ ul.menu {
   overflow: scroll;
 }
 
+.usa-table.usa-table--striped {
+  width: 100%;
+}
+
 /* Keep Bootstrap 4 visual defaults while using Bootstrap 5 assets. */
 body {
   font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,