Add segment sources information (#143)

* Add translation source for document records

* Make numbers substitution automated, not option

* Generate client

* Remove dead options from uploading dialog

* Add UI to show segment sources

Author: ArchiDevil

backend/alembic/versions/3fbb82ea683d_add_segment_translation_source.py

@@ -0,0 +1,46 @@
+"""Add segment translation source
+
+Revision ID: 3fbb82ea683d
+Revises: 32d5a77e6615
+Create Date: 2025-12-06 13:49:58.517637
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# pylint: disable=E1101
+
+# revision identifiers, used by Alembic.
+revision: str = "3fbb82ea683d"
+down_revision: Union[str, None] = "32d5a77e6615"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+segmentsource = sa.Enum(
+    "glossary",
+    "machine_translation",
+    "translation_memory",
+    "full_match",
+    name="recordsource",
+)
+
+
+def upgrade() -> None:
+    segmentsource.create(op.get_bind(), checkfirst=True)
+    op.add_column(
+        "document_record",
+        sa.Column(
+            "target_source",
+            segmentsource,
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("document_record", "target_source")
+    segmentsource.drop(op.get_bind(), checkfirst=True)

backend/app/documents/models.py

@@ -97,6 +97,13 @@ class Document(Base):
     )
 
 
+class RecordSource(Enum):
+    glossary = "glossary"
+    machine_translation = "mt"
+    translation_memory = "tm"
+    full_match = "fm"  # for digits
+
+
 class DocumentRecord(Base):
     __tablename__ = "document_record"
 
@@ -105,6 +112,7 @@ class DocumentRecord(Base):
     source: Mapped[str] = mapped_column()
     target: Mapped[str] = mapped_column()
     approved: Mapped[bool] = mapped_column(default=False)
+    target_source: Mapped[RecordSource] = mapped_column(nullable=True)
 
     document: Mapped["Document"] = relationship(back_populates="records")
     comments: Mapped[list["Comment"]] = relationship(

backend/app/documents/schema.py

@@ -2,10 +2,10 @@
 
 from pydantic import BaseModel, Field
 
-from app.documents.models import TmMode
+from app.documents.models import RecordSource, TmMode
 from app.glossary.schema import GlossaryResponse
 from app.models import DocumentStatus, Identified, MachineTranslationSettings
-from app.translation_memory.schema import TranslationMemory, TranslationMemoryUsage
+from app.translation_memory.schema import TranslationMemory
 
 
 class DocumentRecordFilter(BaseModel):
@@ -31,6 +31,7 @@ class DocumentRecord(Identified):
     approved: bool
     repetitions_count: int
     has_comments: bool
+    translation_src: RecordSource | None
 
 
 class DocumentRecordListResponse(BaseModel):
@@ -52,9 +53,7 @@ class DocumentRecordUpdate(BaseModel):
 
 
 class DocumentProcessingSettings(BaseModel):
-    substitute_numbers: bool
     machine_translation_settings: Optional[MachineTranslationSettings]
-    memory_usage: TranslationMemoryUsage
     similarity_threshold: float = Field(default=1.0, ge=0.0, le=1.0)
 
 

backend/app/routers/document.py

@@ -118,7 +118,6 @@ def get_doc_records(
     query = GenericDocsQuery(db)
     total_records = query.get_document_records_count_filtered(doc, filters)
     records = query.get_document_records_paged(doc, page, filters=filters)
-
     record_list = [
         doc_schema.DocumentRecord(
             id=record.id,
@@ -127,6 +126,9 @@ def get_doc_records(
             approved=record.approved,
             repetitions_count=repetitions_count,
             has_comments=has_comments,
+            translation_src=record.target_source.value
+            if record.target_source
+            else None,
         )
         for record, repetitions_count, has_comments in records
     ]

backend/app/translation_memory/schema.py

@@ -1,5 +1,3 @@
-from enum import Enum
-
 from pydantic import BaseModel, Field
 
 from app.base.schema import Identified
@@ -11,11 +9,6 @@ class MemorySubstitution(BaseModel):
     similarity: float
 
 
-class TranslationMemoryUsage(Enum):
-    NEWEST = "newest"
-    OLDEST = "oldest"
-
-
 class TranslationMemory(Identified):
     name: str
     created_by: int

backend/tests/fixtures/small.xliff

@@ -23,6 +23,10 @@
                 <source xml:space="preserve">123456789</source>
                 <target state="needs-translation" xml:space="preserve"></target>
             </trans-unit>
+            <trans-unit id="675610" approved="no" sc:locked="false" sc:last-modified-date="2023-10-19T18:29:23.968Z" sc:last-modified-user="Person 5">
+                <source xml:space="preserve">Something else</source>
+                <target state="needs-translation" xml:space="preserve"></target>
+            </trans-unit>
         </body>
     </file>
 </xliff>

backend/tests/routers/test_routes_doc_records.py

@@ -10,6 +10,7 @@
     Document,
     DocumentRecord,
     DocumentType,
+    RecordSource,
 )
 from app.translation_memory.models import TranslationMemory, TranslationMemoryRecord
 
@@ -19,7 +20,11 @@
 def test_can_get_doc_records(user_logged_client: TestClient, session: Session):
     with session as s:
         records = [
-            DocumentRecord(source="Regional Effects", target="Translation"),
+            DocumentRecord(
+                source="Regional Effects",
+                target="Translation",
+                target_source=RecordSource.translation_memory,
+            ),
             DocumentRecord(source="User Interface", target="UI", approved=True),
         ]
         s.add(
@@ -46,6 +51,7 @@ def test_can_get_doc_records(user_logged_client: TestClient, session: Session):
             "approved": False,
             "repetitions_count": 1,
             "has_comments": False,
+            "translation_src": "tm",
         },
         {
             "id": 2,
@@ -54,6 +60,7 @@ def test_can_get_doc_records(user_logged_client: TestClient, session: Session):
             "approved": True,
             "repetitions_count": 1,
             "has_comments": False,
+            "translation_src": None,
         },
     ]
 
@@ -94,6 +101,7 @@ def test_doc_records_returns_second_page(
         "approved": False,
         "repetitions_count": 1,
         "has_comments": False,
+        "translation_src": None,
     }
 
 

backend/tests/routers/test_routes_documents.py

@@ -25,7 +25,6 @@
 from app.models import DocumentStatus
 from app.schema import DocumentTask
 from app.translation_memory.models import TranslationMemory
-from app.translation_memory.schema import TranslationMemoryUsage
 
 # pylint: disable=C0116
 
@@ -296,9 +295,7 @@ def test_process_sets_document_in_pending_stage_and_creates_task_xliff(
     response = user_logged_client.post(
         "/document/1/process",
         json={
-            "substitute_numbers": False,
             "machine_translation_settings": None,
-            "memory_usage": TranslationMemoryUsage.NEWEST.value,
         },
     )
 
@@ -318,9 +315,7 @@ def test_process_sets_document_in_pending_stage_and_creates_task_txt(
     response = user_logged_client.post(
         "/document/1/process",
         json={
-            "substitute_numbers": False,
             "machine_translation_settings": None,
-            "memory_usage": TranslationMemoryUsage.NEWEST.value,
         },
     )
 
@@ -340,9 +335,7 @@ def test_process_creates_task_for_xliff(
     response = user_logged_client.post(
         "/document/1/process",
         json={
-            "substitute_numbers": False,
             "machine_translation_settings": None,
-            "memory_usage": TranslationMemoryUsage.NEWEST.value,
         },
     )
 
@@ -356,9 +349,7 @@ def test_process_creates_task_for_xliff(
             "type": "xliff",
             "document_id": 1,
             "settings": {
-                "substitute_numbers": False,
                 "machine_translation_settings": None,
-                "memory_usage": "newest",
                 "similarity_threshold": 1.0,
             },
         }
@@ -371,9 +362,7 @@ def test_process_creates_task_for_txt(user_logged_client: TestClient, session: S
     response = user_logged_client.post(
         "/document/1/process",
         json={
-            "substitute_numbers": False,
             "machine_translation_settings": None,
-            "memory_usage": TranslationMemoryUsage.NEWEST.value,
         },
     )
 
@@ -387,9 +376,7 @@ def test_process_creates_task_for_txt(user_logged_client: TestClient, session: S
             "type": "txt",
             "document_id": 1,
             "settings": {
-                "substitute_numbers": False,
                 "machine_translation_settings": None,
-                "memory_usage": "newest",
                 "similarity_threshold": 1.0,
             },
         }
@@ -401,9 +388,7 @@ def test_returns_404_when_processing_nonexistent_doc(
     response = user_logged_client.post(
         "/document/1/process",
         json={
-            "substitute_numbers": False,
             "machine_translation_settings": None,
-            "memory_usage": TranslationMemoryUsage.NEWEST.value,
         },
     )
     assert response.status_code == 404