initial main commit

Author: PabloVitasso

.gitignore

@@ -69,4 +69,9 @@ env.bak/
 venv.bak/
 
 # logs
-logs/
+logs/
+# Pliki generowane przez testy
+tests/test_anonymizer_output.py
+tests/test_document-big.actual.txt
+# Spacy models
+models/

core/anonymizer.py

@@ -19,10 +19,18 @@ class Entity(NamedTuple):
     nlp = None
 
 def safe_substitute(text: str, substitution_dict: dict) -> str:
+    """
+    Substitutes tokens in a single pass, ensuring that replacements are handled correctly
+    even if they overlap, by processing them from longest to shortest.
+    """
+    # Sort keys by length in descending order to handle nested substitutions correctly.
+    # For example, "Jan Kowalski" should be replaced before "Jan".
     sorted_keys = sorted(substitution_dict.keys(), key=len, reverse=True)
+    
     for original in sorted_keys:
         replacement = substitution_dict[original]
         text = text.replace(original, replacement)
+        
     return text
 
 def anonymize_text(text: str, profile: str = DEFAULT_PROFILE, custom_classes: list[str] = None) -> tuple[str, dict]:
@@ -40,10 +48,11 @@ def anonymize_text(text: str, profile: str = DEFAULT_PROFILE, custom_classes: li
         for match in pattern.finditer(text):
             start, end = match.span()
             # Sprawdź, czy zakres się nie nakłada
-            if not any(start in r or end - 1 in r for r in matched_pos):
+            current_range = range(start, end)
+            if not any(max(r.start, current_range.start) < min(r.stop, current_range.stop) for r in matched_pos):
                 entity = Entity(match.group(0), class_name, start, end)
                 all_entities.append(entity)
-                matched_pos.add(range(start, end))
+                matched_pos.add(current_range)
                 logger.debug(f"Regex found: {entity}")
 
     # Krok 2: Znajdź encje z NER, ignorując te, które nakładają się na regex
@@ -75,7 +84,7 @@ def anonymize_text(text: str, profile: str = DEFAULT_PROFILE, custom_classes: li
 
             if transform_type == "tokenize":
                 count = token_counters.get(entity.type, 0)
-                substitution_dict[entity.text] = f"<{entity.type}_{count}>"
+                substitution_dict[entity.text] = f"__{entity.type}_{count}__"
                 token_counters[entity.type] = count + 1
             elif transform_type == "remove":
                 substitution_dict[entity.text] = ""
@@ -94,5 +103,5 @@ def anonymize_text(text: str, profile: str = DEFAULT_PROFILE, custom_classes: li
         substitution_dict.update(generalize_money(temp_text))
 
     anonymized_text = safe_substitute(text, substitution_dict)
-    token_map = {v: k for k, v in substitution_dict.items() if v.startswith('<')}
+    token_map = {v: k for k, v in substitution_dict.items() if v.startswith('__') and v.endswith('__')}
     return anonymized_text, token_map

core/patterns.py

@@ -11,7 +11,60 @@
     ("REGON", re.compile(r"\b\d{9,14}\b")),
     # Poprawiony, prostszy wzorzec dla organizacji w cudzysłowach
     ("ORGANIZATION", re.compile(r'"([^"]+ S\.A\.)"', re.IGNORECASE)),
+    # Wzorzec dla kwot słownych, np. "sto tysięcy złotych"
+    ("MONEY_TEXT", re.compile(r"""
+        \b(
+            (?:
+                (?:jeden|dwa|trzy|cztery|pięć|sześć|siedem|osiem|dziewięć|dziesięć|
+                jedenaście|dwanaście|trzynaście|czternaście|piętnaście|szesnaście|siedemnaście|osiemnaście|dziewiętnaście|
+                dwadzieścia|trzydzieści|czterdzieści|pięćdziesiąt|sześćdziesiąt|siedemdziesiąt|osiemdziesiąt|dziewięćdziesiąt|
+                sto|dwieście|trzysta|czterysta|pięćset|sześćset|siedemset|osiemset|dziewięćset|
+                tysiąc|tysiące|tysięcy|milion|miliony|milionów|miliard|miliardy|miliardów)
+                [\s,-]*
+            )+
+        )
+        \s+(?:złotych|złote|złoty|pln|zł)\b
+    """, re.IGNORECASE | re.VERBOSE)),
     ("MONEY", re.compile(r"(\d{1,3}(?:[ .]\d{3})*(?:,\d{2})?)\s?(z[łl])", re.IGNORECASE)),
+    # Wzorce dla powierzchni. Najpierw bardziej szczegółowy (z opisem), potem ogólny.
+    # To zapobiega sytuacji, w której ogólny wzorzec dopasowuje tylko część dłuższego wyrażenia.
+    ("AREA", re.compile(r"""
+        \b
+        (?:ok\.\s*)?
+        (?:
+            (?:
+                (?:\d{1,3}(?:[ .,]\d{3})*|\d+)(?:[,.]\d+)?\s*(?:ha|a|m²|m|cm²|cm|mm|km²)
+            )
+            (?:
+                \s*(?:x\s*)?(?:\d{1,3}(?:[ .,]\d{3})*|\d+)(?:[,.]\d+)?\s*(?:a|m²|m|cm²|cm|mm|km²)
+            ){0,2}
+        )
+        \s*\([^)]+\) # Opis w nawiasie jest tutaj obowiązkowy
+    """, re.IGNORECASE | re.VERBOSE)),
+    ("AREA", re.compile(r"""
+        \b
+        (?:ok\.\s*)?
+        (?:
+            (?:
+                (?:\d{1,3}(?:[ .,]\d{3})*|\d+)(?:[,.]\d+)?\s*(?:ha|a|m²|m|cm²|cm|mm|km²)
+            )
+            (?:
+                \s*(?:x\s*)?(?:\d{1,3}(?:[ .,]\d{3})*|\d+)(?:[,.]\d+)?\s*(?:a|m²|m|cm²|cm|mm|km²)
+            ){0,2}
+        )
+        \b
+    """, re.IGNORECASE | re.VERBOSE)),
     ("DATE", re.compile(r"\b\d{1,2}\.\d{1,2}\.\d{4}\b")),
     ("DATE", re.compile(r"\b\d{1,2} [a-zA-Z]+ \d{4} roku\b", re.IGNORECASE)),
+    ("POST_CODE", re.compile(r"\b\d{2}-\d{3}\b")),
+    ("STREET_ADDRESS", re.compile(r"""
+        \b(
+            (?:ul|al|pl|os)\.?\s+
+            (?:[A-ZŻŹĆĄŚĘŁÓŃa-zżźćńółęąś-]+\s+)+
+            \d{1,4}
+            (?:[a-zA-Z])?
+            (?:/\d{1,4})?
+        )\b
+    """, re.IGNORECASE | re.VERBOSE)),
+    ("LAND_PLOT", re.compile(r"\b\d+\s*/\s*\d+\b")),
 ]

core/profile_config.py

@@ -3,29 +3,45 @@
 PROFILES = {
     "pseudonymized": {
         "description": "Pseudonimizacja: Zamiana encji na tagi, z możliwością odwrócenia (generuje mapę).",
-        "classes": ["PESEL", "NIP", "REGON", "KW", "PERSON", "LOCATION", "ORGANIZATION", "MONEY", "DATE"],
+        "classes": [
+            "PESEL", "NIP", "REGON", "KW", "PERSON", "LOCATION", "ORGANIZATION",
+            "MONEY", "MONEY_TEXT", "DATE", "POST_CODE", "STREET_ADDRESS", "LAND_PLOT", "AREA"
+        ],
         "transformations": {
             "DATE": "tokenize",
-            "MONEY": "tokenize"
+            "POST_CODE": "tokenize",
+            "STREET_ADDRESS": "tokenize",
+            "LAND_PLOT": "tokenize",
+            "MONEY": "tokenize",
+            "MONEY_TEXT": "tokenize",
+            "AREA": "tokenize"
         }
     },
     "gdpr": {
         "description": "Anonimizacja RODO: Nieodwracalna generalizacja danych.",
-        "classes": ["PERSON", "LOCATION", "ORGANIZATION", "MONEY", "DATE"],
+        "classes": ["PERSON", "LOCATION", "ORGANIZATION", "MONEY", "MONEY_TEXT", "DATE", "POST_CODE", "STREET_ADDRESS", "LAND_PLOT"],
         "transformations": {
             "DATE": "generalize",
+            "POST_CODE": "remove",
+            "STREET_ADDRESS": "remove",
+            "LAND_PLOT": "remove",
             "MONEY": "generalize",
+            "MONEY_TEXT": "generalize",
             "PERSON": "remove",
             "LOCATION": "generalize_loc",
             "ORGANIZATION": "remove"
         }
     },
     "llm-safe": {
         "description": "Profil LLM-Safe: Maksymalna anonimizacja, zastępowanie semantyczne.",
-        "classes": ["PESEL", "NIP", "REGON", "KW", "PERSON", "LOCATION", "ORGANIZATION", "MONEY", "DATE"],
+        "classes": ["PESEL", "NIP", "REGON", "KW", "PERSON", "LOCATION", "ORGANIZATION", "MONEY", "MONEY_TEXT", "DATE", "POST_CODE", "STREET_ADDRESS", "LAND_PLOT"],
         "transformations": {
             "DATE": "generalize",
+            "POST_CODE": "tokenize",
+            "STREET_ADDRESS": "tokenize",
+            "LAND_PLOT": "tokenize",
             "MONEY": "generalize",
+            "MONEY_TEXT": "generalize",
             "PERSON": "replace_semantic",
             "LOCATION": "replace_semantic",
             "ORGANIZATION": "replace_semantic"

interfaces/__init__.py

@@ -1,12 +1,13 @@
 import argparse
+import os
 from core.anonymizer import anonymize_text
 from .file_io import read_file, write_file, save_map_dict
 from utils.log import setup_logger
 
 def main():
     parser = argparse.ArgumentParser(description="Anonymize documents from the command line.")
-    parser.add_argument("--input", required=True, help="Path to the source file.")
-    parser.add_argument("--output", required=True, help="Path to the output anonymized file.")
+    parser.add_argument("-i", "--input", required=True, help="Path to the source file.")
+    parser.add_argument("-o", "--output", help="Path to the output anonymized file. Defaults to <input>.anon.<ext>")
     parser.add_argument("--profile", default="pseudonymized", choices=["pseudonymized", "gdpr", "llm-safe"],
                         help="Anonymization profile.")
 
@@ -16,6 +17,11 @@ def main():
 
     args = parser.parse_args()
 
+    # Jeśli plik wyjściowy nie jest podany, utwórz go na podstawie nazwy pliku wejściowego
+    if not args.output:
+        base, ext = os.path.splitext(args.input)
+        args.output = f"{base}.anon{ext}"
+
     # Skonfiguruj logger z nazwą pliku wejściowego
     logger = setup_logger(args.input)
 

interfaces/cli.py

@@ -1,54 +1,133 @@
+import os
 import gradio as gr
 from core.anonymizer import anonymize_text
 import tempfile
 import json
-
 from core.profile_config import PROFILES
 
-def anonymize_interface(file_obj, profile):
-    if file_obj is None:
-        return "Proszę wgrać plik.", "Brak wyników.", None
+# Wyłącz analytics
+os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+os.environ['HF_HUB_OFFLINE'] = '1'
 
-    original_text = file_obj.decode('utf-8')
+class AnonymizerInterface:
+    """Wrapper dla interfejsu anonimizatora"""
 
-    anonymized_text, substitution_map = anonymize_text(original_text, profile)
+    @staticmethod
+    def create_temp_file(content, filename, extension):
+        """Tworzy tymczasowy plik z contentem"""
+        if not content:
+            return None
+        
+        with tempfile.NamedTemporaryFile(
+            delete=False,
+            mode="w",
+            suffix=extension,
+            prefix=filename + "_",
+            encoding="utf-8"
+        ) as tmp:
+            if extension == ".json":
+                json.dump(content, tmp, ensure_ascii=False, indent=2)
+            else:
+                tmp.write(content)
+            return tmp.name
 
-    # Tworzenie pliku do pobrania
-    if substitution_map:
-        with tempfile.NamedTemporaryFile(delete=False, mode="w", suffix=".json", encoding="utf-8") as tmp:
-            json.dump(substitution_map, tmp, ensure_ascii=False, indent=2)
-            map_file_path = tmp.name
-    else:
-        map_file_path = None
-        
-    return anonymized_text, substitution_map, map_file_path
+    @staticmethod
+    def process_file(file_obj, profile):
+        """Przetwarza plik i zwraca wyniki anonimizacji"""
+        if file_obj is None:
+            return "Proszę wgrać plik.", "Brak wyników.", None, None
+        
+        original_text = file_obj.decode('utf-8')
+        anonymized_text, substitution_map = anonymize_text(original_text, profile)
+        
+        # Tworzenie plików do pobrania
+        map_file_path = AnonymizerInterface.create_temp_file(
+            substitution_map,
+            "mapowania",
+            ".json"
+        )
+        
+        text_file_path = AnonymizerInterface.create_temp_file(
+            anonymized_text,
+            "tekst_anonimizowany",
+            ".txt"
+        )
+        
+        return anonymized_text, substitution_map, map_file_path, text_file_path
 
-def launch():
-    with gr.Blocks() as demo:
-        gr.Markdown("# Anonimizator Umów Notarialnych")
+def create_ui():
+    """Tworzy interfejs użytkownika"""
+    with gr.Blocks(
+        css="""
+        * {
+            font-family: system-ui, -apple-system, BlinkMacSystemFont, sans-serif !important;
+        }
+        link[rel="manifest"] { display: none !important; }
+        """,
+        theme=gr.themes.Soft()
+    ) as demo:
 
+        gr.Markdown("# Janusz Danych Rodo - Anonimizator Umów Notarialnych")
+        
+        # Wiersz 1: Wgranie pliku, profil, przycisk
+        with gr.Row():
+            file_input = gr.File(
+                label="Wgraj plik (.txt, .md)",
+                type="binary",
+                file_count="single"
+            )
+            profile_dropdown = gr.Dropdown(
+                choices=list(PROFILES.keys()),
+                value="pseudonymized",
+                label="Profil Anonimizacji"
+            )
+            submit_btn = gr.Button("Anonimizuj")
+        
+        # Wiersz 2: Tekst zanonimizowany
+        with gr.Row():
+            output_text = gr.Textbox(
+                label="Tekst zanonimizowany",
+                lines=15,
+                interactive=True
+            )
+        
+        # Wiersz 3: Słownik mapowań i pobieranie
         with gr.Row():
             with gr.Column():
-                file_input = gr.File(label="Wgraj plik (.txt, .md)", type="binary")
-                profile_dropdown = gr.Dropdown(
-                    choices=list(PROFILES.keys()),
-                    value="pseudonymized",
-                    label="Profil Anonimizacji"
-                )
-                submit_btn = gr.Button("Anonimizuj")
+                output_map = gr.JSON(label="Słownik mapowań")
 
             with gr.Column():
-                output_text = gr.Textbox(label="Tekst zanonimizowany", lines=15)
-                output_map = gr.JSON(label="Słownik mapowań")
-                download_map_btn = gr.File(label="Pobierz słownik mapowań")
-
+                gr.Markdown("### Pobierz pliki")
+                with gr.Group():
+                    download_map_btn = gr.File(label="Słownik mapowań")
+                    download_text_btn = gr.File(label="Tekst anonimizowany")
+        
         submit_btn.click(
-            fn=anonymize_interface,
+            fn=AnonymizerInterface.process_file,
             inputs=[file_input, profile_dropdown],
-            outputs=[output_text, output_map, download_map_btn]
+            outputs=[output_text, output_map, download_map_btn, download_text_btn]
         )
+        
+        return demo
+
+def launch():
+    """Uruchamia aplikację"""
+    print("Working directory:", os.getcwd())
+    print("Static dir exists:", os.path.exists("static"))
+    print("Manifest exists:", os.path.exists("static/manifest.json"))
+    print(gr.__version__)
+    print(hasattr(gr, 'set_static_paths'))
 
-    demo.launch()
+    demo = create_ui()
+    demo.launch(
+        share=False,
+        server_name="127.0.0.1",
+        server_port=7860,
+        inbrowser=True,
+        quiet=True,
+        show_error=True,
+        favicon_path=None,
+    )
 
 if __name__ == "__main__":
     launch()