ci: check all files with ruff and update config

doc/config.py

@@ -4,39 +4,37 @@
 
 from promnesia.common import Source
 
-
 # now, let's import some indexers
 # NOTE: you might need extra dependencies before using some of the indexers
 # see https://github.com/karlicoss/HPI/blob/master/doc/SOURCES.org
 
 # 'auto' indexer tries its best at indexing plaintext stuff
 # - plaintext like org-mode/markdown/HTML
 # - structured formats like JSON and CSV
-from promnesia.sources import auto
+from promnesia.sources import auto  # isort: skip
 
 # 'guess' indexer can do even more in addition:
 # - HTTP links (to index the contents of a website)
 # - Github links (to index the contents of a git repository
-from promnesia.sources import guess
+from promnesia.sources import guess  # isort: skip
 # TODO there is a very thin link between 'auto' and 'guess'... I might merge them in the future?
 
 
 # this is an incomplete list, just the (perhaps) most interesting ones
-from promnesia.sources import telegram
 from promnesia.sources import (
     fbmessenger,
     hypothesis,
     instapaper,
     pocket,
     roamresearch,
     rss,
+    signal,
     takeout,
+    telegram,
     twitter,
     viber,
-    signal,
 )
 
-
 # NOTE: at the moment try to avoid using complex sources names
 # it's best to stick to digits, latin characters, dashes and underscores
 # there might be some UI problems related to that

ruff.toml

@@ -1,8 +1,3 @@
-# NOTE: version is inferred from pyproject.toml if present
-#   however, if ruff.toml is separate, pyproject isn't even parsed..
-#   see https://github.com/astral-sh/ruff/issues/10299
-target-version = "py39"
-
 lint.extend-select = [
     "F",    # flakes rules -- default, but extend just in case
     "E",    # pycodestyle  -- default, but extend just in case
@@ -39,6 +34,11 @@ lint.extend-select = [
     # "ALL", # uncomment this to check for new rules!
 ]
 
+# Preserve types, even if a file imports `from __future__ import annotations`
+# we need this for cachew to work with HPI types on 3.9
+# can probably remove after 3.10?
+lint.pyupgrade.keep-runtime-typing = true
+
 lint.ignore = [
     "D",     # annoying nags about docstrings
     "N",     # pep naming
@@ -119,7 +119,7 @@ lint.ignore = [
     "TRY400",  # TODO double check this, might be useful
     "TRY401",  # redundant exception in logging.exception call? TODO double check, might result in excessive logging
 
-    "PGH",  # TODO force error code in mypy instead
+    "PGH",  # TODO force error code in mypy instead? although it also has blanket noqa rule
 
     "TID252",  # Prefer absolute imports over relative imports from parent modules
 
@@ -138,8 +138,18 @@ lint.ignore = [
     "RSE102",  # complains about missing parens in exceptions
     ##
 
+    "PLC0415", # "imports should be at the top level" -- not realistic
     "ARG001",  # ugh, kinda annoying when using pytest fixtures
     "RUF001", "RUF002", "RUF003",  # spams about non-latin characters that we do use for testing
-
     "A005",  #  we're using promnesia.logging module
 ]
+
+extend-exclude = [
+    "untracked/**",
+    "old/**",
+    "tests/testdata/**",
+    # FIXME @karlicoss's private files.. sort out later
+    "config.py",
+    "adhoc_config.py",
+    "aaa_config.py",
+]

scripts/browser_history.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 DEPRECATION = 'NOTE: this is DEPRECATED! Please use https://github.com/purarue/browserexport instead'
 
-from datetime import datetime, timezone
-from pathlib import Path
-from subprocess import check_output
 import filecmp
 import logging
-import warnings
 import sys
+import warnings
+from datetime import datetime, timezone
+from pathlib import Path
+from subprocess import check_output
 
 warnings.warn(DEPRECATION, DeprecationWarning)
 

src/promnesia/logging.py

@@ -54,7 +54,7 @@ def mklevel(level: LevelIsh) -> Level:
 FORMAT_NOCOLOR = FORMAT.format(start='', end='')
 DATEFMT = '%Y-%m-%d %H:%M:%S'
 
-COLLAPSE_DEBUG_LOGS = os.environ.get('COLLAPSE_DEBUG_LOGS', False)
+COLLAPSE_DEBUG_LOGS = os.environ.get('COLLAPSE_DEBUG_LOGS', False)  # noqa: PLW1508
 
 _init_done = 'lazylogger_init_done'
 

src/promnesia/sources/browser_legacy.py

@@ -280,7 +280,7 @@ class Firefox(Extr):
     ], key=('url', 'visit_date', 'vid', 'pid'))
     query='FROM chunk.moz_historyvisits as V, chunk.moz_places as P WHERE V.place_id = P.id'
 
-    row2visit = _row2visit_firefox
+    row2visit = _row2visit_firefox  # type: ignore[assignment]
 
 
 class FirefoxPhone(Extr):
@@ -299,4 +299,4 @@ class FirefoxPhone(Extr):
     ], key=('url', 'date', 'vid', 'hid'))
     query='FROM chunk.visits as V, chunk.history as H  WHERE V.history_guid = H.guid'
 
-    row2visit = _row2visit_firefox
+    row2visit = _row2visit_firefox  # type: ignore[assignment]

src/promnesia/sources/org.py

@@ -72,7 +72,11 @@ def _parse_node(n: OrgNode) -> Parsed:
             dt = None
         else:
             [odt] = OrgDate.list_from_str(createds)
-            dt = odt.start
+            start = odt.start
+            if not isinstance(start, datetime):  # could be date
+                dt = datetime.combine(start, datetime.min.time())  # meh, but the best we can do?
+            else:
+                dt = start
     else:
         dt = None
     return Parsed(dt=dt, heading=heading)

src/promnesia/sources/viber.py

@@ -40,7 +40,7 @@ def index(
         yield from _harvest_db(db, msgs_query, locator_schema)
 
 
-def messages_query(http_only: bool | None) -> str:
+def messages_query(http_only: bool | None) -> str:  # noqa: FBT001
     """
     An SQL-query returning 1 row for each message
 

tests/addon.py

@@ -3,14 +3,16 @@
 """
 from __future__ import annotations
 
+import json
+from collections.abc import Iterator, Sequence
 from contextlib import contextmanager
 from dataclasses import dataclass
-import json
 from pathlib import Path
 from time import sleep
-from typing import Optional, Iterator, Sequence
+from typing import Optional
 
 import pytest
+from addon_helper import AddonHelper
 from selenium.common.exceptions import TimeoutException
 from selenium.webdriver import Remote as Driver
 from selenium.webdriver.common.action_chains import ActionChains
@@ -19,17 +21,16 @@
 from selenium.webdriver.remote.webelement import WebElement
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait as Wait
-
-from promnesia.logging import LazyLogger
-
-from addon_helper import AddonHelper
 from webdriver_utils import frame_context, is_visible, wait_for_alert
 
+from promnesia.logging import LazyLogger
 
 logger = LazyLogger('promnesia-tests', level='debug')
 
 
 from promnesia.common import measure as measure_orig
+
+
 @contextmanager
 def measure(*args, **kwargs):
     kwargs['logger'] = logger
@@ -61,7 +62,7 @@ class Command:
 
 @dataclass
 class Sidebar:
-    addon: 'Addon'
+    addon: Addon
 
     @property
     def driver(self) -> Driver:
@@ -75,7 +76,7 @@ def ctx(self) -> Iterator[WebElement]:
             EC.presence_of_element_located(selector),
         )
 
-        frames = self.driver.find_elements(*selector)
+        # frames = self.driver.find_elements(*selector)
         # TODO uncomment it later when sidebar is injected gracefully...
         # assert len(frames) == 1, frames  # just in case
 
@@ -170,7 +171,7 @@ def configure_extension(
     ) -> None:
         driver = self.helper.driver
 
-        def set_checkbox(cid: str, value: bool) -> None:
+        def set_checkbox(cid: str, value: bool) -> None:  # noqa: FBT001
             cb = driver.find_element(By.ID, cid)
             selected = cb.is_selected()
             if selected != value:
@@ -358,7 +359,7 @@ def wait_for_search_tab(self, cur_window_handles) -> None:
 
 
 @pytest.fixture
-def addon(driver: Driver) -> Iterator[Addon]:
+def addon(driver: Driver) -> Addon:
     addon_source = get_addon_source(kind=driver.name)
     helper = AddonHelper(driver=driver, addon_source=addon_source)
-    yield Addon(helper=helper)
+    return Addon(helper=helper)

tests/addon_helper.py

@@ -1,15 +1,14 @@
-from dataclasses import dataclass
-from functools import cached_property
 import json
-from pathlib import Path
 import re
 import subprocess
+from dataclasses import dataclass
+from functools import cached_property
+from pathlib import Path
 from typing import Any
 
 from loguru import logger
 from selenium import webdriver
-
-from webdriver_utils import is_headless, get_browser_process
+from webdriver_utils import get_browser_process, is_headless
 
 
 @dataclass

tests/common.py

@@ -3,10 +3,11 @@
 import os
 import sys
 import time
+from collections.abc import Iterator
 from contextlib import contextmanager
 from functools import wraps
 from pathlib import Path
-from typing import Iterator, TypeVar
+from typing import TypeVar
 
 import pytest
 import requests
@@ -55,7 +56,7 @@ def local_http_server(path: Path) -> Iterator[str]:
     address = '127.0.0.1'
     with (
         free_port() as port,
-        tmp_popen([sys.executable, '-m', 'http.server', '--directory', path, '--bind', address, str(port)]) as popen,
+        tmp_popen([sys.executable, '-m', 'http.server', '--directory', path, '--bind', address, str(port)]),
     ):
         endpoint = f'http://{address}:{port}'
 

tests/demos.py

@@ -1,24 +1,28 @@
+from __future__ import annotations
+
 from contextlib import contextmanager
-from datetime import timedelta, datetime
+from datetime import datetime, timedelta
 from pathlib import Path
-from time import sleep
 from subprocess import check_call
-from typing import Optional
-
-from promnesia.tests.utils import index_urls
+from time import sleep
 
-from common import uses_x
-from end2end_test import FF, CH, browsers, _test_helper  # type: ignore[attr-defined]
-from end2end_test import confirm
-from end2end_test import configure_extension  # type: ignore[attr-defined]
 from addon_helper import get_window_id
-
-from record import record, hotkeys, CURSOR_SCRIPT, SELECT_SCRIPT
+from common import uses_x
+from end2end_test import (  # type: ignore[attr-defined]
+    CH,
+    FF,
+    _test_helper,
+    browsers,
+    configure_extension,
+    confirm,
+)
+from record import CURSOR_SCRIPT, SELECT_SCRIPT, hotkeys, record
 
 
 def real_db():
-    from private import real_db_path, test_filter  # type: ignore[import-not-found]
     import shutil
+
+    from private import real_db_path, test_filter  # type: ignore[import-not-found]
     def indexer(tdir: Path):
         tdb = tdir / 'promnesia.sqlite'
         # tdb.touch()
@@ -50,7 +54,7 @@ def annotate(self, text: str, length=2) -> None:
         self.l.append((now, text, length))
 
     def build(self, **extra):
-        from pysubs2 import SSAFile, SSAEvent, Color # type: ignore[import-not-found]
+        from pysubs2 import Color, SSAEvent, SSAFile  # type: ignore[import-not-found]
         millis = lambda td: td / timedelta(milliseconds=1)
         subs = (
             SSAEvent(
@@ -156,7 +160,7 @@ def set_geometry(wid: str):
 
             subs = path.with_suffix('.ssa')
             subs.write_text(ann.build(alignment=spos))
-            out  = path.with_suffix('.webm')
+            out  = path.with_suffix('.webm')  # noqa: F841
 
             converter = Path(__file__).parent.absolute() / 'convert_screencast.py'
             check_call([
@@ -283,8 +287,8 @@ def before(driver):
         driver.get(url)
 
     with demo_helper(tmp_path=tmp_path, browser=browser, path=path, before=before) as (helper, annotate):
-        driver = helper.driver
-       
+        driver = helper.driver  # noqa: F841
+
         wait(2)
 
         annotate('''
@@ -445,7 +449,6 @@ def before(driver):
         wait(8)
 
 
-from selenium import webdriver
 
 
 def scroll_to_text(driver, text: str):
@@ -460,7 +463,7 @@ def scroll_to_text(driver, text: str):
 
     driver.execute_script(f'window.scrollTo(0, {y})')
     # TODO a bit of wait??
-   
+
 
 from end2end_test import get_webdriver