indexer: more consistent and friendly error handling in auto indexer

Author: karlicoss

src/promnesia/common.py

@@ -301,9 +301,16 @@ def last(path: PathIsh, *parts: str) -> Path:
 
 from .logging import setup_logger
 
+from copy import copy
 def echain(ex: Exception, cause: Exception) -> Exception:
-    ex.__cause__ = cause
-    return ex
+    e = copy(ex)
+    e.__cause__ = cause
+    # right.. even if we attach cause it doesn't help much because when we return/yield exception, we lose the stacktrace
+    # so only 'ex' gets logged, cause is completely lost
+    # hopefully this is safe? at least on runtimeerrors
+    # might also do something smarter and collapes if both are strings or something..
+    e.args += cause.args
+    return e
 
 
 def slugify(x: str) -> str:

src/promnesia/sources/auto.py

@@ -17,7 +17,7 @@
 
 import pytz
 
-from ..common import Visit, Url, PathIsh, get_logger, Loc, get_tmpdir, extract_urls, Extraction, Results, mime, traverse, file_mtime
+from ..common import Visit, Url, PathIsh, get_logger, Loc, get_tmpdir, extract_urls, Extraction, Results, mime, traverse, file_mtime, echain, logger
 from ..config import use_cores
 from ..py37 import nullcontext
 
@@ -91,12 +91,10 @@ def wrapped(path: Path):
                 # ugh. keeping yeild in the try section is not ideal, but need this because of lazy yield semantics
                 yield from it
             except ModuleNotFoundError as me:
-                # TODO ugh. need to figure out how to attach cause/traceback
                 logger = get_logger()
                 logger.exception(me)
-                logger.warn('while indexing "%s": %s not found, falling back to grep! "pip3 install --user %s" for better support!', path, me.name, me.name)
+                logger.warn('%s: %s not found, falling back to grep! "pip3 install --user %s" for better support!', path, me.name, me.name)
                 yield me
-
                 fallback_active[ex] = True
                 do_fallback = True
         if do_fallback:
@@ -276,26 +274,35 @@ def _index_file(pp: Path, opts: Options) -> Results:
         yield from _index(path=uncomp, opts=opts)
         return
 
+    ex = RuntimeError(f'While indexing {pp}')
+
     ip, pm = by_path(pp)
     if ip is None:
         # TOOD use warning (with mime/ext as key?)
         # TODO only log once? # hmm..
         msg = f'No extractor for suffix {suf}, mime {pm}'
         warnings.warn(msg)
-        yield RuntimeError(msg + f', path {pp}')
+        yield echain(ex, RuntimeError(msg))
         return
 
     logger.debug('indexing via %s: %s', ip.__name__, pp)
-    indexer: Union[Urls, Results] = ip(pp) # type: ignore
-    # TODO careful, filter out obviously not plaintext? maybe mime could help here??
+
+    def indexer() -> Union[Urls, Results]:
+        # eh, annoying.. need to make more generic..
+        idx = ip(pp) # type: ignore
+        try:
+            yield from idx
+        except Exception as e:
+            yield e
 
     root = opts.root
     fallback_dt = file_mtime(pp)
     fallback_loc = Loc.file(pp)
     replacer = opts.replacer
-    for r in indexer:
+    for r in indexer():
         if isinstance(r, Exception):
-            yield r
+            # indexers can rely on this method setting the error context
+            yield echain(ex, r)
             continue
         if isinstance(r, EUrl):
             v = Visit(

src/promnesia/sources/markdown.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Iterator, NamedTuple, Optional
 
-from ..common import get_logger, Extraction, Url, PathIsh, Res, Visit, echain, Loc, file_mtime
+from ..common import get_logger, Extraction, Url, PathIsh, Res, Visit, Loc, file_mtime, logger
 
 
 import mistletoe # type: ignore
@@ -72,7 +72,7 @@ def _walk(self, cur, last_block) -> Iterator[Result]:
         try:
             yield from self._extract(cur, last_block)
         except Exception as e:
-            # TODO log context??
+            logger.exception(e)
             yield e
 
         children = getattr(cur, 'children', [])
@@ -88,12 +88,10 @@ def extract_from_file(fname: PathIsh) -> Iterator[Extraction]:
     path = Path(fname)
     fallback_dt = file_mtime(path)
 
-    ex = RuntimeError(f'while extracting from {path}')
-
     p = Parser(path)
     for r in p.walk():
         if isinstance(r, Exception):
-            yield echain(ex, r)
+            yield r
         else:
             yield Visit(
                 url=r.url,

src/promnesia/sources/org.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 
 
-from ..common import Visit, get_logger, Results, Url, Loc, from_epoch, echain, iter_urls, PathIsh, Res, file_mtime
+from ..common import Visit, get_logger, Results, Url, Loc, from_epoch, iter_urls, PathIsh, Res, file_mtime
 
 
 import orgparse
@@ -130,11 +130,9 @@ def extract_from_file(fname: PathIsh) -> Results:
 
     fallback_dt = file_mtime(path)
 
-    ex = RuntimeError(f'while extracting from {fname}')
-
     for wr in walk_node(node=root, dt=fallback_dt):
         if isinstance(wr, Exception):
-            yield echain(ex, wr)
+            yield wr
             continue
 
         (parsed, node) = wr
@@ -148,7 +146,7 @@ def extract_from_file(fname: PathIsh) -> Results:
                 # TODO not sure... perhaps keep the whole heading intact? unclear how to handle file tags though
                 ctx = parsed.heading + tagss + '\n' + get_body_compat(node)
             except Exception as e:
-                yield echain(ex, e)
+                yield e
                 ctx = 'ERROR' # TODO more context?
 
             if isinstance(r, Url):
@@ -162,4 +160,4 @@ def extract_from_file(fname: PathIsh) -> Results:
                     context=ctx,
                 )
             else: # error
-                yield echain(ex, r)
+                yield r