diff --git a/docs/source/changes.md b/docs/source/changes.md
index 893eef590..0241a5396 100644
--- a/docs/source/changes.md
+++ b/docs/source/changes.md
@@ -16,6 +16,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 - {pull}`627` adds a warning when users explicitly pass files to pytask that pytask is
   going to ignore because they do not match a pattern. Happens quite often when the task
   module's name does not start with `task_`.
+- {pull}`628` fixes duplicated collection of task modules. Fixes {issue}`624`. Thanks to
+  {user}`timmens` for the issue.
 
 ## 0.5.0 - 2024-05-26
 
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 05e237cbb..006f2e432 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -94,7 +94,7 @@ def _collect_from_paths(session: Session) -> None:
     Go through all paths, check if the path is ignored, and collect the file if not.
 
     """
-    for path in _not_ignored_paths(session.config["paths"], session):
+    for path in _not_ignored_paths(session.config["paths"], session, set()):
         reports = session.hook.pytask_collect_file_protocol(
             session=session, path=path, reports=session.collection_reports
         )
@@ -519,7 +519,7 @@ def _raise_error_if_casing_of_path_is_wrong(
 
 
 def _not_ignored_paths(
-    paths: Iterable[Path], session: Session
+    paths: Iterable[Path], session: Session, seen: set[Path]
 ) -> Generator[Path, None, None]:
     """Traverse paths and yield not ignored paths.
 
@@ -532,8 +532,9 @@ def _not_ignored_paths(
         if not session.hook.pytask_ignore_collect(path=path, config=session.config):
             if path.is_dir():
                 files_in_dir = path.iterdir()
-                yield from _not_ignored_paths(files_in_dir, session)
-            else:
+                yield from _not_ignored_paths(files_in_dir, session, seen)
+            elif path not in seen:
+                seen.add(path)
                 yield path
 
 
diff --git a/tests/test_collect.py b/tests/test_collect.py
index b8ec63a65..b05a06794 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -99,6 +99,16 @@ def test_collect_same_task_different_ways(tmp_path, path_extension):
     assert len(session.tasks) == 1
 
 
+def test_modules_are_not_collected_twice(runner, tmp_path):
+    """See #624."""
+    tmp_path.joinpath("task_module.py").write_text("def task_example(): pass")
+    tmp_path.joinpath("pyproject.toml").write_text(
+        "[tool.pytask.ini_options]\npaths = ['.', '.']"
+    )
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert "Collected 1 task" in result.output
+
+
 @pytest.mark.end_to_end()
 @pytest.mark.parametrize(
     ("task_files", "pattern", "expected_collected_tasks"),