diff --git a/docs/en_us/internal/testing.md b/docs/en_us/internal/testing.md
index 7d05c5f46f0d..afd24f665174 100644
--- a/docs/en_us/internal/testing.md
+++ b/docs/en_us/internal/testing.md
@@ -404,6 +404,10 @@ More specific options are below.
 
 		paver run_quality --percentage=100
 
+* Note that 'fixme' violations are not counted with run_quality. To see all 'TODO' lines, use:
+
+		paver find_fixme --system=lms
+`system` is an optional argument here. It defaults to `cms,lms,common`.
 
 
 ## Testing using queue servers
diff --git a/pavelib/quality.py b/pavelib/quality.py
index 7ad4d76316b7..8d3cd49a5745 100644
--- a/pavelib/quality.py
+++ b/pavelib/quality.py
@@ -8,6 +8,53 @@
 from .utils.envs import Env
 
 
+@task
+@needs('pavelib.prereqs.install_python_prereqs')
+@cmdopts([
+    ("system=", "s", "System to act on"),
+])
+def find_fixme(options):
+    """
+    Run pylint on system code, only looking for fixme items.
+    """
+    num_fixme = 0
+    systems = getattr(options, 'system', 'lms,cms,common').split(',')
+
+    for system in systems:
+        # Directory to put the pylint report in.
+        # This makes the folder if it doesn't already exist.
+        report_dir = (Env.REPORT_DIR / system).makedirs_p()
+
+        apps = [system]
+
+        for directory in ['djangoapps', 'lib']:
+            dirs = os.listdir(os.path.join(system, directory))
+            apps.extend([d for d in dirs if os.path.isdir(os.path.join(system, directory, d))])
+
+        apps_list = ' '.join(apps)
+
+        pythonpath_prefix = (
+            "PYTHONPATH={system}:{system}/djangoapps:{system}/"
+            "lib:common/djangoapps:common/lib".format(
+                system=system
+            )
+        )
+
+        sh(
+            "{pythonpath_prefix} pylint --disable R,C,W,E --enable=fixme "
+            "-f parseable {apps} | tee {report_dir}/pylint_fixme.report".format(
+                pythonpath_prefix=pythonpath_prefix,
+                apps=apps_list,
+                report_dir=report_dir
+            )
+        )
+
+        num_fixme += _count_pylint_violations(
+            "{report_dir}/pylint_fixme.report".format(report_dir=report_dir))
+
+    print("Number of pylint fixmes: " + str(num_fixme))
+
+
 @task
 @needs('pavelib.prereqs.install_python_prereqs')
 @cmdopts([
diff --git a/scripts/all-tests.sh b/scripts/all-tests.sh
index d0b2d38eff8d..cb4f502eb06f 100755
--- a/scripts/all-tests.sh
+++ b/scripts/all-tests.sh
@@ -107,6 +107,7 @@ SHARD=${SHARD:="all"}
 case "$TEST_SUITE" in
 
     "quality")
+        paver find_fixme > fixme.log || { cat fixme.log; EXIT=1; }
         paver run_pep8 -l $PEP8_THRESHOLD > pep8.log || { cat pep8.log; EXIT=1; }
         paver run_pylint -l $PYLINT_THRESHOLD > pylint.log || { cat pylint.log; EXIT=1; }
         # Run quality task. Pass in the 'fail-under' percentage to diff-quality