diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py
index 0c25914bbf1..e8e30c402c4 100755
--- a/src/_pytest/cacheprovider.py
+++ b/src/_pytest/cacheprovider.py
@@ -139,13 +139,18 @@ def set(self, key, value):
     def _ensure_supporting_files(self):
         """Create supporting files in the cache dir that are not really part of the cache."""
         readme_path = self._cachedir / "README.md"
+        assert not readme_path.exists(), "{} exists already".format(readme_path)
         readme_path.write_text(README_CONTENT)
 
         gitignore_path = self._cachedir.joinpath(".gitignore")
         msg = u"# Created by pytest automatically.\n*"
+        assert not gitignore_path.exists(), "{} exists already".format(gitignore_path)
         gitignore_path.write_text(msg, encoding="UTF-8")
 
         cachedir_tag_path = self._cachedir.joinpath("CACHEDIR.TAG")
+        assert not cachedir_tag_path.exists(), "{} exists already".format(
+            cachedir_tag_path
+        )
         cachedir_tag_path.write_bytes(CACHEDIR_TAG_CONTENT)
 
 
diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py
index 35a7232ab0e..082b097ee91 100644
--- a/testing/test_cacheprovider.py
+++ b/testing/test_cacheprovider.py
@@ -884,7 +884,7 @@ def test_always_passes():
     def test_readme_failed(self, testdir):
         testdir.makepyfile(
             """
-            def test_always_passes():
+            def test_always_fails():
                 assert 0
         """
         )