Make the tests pass on windows.

The main points of note are:
 - Use temporary_filename() to get a filename without any open handles (Windows will not let you unlink a file that has open handles).
 - Get rid of NamedTemporaryFile/temporary_file in lieu of using our helper in util.
 - Pexs aren't executable on windows directly - run them as `python ...`
 - Skip soft/hard link tests on windows
 - Small fixes w.r.t. / vs \ and s/\r//g

Author: mikekap

pex/testing.py

@@ -5,6 +5,7 @@
 import os
 import random
 import subprocess
+import sys
 import tempfile
 import zipfile
 from textwrap import dedent
@@ -13,7 +14,7 @@
 from .compatibility import nested
 from .installer import EggInstaller, Packager
 from .pex_builder import PEXBuilder
-from .util import DistributionHelper
+from .util import DistributionHelper, named_temporary_file
 
 
 @contextlib.contextmanager
@@ -25,6 +26,19 @@ def temporary_dir():
     safe_rmtree(td)
 
 
+@contextlib.contextmanager
+def temporary_filename():
+  """Creates a temporary filename.
+
+  This is useful when you need to pass a filename to an API. Windows requires all
+  handles to a file be closed before deleting/renaming it, so this makes it a bit
+  simpler."""
+  with named_temporary_file() as fp:
+    fp.write(b'')
+    fp.close()
+    yield fp.name
+
+
 def random_bytes(length):
   return ''.join(
       map(chr, (random.randint(ord('a'), ord('z')) for _ in range(length)))).encode('utf-8')
@@ -162,12 +176,12 @@ def write_simple_pex(td, exe_contents, dists=None, coverage=False):
 # TODO(wickman) Why not PEX.run?
 def run_simple_pex(pex, args=(), env=None):
   po = subprocess.Popen(
-      [pex] + list(args),
+      [sys.executable, pex] + list(args),
       stdout=subprocess.PIPE,
       stderr=subprocess.STDOUT,
       env=env)
   po.wait()
-  return po.stdout.read(), po.returncode
+  return po.stdout.read().replace(b'\r', b''), po.returncode
 
 
 def run_simple_pex_test(body, args=(), env=None, dists=None, coverage=False):

tests/test_environment.py

@@ -4,13 +4,13 @@
 import os
 from contextlib import contextmanager
 
-from twitter.common.contextutil import temporary_dir, temporary_file
+from twitter.common.contextutil import temporary_dir
 
 from pex.compatibility import nested
 from pex.environment import PEXEnvironment
 from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
-from pex.testing import make_bdist
+from pex.testing import make_bdist, temporary_filename
 
 
 @contextmanager
@@ -22,64 +22,65 @@ def yield_pex_builder(zip_safe=True):
 
 
 def test_force_local():
-  with nested(yield_pex_builder(), temporary_dir(), temporary_file()) as (pb, pex_root, pex_file):
+  with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (
+          pb, pex_root, pex_file):
     pb.info.pex_root = pex_root
-    pb.build(pex_file.name)
+    pb.build(pex_file)
 
-    code_cache = PEXEnvironment.force_local(pex_file.name, pb.info)
+    code_cache = PEXEnvironment.force_local(pex_file, pb.info)
     assert os.path.exists(pb.info.zip_unsafe_cache)
     assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
     assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
     assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py', '__main__.pyc'])
 
     # idempotence
-    assert PEXEnvironment.force_local(pex_file.name, pb.info) == code_cache
+    assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
 
 
 def normalize(path):
-  return os.path.normpath(os.path.realpath(path))
+  return os.path.normpath(os.path.realpath(path)).lower()
 
 
 def test_write_zipped_internal_cache():
   # zip_safe pex will not be written to install cache unless always_write_cache
-  with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
+  with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_filename()) as (
       pb, pex_root, pex_file):
 
     pb.info.pex_root = pex_root
-    pb.build(pex_file.name)
+    pb.build(pex_file)
 
-    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
+    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
     assert len(zip_safe) == 1
     assert normalize(zip_safe[0].location).startswith(
-        normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
+        normalize(os.path.join(pex_file, pb.info.internal_cache))), (
             'loc: %s, cache: %s' % (
                 normalize(zip_safe[0].location),
-                normalize(os.path.join(pex_file.name, pb.info.internal_cache))))
+                normalize(os.path.join(pex_file, pb.info.internal_cache))))
 
     pb.info.always_write_cache = True
-    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
+    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
     assert len(new) == 1
     assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
 
     # Check that we can read from the cache
-    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
+    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
     assert len(existing) == 1
     assert normalize(existing[0].location).startswith(normalize(pb.info.install_cache))
 
   # non-zip_safe pex will be written to install cache
-  with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
+  with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_filename()) as (
       pb, pex_root, pex_file):
 
     pb.info.pex_root = pex_root
-    pb.build(pex_file.name)
+    pb.build(pex_file)
 
-    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
+    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
     assert len(new) == 1
     assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
     original_location = normalize(new[0].location)
 
     # do the second time to validate idempotence of caching
-    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
+    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
     assert len(existing) == 1
     assert normalize(existing[0].location) == original_location
 

tests/test_http.py

@@ -6,11 +6,11 @@
 from io import BytesIO
 
 import pytest
-from twitter.common.contextutil import temporary_file
 
 from pex.compatibility import PY2
 from pex.http import Context, RequestsContext, StreamFilelike, UrllibContext
 from pex.link import Link
+from pex.util import named_temporary_file
 from pex.variables import Variables
 
 try:
@@ -107,7 +107,7 @@ def test_requests_context():
       assert fp.read() == BLOB
 
   # test local reading
-  with temporary_file() as tf:
+  with named_temporary_file() as tf:
     tf.write(b'goop')
     tf.flush()
     assert context.read(Link.wrap(tf.name)) == b'goop'
@@ -225,7 +225,7 @@ def test_requests_context_retries_read_timeout_retries_exhausted():
 def test_urllib_context_utf8_encoding():
   BYTES = b'this is a decoded utf8 string'
 
-  with temporary_file() as tf:
+  with named_temporary_file() as tf:
     tf.write(BYTES)
     tf.flush()
     local_link = Link.wrap(tf.name)

tests/test_integration.py

@@ -4,9 +4,12 @@
 import os
 import sys
 
-from twitter.common.contextutil import environment_as, temporary_dir, temporary_file
+import pytest
+from twitter.common.contextutil import environment_as, temporary_dir
 
+from pex.compatibility import WINDOWS
 from pex.testing import run_simple_pex_test
+from pex.util import named_temporary_file
 
 
 def test_pex_execute():
@@ -21,7 +24,7 @@ def test_pex_raise():
 
 
 def test_pex_interpreter():
-  with temporary_file() as fp:
+  with named_temporary_file() as fp:
     fp.write(b"print('Hello world')")
     fp.flush()
 
@@ -33,6 +36,7 @@ def test_pex_interpreter():
     assert rc == 0
 
 
+@pytest.mark.skipif(WINDOWS, reason='No symlinks on windows')
 def test_pex_python_symlink():
   with temporary_dir() as td:
     with environment_as(HOME=td):

tests/test_pex.py

@@ -9,7 +9,7 @@
 import pytest
 from twitter.common.contextutil import temporary_dir
 
-from pex.compatibility import nested, to_bytes
+from pex.compatibility import WINDOWS, nested, to_bytes
 from pex.installer import EggInstaller, WheelInstaller
 from pex.pex import PEX
 from pex.testing import make_installer, run_simple_pex_test
@@ -137,6 +137,7 @@ def test_site_libs():
     assert site_packages in site_libs
 
 
+@pytest.mark.skipif(WINDOWS, reason='No symlinks on windows')
 def test_site_libs_symlink():
   with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
           mock_site_packages, tempdir):

tests/test_pex_builder.py

@@ -6,10 +6,11 @@
 import zipfile
 from contextlib import closing
 
+import pytest
 from twitter.common.contextutil import temporary_dir
 from twitter.common.dirutil import safe_mkdir
 
-from pex.compatibility import nested
+from pex.compatibility import WINDOWS, nested
 from pex.pex import PEX
 from pex.pex_builder import PEXBuilder
 from pex.testing import write_simple_pex as write_pex
@@ -102,6 +103,7 @@ def build_and_check(path, precompile):
     build_and_check(td3, True)
 
 
+@pytest.mark.skipif(WINDOWS, reason='No hardlinks on windows')
 def test_pex_builder_copy_or_link():
   with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
     src = os.path.join(td1, 'exe.py')

tests/test_pex_info.py

@@ -1,6 +1,8 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import os.path
+
 import pytest
 
 from pex.orderedset import OrderedSet
@@ -46,15 +48,16 @@ def test_from_empty_env():
 
 
 def test_from_env():
-  environ = dict(PEX_ROOT='/pex_root',
+  pex_root = os.path.realpath('/pex_root')
+  environ = dict(PEX_ROOT=pex_root,
                  PEX_MODULE='entry:point',
                  PEX_SCRIPT='script.sh',
                  PEX_FORCE_LOCAL='true',
                  PEX_INHERIT_PATH='true',
                  PEX_IGNORE_ERRORS='true',
                  PEX_ALWAYS_CACHE='true')
 
-  info = dict(pex_root='/pex_root',
+  info = dict(pex_root=pex_root,
               entry_point='entry:point',
               script='script.sh',
               zip_safe=False,

tests/test_util.py

@@ -4,18 +4,17 @@
 import contextlib
 import functools
 import os
-import subprocess
 import zipfile
 from hashlib import sha1
 from textwrap import dedent
 
-from twitter.common.contextutil import temporary_dir, temporary_file
+from twitter.common.contextutil import temporary_dir
 
 from pex.common import safe_mkdir
 from pex.compatibility import nested
 from pex.installer import EggInstaller, WheelInstaller
 from pex.pex_builder import PEXBuilder
-from pex.testing import make_bdist, temporary_content, write_zipfile
+from pex.testing import make_bdist, run_simple_pex, temporary_content, write_zipfile
 from pex.util import CacheHelper, DistributionHelper, named_temporary_file
 
 try:
@@ -27,17 +26,17 @@
 def test_hash():
   empty_hash = sha1().hexdigest()
 
-  with temporary_file() as fp:
+  with named_temporary_file() as fp:
     fp.flush()
     assert empty_hash == CacheHelper.hash(fp.name)
 
-  with temporary_file() as fp:
+  with named_temporary_file() as fp:
     string = b'asdf' * 1024 * sha1().block_size + b'extra padding'
     fp.write(string)
     fp.flush()
     assert sha1(string).hexdigest() == CacheHelper.hash(fp.name)
 
-  with temporary_file() as fp:
+  with named_temporary_file() as fp:
     empty_hash = sha1()
     fp.write(b'asdf')
     fp.flush()
@@ -59,7 +58,7 @@ def test_hash_consistency():
   for reverse in (False, True):
     with temporary_content(CONTENT) as td:
       dir_hash = CacheHelper.dir_hash(td)
-      with temporary_file() as tf:
+      with named_temporary_file() as tf:
         write_zipfile(td, tf.name, reverse=reverse)
         with contextlib.closing(zipfile.ZipFile(tf.name, 'r')) as zf:
           zip_hash = CacheHelper.zip_hash(zf)
@@ -147,18 +146,13 @@ def test_access_zipped_assets_integration():
     pex = os.path.join(td2, 'app.pex')
     pb.build(pex)
 
-    po = subprocess.Popen(
-        [pex],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT)
-    po.wait()
-    output = po.stdout.read()
+    output, returncode = run_simple_pex(pex)
     try:
       output = output.decode('UTF-8')
     except ValueError:
       pass
     assert output == 'accessed\n'
-    assert po.returncode == 0
+    assert returncode == 0
 
 
 def test_named_temporary_file():

tests/test_variables.py

@@ -1,10 +1,9 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import tempfile
-
 import pytest
 
+from pex.util import named_temporary_file
 from pex.variables import Variables
 
 
@@ -85,23 +84,23 @@ def test_pex_get_kv():
 
 
 def test_pex_from_rc():
-  with tempfile.NamedTemporaryFile(mode='w') as pexrc:
+  with named_temporary_file(mode='w') as pexrc:
     pexrc.write('HELLO=42')
     pexrc.flush()
     v = Variables(rc=pexrc.name)
     assert v._get_int('HELLO') == 42
 
 
 def test_pexrc_precedence():
-  with tempfile.NamedTemporaryFile(mode='w') as pexrc:
+  with named_temporary_file(mode='w') as pexrc:
     pexrc.write('HELLO=FORTYTWO')
     pexrc.flush()
     v = Variables(environ={'HELLO': 42}, rc=pexrc.name)
     assert v._get_int('HELLO') == 42
 
 
 def test_rc_ignore():
-  with tempfile.NamedTemporaryFile(mode='w') as pexrc:
+  with named_temporary_file(mode='w') as pexrc:
     pexrc.write('HELLO=FORTYTWO')
     pexrc.flush()
     v = Variables(environ={'PEX_IGNORE_RCFILES': 'True'}, rc=pexrc.name)