Partial merge from devel

Pilot/pilotCommands.py

@@ -508,7 +508,7 @@ def execute(self):
         try:
             import json
 
-            resourceDict = json.loads(resourceDict.split("\n")[-1])
+            resourceDict = json.loads(resourceDict.strip().split("\n")[-1])
         except ValueError:
             self.log.error("The pilot command output is not json compatible.")
             sys.exit(1)
@@ -579,7 +579,7 @@ def execute(self):
             self.exitWithError(retCode)
 
         try:
-            result = result.split("\n")[-1].split(" ")
+            result = result.strip().split("\n")[-1].split(" ")
             numberOfProcessorsOnWN = int(result[0])
             maxRAM = int(result[1])
             try:
@@ -853,7 +853,7 @@ def execute(self):
         if retCode:
             self.log.error("There was an error updating the platform [ERROR %d]" % retCode)
             self.exitWithError(retCode)
-        self.log.info("Architecture determined: %s" % localArchitecture.split("\n")[-1])
+        self.log.info("Architecture determined: %s" % localArchitecture.strip().split("\n")[-1])
 
         # standard options
         cfg = ["-FDMH"]  # force update, skip CA checks, skip CA download, skip VOMS
@@ -868,7 +868,7 @@ def execute(self):
             cfg.append("-ddd")
 
         # real options added here
-        localArchitecture = localArchitecture.split("\n")[-1].strip()
+        localArchitecture = localArchitecture.strip().split("\n")[-1].strip()
         cfg.append('-S "%s"' % self.pp.setup)
         cfg.append("-o /LocalSite/Architecture=%s" % localArchitecture)
 

Pilot/pilotTools.py

@@ -17,6 +17,7 @@
 import re
 import signal
 import subprocess
+import select
 from distutils.version import LooseVersion
 
 ############################
@@ -395,18 +396,43 @@ def executeAndGetOutput(self, cmd, environDict=None):
                 "%s" % cmd, shell=True, env=environDict, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=False
             )
 
-            # standard output
-            outData = _p.stdout.read()
-            # always interpret the output as ascii
-            outData = outData.decode("ascii", "replace")
-            # replace any invalid characters with "?" to avoid having unicode output
-            outData = str(outData.replace(u"\ufffd", "?").strip())
-            # write to stdout for debugging
-            sys.stdout.write(outData + "\n")
-
-            for line in _p.stderr:
-                sys.stderr.write(str(line))
+            # simple filter to strip out non-ascii characters
+            def ascii_filter(in_chr):
+                if ord(in_chr) < 128:
+                    return in_chr
+                else:
+                    return ''
+
+            outData = ""
+            isRunning = True
+            while isRunning:
+                readfd, _, _ = select.select([_p.stdout, _p.stderr], [], [])
+                if not readfd:
+                    # not sure if this error is possible
+                    break
+                for stream in readfd:
+                    # ignore codepoint splitting problems; not worth it
+                    outChunk = stream.read(1024).decode("ascii", "replace")
+                    outChunk = ''.join(filter(ascii_filter, outChunk))
+                    if not outChunk:
+                        # file has reached EOF, program finished
+                        isRunning = False
+                        # Finish processing FDs in case there is still some
+                        # remaining data on other file handle...
+                        continue
+                    if stream == _p.stderr:
+                        sys.stderr.write(outChunk)
+                        sys.stderr.flush()
+                    else:
+                        sys.stdout.write(outChunk)
+                        sys.stdout.flush()
+                        outData += outChunk
+
+            # Ensure output ends on a newline
+            sys.stdout.write("\n")
+            sys.stdout.flush()
             sys.stderr.write("\n")
+            sys.stderr.flush()
 
             # return code
             returnCode = _p.wait()