Add sanitizer support on buildbots

master/custom_steps.py

@@ -133,21 +133,26 @@ def run(self):
 class CTest(ShellMixin, CompositeStepMixin, BuildStep):
     name = 'ctest'
 
-    def __init__(self, *, build_config, jobs=None, tests=None, exclude_tests=None, labels=None, exclude_labels=None,
-                 **kwargs):
+    def __init__(self, *, build_config=None, preset=None, jobs=None, tests=None, exclude_tests=None,
+                 labels=None, exclude_labels=None, test_dir=None, **kwargs):
         kwargs['command'] = [
             'ctest',
-            '--build-config', build_config,
             # Note, jobs may be a renderable, don't explicitly convert to str
             *(['--parallel', jobs] if jobs else []),
             *(['--tests-regex', '|'.join(tests)] if tests else []),
             *(['--exclude-regex', '|'.join(exclude_tests)] if exclude_tests else []),
             *(['--label-regex', '|'.join(labels)] if labels else []),
             *(['--label-exclude', '|'.join(exclude_labels)] if exclude_labels else []),
+            *(['--test-dir', test_dir] if test_dir else []),
             '--output-on-failure',
             '--test-action', 'Test',
             '--no-compress-output'
         ]
+        assert (build_config is None) ^ (preset is None), "You must pass either build_config or preset, but not both"
+        if build_config:
+            kwargs['command'] += ['--build-config', build_config]
+        if preset:
+            kwargs['command'] += ['--preset', preset]
 
         kwargs = self.setupShellMixin(kwargs)
         super().__init__(**kwargs)

master/master.cfg

@@ -158,6 +158,10 @@ c['workers'] = [Worker(n,
                        max_builds=cfg.max_builds,
                        properties={'WORKER_BUILD_PARALLELISM': cfg.j}) for n, cfg in _WORKERS]
 
+_SANITIZERS = [
+    'asan'
+]
+
 # LOCKS
 
 # Performance testing requires exclusive use of a worker
@@ -236,6 +240,7 @@ class BuilderType:
        - LLVM branch to be used
        - CMake vs Make
        - main-branch vs testbranch vs llvm-nightly
+       - sanitizers vs none
 
        It doesn't currently include any 'features' because we don't currently
        bake any in at build time.
@@ -245,7 +250,8 @@ class BuilderType:
        setup. (If we ever need to do so, compiler should be added to this.)
     """
 
-    def __init__(self, arch, bits, os, halide_branch, llvm_branch, purpose, buildsystem=BuildSystem.cmake):
+    def __init__(self, arch, bits, os, halide_branch, llvm_branch, purpose, sanitizer=None,
+                 buildsystem=BuildSystem.cmake):
         assert arch in ['arm', 'x86']
         assert bits in [32, 64]
         assert os in ['linux', 'windows', 'osx']
@@ -258,6 +264,7 @@ class BuilderType:
         self.llvm_branch = llvm_branch
         self.buildsystem = buildsystem
         self.purpose = purpose
+        self.sanitizer = sanitizer
 
         if self.halide_branch:
             assert self.purpose != Purpose.llvm_nightly
@@ -267,6 +274,9 @@ class BuilderType:
         else:
             assert self.purpose == Purpose.llvm_nightly
 
+        if self.sanitizer:
+            assert self.sanitizer in _SANITIZERS
+
     # The armbots aren't configured with Python at all,
     # and supporting 32-bit Python on our 64-bit buildbots is painful
     # (and usage of the python bindings on 32-bit hosts is unlikely anyway)
@@ -279,6 +289,21 @@ class BuilderType:
 
         return True
 
+    def handles_sanitizers(self):
+        if self.buildsystem != BuildSystem.cmake:
+            return False
+
+        return (self.arch == 'x86'
+                and self.bits == 64
+                and self.os == 'linux'
+                and self.llvm_branch == LLVM_MAIN)
+
+    def sanitizer_preset(self):
+        if self.handles_sanitizers():
+            return 'linux-x64-asan'
+
+        return None
+
     def handles_hexagon(self):
         return (self.arch == 'x86'
                 and self.bits == 64
@@ -319,6 +344,8 @@ class BuilderType:
         # but is arbitrary. (If changed, manual purging of buildbot temporaries
         # is appropriate)
         a = ['halide']
+        if self.sanitizer:
+            a.append(self.sanitizer)
         if self.purpose == Purpose.halide_testbranch:
             a.append('testbranch')
         a.append(self.halide_branch)
@@ -382,8 +409,11 @@ def get_halide_build_path(*subpaths):
     return get_builddir_subpath(os.path.join('halide-build', *subpaths))
 
 
-def get_halide_install_path(*subpaths):
-    return get_builddir_subpath(os.path.join('halide-install', *subpaths))
+def get_halide_install_path(builder_type, *subpaths):
+    s = 'halide-install'
+    if builder_type.sanitizer:
+        s += '-' + builder_type.sanitizer
+    return get_builddir_subpath(os.path.join(s, *subpaths))
 
 
 def add_get_halide_source_steps(factory, builder_type):
@@ -569,22 +599,47 @@ def get_cmake_generator(builder_type):
     return 'Ninja'
 
 
-def get_cmake_options(builder_type):
+def get_llvm_cmake_options(builder_type):
+    options = []
+    return options
+
+
+def get_halide_cmake_options(builder_type, build_dir):
     options = []
+
+    if builder_type.sanitizer and builder_type.purpose != Purpose.llvm_nightly:
+        assert builder_type.handles_sanitizers()
+        options.append("--preset=%s" % builder_type.sanitizer_preset())
+        # append *after* preset so we override the build dir
+        options += ['-B', build_dir]
+
     return options
 
 
+def get_ctest_options(builder_type, build_dir):
+    if builder_type.sanitizer and builder_type.purpose != Purpose.llvm_nightly:
+        assert builder_type.handles_sanitizers()
+        return {'preset': builder_type.sanitizer_preset(), 'test_dir': build_dir}
+    else:
+        return {'build_config': 'Release'}
+
+
 def get_halide_cmake_definitions(builder_type, halide_target='host', wasm_jit='wabt'):
     cmake_definitions = {
         'Clang_DIR': get_llvm_install_path(builder_type, 'lib/cmake/clang'),
-        'CMAKE_INSTALL_PREFIX': get_halide_install_path(),
-        'CMAKE_BUILD_TYPE': 'Release',
+        'CMAKE_INSTALL_PREFIX': get_halide_install_path(builder_type),
         'Halide_TARGET': halide_target,
         'LLD_DIR': get_llvm_install_path(builder_type, 'lib/cmake/lld'),
         'LLVM_DIR': get_llvm_install_path(builder_type, 'lib/cmake/llvm'),
+        'LLVM_ROOT': get_llvm_install_path(builder_type),
         'WITH_PYTHON_BINDINGS': 'ON' if builder_type.handles_python() else 'OFF'
     }
 
+    if builder_type.sanitizer and builder_type.handles_sanitizers():
+        pass
+    else:
+        cmake_definitions['CMAKE_BUILD_TYPE'] = 'Release'
+
     if builder_type.has_ccache():
         cmake_definitions['Halide_CCACHE_BUILD'] = 'ON'
 
@@ -623,18 +678,7 @@ def get_cmake_build_command(builder_type, build_dir, targets=None):
            '-C', build_dir,
            '-j', Property('WORKER_BUILD_PARALLELISM')]
     if targets:
-        cmd += targets
-
-    # Since everything is using Ninja right now, we just call it directly;
-    # if we ever find the need to go back to bottlenecking thru CMake,
-    # here's the cmd we need:
-    #
-    # cmd = ['cmake',
-    #        '--build', build_dir,
-    #        '--config', 'Release',
-    #        '-j', Property('WORKER_BUILD_PARALLELISM')]
-    # if target:
-    #     cmd.extend(['--target', target])
+        cmd.extend(targets)
 
     return cmd
 
@@ -646,13 +690,16 @@ def get_llvm_cmake_definitions(builder_type):
         'LLVM_BUILD_32_BITS': ('ON' if builder_type.bits == 32 else 'OFF'),
         'LLVM_ENABLE_ASSERTIONS': 'ON',
         'LLVM_ENABLE_LIBXML2': 'OFF',
-        'LLVM_ENABLE_PROJECTS': 'clang;lld',
+        'LLVM_ENABLE_PROJECTS': 'clang;lld;clang-tools-extra',
         'LLVM_ENABLE_RTTI': 'ON',
         'LLVM_ENABLE_TERMINFO': 'OFF',
         'LLVM_ENABLE_ZSTD': 'OFF',
         'LLVM_TARGETS_TO_BUILD': 'X86;ARM;NVPTX;AArch64;Mips;Hexagon;PowerPC;WebAssembly',
     }
 
+    if builder_type.handles_sanitizers():
+        definitions['LLVM_ENABLE_RUNTIMES'] = "compiler-rt;libcxx;libcxxabi;libunwind"
+
     # Some versions of GCC will flood the output with useless warnings about
     # "parameter passing for argument of type foo changed in GCC 7.1" unless
     # we disable this warning. This isn't *essential*, but it makes looking at the
@@ -823,7 +870,7 @@ def add_llvm_steps(factory, builder_type, clean_rebuild):
               path=get_llvm_source_path('llvm'),
               generator=get_cmake_generator(builder_type),
               definitions=get_llvm_cmake_definitions(builder_type),
-              options=get_cmake_options(builder_type)))
+              options=get_llvm_cmake_options(builder_type)))
 
     factory.addStep(
         ShellCommand(name='Build LLVM %s' % llvm_name,
@@ -849,7 +896,7 @@ def add_halide_cmake_build_steps(factory, builder_type):
     # Always do a clean build for Halide
     source_dir = get_halide_source_path()
     build_dir = get_halide_build_path()
-    install_dir = get_halide_install_path()
+    install_dir = get_halide_install_path(builder_type)
     factory.addStep(RemoveDirectory(name="Remove Halide Build Dir",
                                     locks=[performance_lock.access('counting')],
                                     dir=build_dir,
@@ -877,7 +924,7 @@ def add_halide_cmake_build_steps(factory, builder_type):
                           path=source_dir,
                           generator=get_cmake_generator(builder_type),
                           definitions=get_halide_cmake_definitions(builder_type),
-                          options=get_cmake_options(builder_type)))
+                          options=get_halide_cmake_options(builder_type, build_dir)))
 
     factory.addStep(
         ShellCommand(name='Build Halide',
@@ -1039,7 +1086,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
 
     source_dir = get_halide_source_path()
     build_dir = get_halide_build_path()
-    install_dir = get_halide_install_path()  # NOQA
+    install_dir = get_halide_install_path(builder_type)  # NOQA
 
     # Since we need to do at least a partial rebuild for each different target,
     # we want to group things by target. Do host first, followed by a key-sorted
@@ -1085,7 +1132,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
                   generator=get_cmake_generator(builder_type),
                   definitions=get_halide_cmake_definitions(
                       builder_type, halide_target=halide_target, wasm_jit=wasm_jit),
-                  options=get_cmake_options(builder_type)))
+                  options=get_halide_cmake_options(builder_type, build_dir)))
 
         factory.addStep(
             ShellCommand(name='Rebuild',
@@ -1141,10 +1188,10 @@ def add_halide_cmake_test_steps(factory, builder_type):
                       workdir=build_dir,
                       env=env,
                       timeout=3600,
-                      build_config='Release',
                       labels=parallel_test_labels,
                       exclude_tests=exclude_tests,
-                      jobs=parallelism))
+                      jobs=parallelism,
+                      **get_ctest_options(builder_type, build_dir)))
 
         if exclusive_test_labels:
             test_set = ', '.join(exclusive_test_labels)
@@ -1155,16 +1202,16 @@ def add_halide_cmake_test_steps(factory, builder_type):
                       workdir=build_dir,
                       env=env,
                       timeout=3600,
-                      build_config='Release',
-                      labels=exclusive_test_labels))
+                      labels=exclusive_test_labels,
+                      **get_ctest_options(builder_type, build_dir)))
 
         if do_apps:
             apps_build_dir = get_halide_build_path("apps")
             apps_source_dir = get_halide_source_path("apps")
 
             # We currently don't attempt to build any of the apps with wasm
             apps_cmake_defs = get_halide_cmake_definitions(builder_type, halide_target=halide_target)
-            apps_cmake_defs['CMAKE_PREFIX_PATH'] = get_halide_install_path()
+            apps_cmake_defs['CMAKE_PREFIX_PATH'] = get_halide_install_path(builder_type)
 
             # apps/hannk is expensive to build, and doesn't (yet) build on all systems, so special-case it here
             apps_cmake_defs['ENABLE_APPS_HANNK'] = 'ON' if builder_type.has_tflite() else 'OFF'
@@ -1179,7 +1226,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
                       path=apps_source_dir,
                       generator=get_cmake_generator(builder_type),
                       definitions=apps_cmake_defs,
-                      options=get_cmake_options(builder_type)))
+                      options=get_halide_cmake_options(builder_type, build_dir)))
 
             factory.addStep(
                 ShellCommand(name='Build apps for %s' % desc,
@@ -1207,9 +1254,9 @@ def add_halide_cmake_test_steps(factory, builder_type):
                       workdir=apps_build_dir,
                       env=env,
                       timeout=3600,
-                      build_config='Release',
                       exclude_tests=exclude_tests,
-                      exclude_labels=['slow_tests']))
+                      exclude_labels=['slow_tests'],
+                      **get_ctest_options(builder_type, apps_build_dir)))
 
 
 def create_halide_make_factory(builder_type):
@@ -1322,20 +1369,30 @@ def get_interesting_halide_targets():
 
 
 def create_halide_builder(arch, bits, os, halide_branch, llvm_branch, purpose, buildsystem=BuildSystem.cmake):
-    builder_type = BuilderType(arch, bits, os, halide_branch, llvm_branch, purpose, buildsystem)
-    workers = builder_type.get_worker_names()
-    builder = BuilderConfig(name=builder_type.builder_label(),
-                            workernames=workers,
-                            factory=create_halide_factory(builder_type),
-                            collapseRequests=True,
-                            # We need counting access to our llvm branch during Halide builds.
-                            # (We could probably get by with access during only a subset of
-                            # our steps, but there doesn't appear to be a way to group
-                            # lock requests across multiple-but-not-all-steps in a Build.)
-                            locks=[llvm_build_locks[llvm_branch + str(bits)].access('counting')],
-                            tags=builder_type.builder_tags())
-    builder.builder_type = builder_type
-    return builder
+    # TODO: sanitizer build/test disabling pending investigation of
+    # https://discourse.cmake.org/t/ctest-preset-doesnt-appear-to-allow-for-out-of-tree-builds/6275,
+    # which limits the ability of CTest to work properly with out-of-tree builds when using presets
+    # for san in _SANITIZERS + [None]:
+    for san in [None]:
+        builder_type = BuilderType(arch, bits, os, halide_branch, llvm_branch, purpose, san, buildsystem)
+        if san and purpose == Purpose.llvm_nightly:
+            continue
+        if san and not builder_type.handles_sanitizers():
+            continue
+
+        workers = builder_type.get_worker_names()
+        builder = BuilderConfig(name=builder_type.builder_label(),
+                                workernames=workers,
+                                factory=create_halide_factory(builder_type),
+                                collapseRequests=True,
+                                # We need counting access to our llvm branch during Halide builds.
+                                # (We could probably get by with access during only a subset of
+                                # our steps, but there doesn't appear to be a way to group
+                                # lock requests across multiple-but-not-all-steps in a Build.)
+                                locks=[llvm_build_locks[llvm_branch + str(bits)].access('counting')],
+                                tags=builder_type.builder_tags())
+        builder.builder_type = builder_type
+        yield builder
 
 
 def create_halide_builders():
@@ -1344,27 +1401,31 @@ def create_halide_builders():
         # (but only against their 'native' LLVM versions)
         for halide_branch in HALIDE_BRANCHES:
             for llvm_branch in LLVM_FOR_HALIDE[halide_branch]:
-                yield create_halide_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.halide_main)
+                yield from create_halide_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.halide_main)
 
         # Create the builders for testing pull requests to releases.
         for halide_branch in _HALIDE_RELEASES:
             for llvm_branch in LLVM_FOR_HALIDE[halide_branch]:
-                yield create_halide_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.halide_testbranch)
+                yield from create_halide_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.halide_testbranch)
 
         # Create the builders for testing pull requests to main.
-        yield create_halide_builder(arch, bits, os, HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch)
+        yield from create_halide_builder(arch, bits, os, HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch)
 
     # Also test Makefiles on x86-linux & osx (but only on Halide main) to ensure they
     # stay healthy. (Note: deliberately skip arm-linux, since they are the slowest bots.)
-    yield create_halide_builder('x86', 64, 'linux', HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch, BuildSystem.make)
-    yield create_halide_builder('x86', 32, 'linux', HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch, BuildSystem.make)
-    yield create_halide_builder('x86', 64, 'osx', HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch, BuildSystem.make)
-    yield create_halide_builder('arm', 64, 'osx', HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch, BuildSystem.make)
+    yield from create_halide_builder('x86', 64, 'linux', HALIDE_MAIN, LLVM_MAIN,
+                                     Purpose.halide_testbranch, BuildSystem.make)
+    yield from create_halide_builder('x86', 32, 'linux', HALIDE_MAIN, LLVM_MAIN,
+                                     Purpose.halide_testbranch, BuildSystem.make)
+    yield from create_halide_builder('x86', 64, 'osx', HALIDE_MAIN, LLVM_MAIN,
+                                     Purpose.halide_testbranch, BuildSystem.make)
+    yield from create_halide_builder('arm', 64, 'osx', HALIDE_MAIN, LLVM_MAIN,
+                                     Purpose.halide_testbranch, BuildSystem.make)
 
     # Test pull requests for Halide master against the current and previous LLVM, for at least one target.
     for llvm_branch in LLVM_BRANCHES:
         if abs(LLVM_BRANCHES[llvm_branch].version.major - LLVM_BRANCHES[LLVM_MAIN].version.major) in [1, 2]:
-            yield create_halide_builder('x86', 64, 'linux', HALIDE_MAIN, llvm_branch, Purpose.halide_testbranch)
+            yield from create_halide_builder('x86', 64, 'linux', HALIDE_MAIN, llvm_branch, Purpose.halide_testbranch)
 
 
 def create_halide_scheduler(halide_branch):