buildbot2.cfg

# -*- python -*-
# ex: set syntax=python:

from socket import gethostname, getfqdn
from time import strftime, time, gmtime
from re import compile
from twisted.python import log
from twisted.internet.defer import inlineCallbacks, returnValue
from buildbot.schedulers import basic, timed, forcesched
from buildbot.schedulers.filter import ChangeFilter
from buildbot.changes.pb import PBChangeSource
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.triggerable import Triggerable
from buildbot.status.web.grid import TransposedGridStatusResource
from buildbot.status.words import IRC
from buildbot.status.html import WebStatus
from buildbot.status.web import authz, auth
from buildbot.status.web.base import HtmlResource
from buildbot.status.web.grid import GridStatusMixin, ANYBRANCH
from buildbot.steps.shell import Compile
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand, SetPropertyFromCommand
from buildbot.steps.slave import MakeDirectory
from buildbot.steps.trigger import Trigger
from buildbot.process.properties import Interpolate
from buildbot.config import BuilderConfig
from buildbot.process import factory
from buildbot.process.properties import WithProperties, Property
from buildbot.process.buildstep import LogLineObserver, LoggingBuildStep, SUCCESS, FAILURE, WARNINGS
from buildbot.sourcestamp import SourceStamp
from buildbot.buildslave import BuildSlave
from buildbot.locks import MasterLock
from json import load
from math import floor

XT_TAG = 'XT_Tag'
XT_POLL = 'XT_Poll'
XT_WINDOWS = 'XT_Windows'
XT_DOCS = 'XT_docs'
XT_INSTALLER = 'XT_installer'
XT_SYNCXT = 'XT_syncXT'
XT_DEBIANREPO = 'XT_debianrepo'

c = BuildmasterConfig = {}
c['properties'] = {'git_base':'git://git.xci-test.com',
                   'site':'cam', 
                   'inspection_branch':gethostname().split('.')[0] if gethostname().startswith('buildbot-dev') else 'master',
                   'build_user':'1004', # local user on rsync.cam.xci-test.com
                   'build_user_name':'build', # local user for squeeze32 slaves
                   'build_type': 'oeprod' if gethostname() =='buildbot2' else 'oetest',
                   'rsync_build_destination':'build@rsync.cam.xci-test.com:',
                   'reference_repository':'/home/xc_source/git/xenclient/build-scripts.git',
                   'git_ssh_user': 'git',
                   'master_download_cache':'/home/xc_dist/oe/oe-download',
                   'windows_build_staging' : '/home/xc_buildoutput/xenclient-windows',
                   'central_build_directory' : '/home/xc_builds',
                   'git_ssh_server': 'git.cam.xci-test.com',
                   }

####### SCHEDULERS

## configure the Schedulers

def tag_filter(branch):
    return branch != '(tag)'
def force(builder):
    return [forcesched.ForceScheduler(name='force_'+builder, builderNames=[builder])]
def force_and_trigger(builder):
    return force(builder)+[Triggerable(name='trigger_'+builder, builderNames=[builder])]

c['schedulers'] = []
# the branch parameter is used to record which changesets affect the branch; but we don't use
# changeset so can pass branch as None
# (not specifying branch causes an error because Nightly is overly fussy)
c['schedulers'].append(timed.Nightly(name='pollXT', builderNames=[XT_POLL], minute=0, branch=Property('inspection_branch')))
c['schedulers'].append(AnyBranchScheduler(name='runXT', treeStableTimer=5*60, builderNames=[XT_POLL], change_filter=ChangeFilter(branch_fn=tag_filter)))
c['schedulers'].append(forcesched.ForceScheduler(name='force_'+XT_POLL, builderNames=[XT_POLL]))
c['schedulers'].append(Triggerable(name='trigger_'+XT_POLL, builderNames=[XT_POLL]))

c['schedulers'] += force_and_trigger(XT_TAG)
PRIMARY_BUILDS = [XT_WINDOWS, XT_DOCS, XT_SYNCXT]
for build in PRIMARY_BUILDS:
    c['schedulers'] += force(build)
complete_trigger = Triggerable(name='trigger_complete_XT_build', builderNames=PRIMARY_BUILDS)
c['schedulers'].append(complete_trigger)
####### DB URL
c['db_url'] = "sqlite:///state.sqlite"
####### BUILDSLAVES
NOTIFY_ON_MISSING = ['someone@somwhere.com']
if gethostname().startswith('buildbot-dev'):
    SQUEEZE32_SLAVES = ['scratch2']
    WINDOWS_SLAVES = [ 'cbgwinbuild04']
    CENTOS_SLAVES = ['cbgcentosbuild02']
else:
    SQUEEZE32_SLAVES = ['gripper', 'scratch3', 'scratch4', 'scratch5', 'scratch6', 'scratch7', 'scratch8', 'scratch9', 'scoot']
    WINDOWS_SLAVES = ['cbgwinbuild04', 'cbgwinnative01']
    CENTOS_SLAVES = ['cbgcentosbuild01']
SLAVES = CENTOS_SLAVES + SQUEEZE32_SLAVES + WINDOWS_SLAVES
LINUX_SLAVES = SQUEEZE32_SLAVES # less specific but for now simply an alias
c['slaves'] = [BuildSlave(name, "password", notify_on_missing=NOTIFY_ON_MISSING, max_builds=1) for name in SLAVES]
c['slavePortnum'] = 9989
c['change_source'] = PBChangeSource(user='git',passwd='password')


# we used to have a change source but now we poll git and create tags
# if we see differences, and that doesn't mappen to the buildbot concept of 
# change sources


class NecessaryCommand(ShellCommand):
    def __init__(self, **kw):
        if 'description' not in kw and 'name' in kw:
            kw['description'] = kw['name']
        ShellCommand.__init__(self, **kw)
    haltOnFailure = True

class XcLogLineObserver(LogLineObserver):
    task_re = compile(r'''^\[(?P<time>\d+:\d+:\d+\.\d+)\]: NOTE: Running task (?P<bb_current_task>\d+) of (?P<bb_task_number>\d+) \(ID: \d+, (?P<bbname>.*), (?P<taskstep>.*)\)$''')
    start_re = compile(r"^STARTING STEP (.*)$")
    oe_re = compile(r"^STARTING OE BUILD (.*)$")
    problem_re = compile(r'^\[\d+:\d+:\d+\.\d+\]: (WARNING:|ERROR:)')
    def __init__(self, buildstep):
        LogLineObserver.__init__(self)
        self.buildstep = buildstep
        self.warnings = 0
        self.errors = 0
    def outLineReceived(self, line):
        smatch = self.start_re.match(line)
        dirty = False
        status = self.buildstep.step_status
        if smatch:
            status.setStatistic('do_build_step', smatch.group(1))
            if 'oe_build' in status.statistics:
                del status.statistics['oe_build']
            dirty = True
        tmatch = self.task_re.match(line)
        if tmatch:
            values = tmatch.groupdict()
            bb_current_task = int(values['bb_current_task'])
            bb_task_number = int(values['bb_task_number'])
            status.setStatistic('bb_current_task', bb_current_task)
            status.setStatistic('bb_task_number', bb_task_number)
            dirty = True
        omatch = self.oe_re.match(line)
        if omatch:
            status.setStatistic('oe_build', omatch.group(1))
            for k in ['bb_current_task', 'bb_task_number']:
                if k in status.statistics:
                    del status.statistics[k]
            dirty = True
        pmatch = self.problem_re.match(line)
        if pmatch:
            if pmatch.group(1) == 'WARNING:':
                self.warnings += 1
                status.setStatistic('warnings', self.warnings)
            elif pmatch.group(1) == 'ERROR:':
                self.errors += 1
                status.setStatistic('errors', self.errors)
                if status.statistics.get('firstError', '') == '':
                    status.setStatistic('firstError', line)
            else:
                log.msg('unexpected match %r' % pmatch.group(1))
            dirty = True
        if dirty:
            status.setText(self.buildstep.getText(None, None))

class XcBBShellCommand(NecessaryCommand):
    def __init__(self, *args, **kwargs):
        ShellCommand.__init__(self, *args, **kwargs)
        self.addLogObserver('stdio', XcLogLineObserver(self))
    def getText(self, cmd, results):
        words = [self.name]
        step = self.step_status.statistics.get('do_build_step')
        oestep = self.step_status.statistics.get('oe_build')
        if results == FAILURE:
            words.append('Failed on ')
        elif results == SUCCESS:
            words.append('Succeeded after ')
        else:
            words.append('Running ')
        words.append((' step '+step+'.') if step else 'preliminaries.')

        ctask = self.step_status.statistics.get('bb_current_task')
        ttask = self.step_status.statistics.get('bb_task_number')
        if ctask and ttask and oestep:
            words.append('Within step %s doing bitbake %s task %d of %d.' % (step.split()[0], oestep, ctask, ttask))
        for k in ['warnings', 'errors']:
            n = self.step_status.statistics.get(k, 0)
            words.append('%d %s so far across all steps.' % (n, k))
        firsterr = self.step_status.statistics.get('firstError', '')
        if firsterr:
            words.append('First error: '+ firsterr.strip())
        return words


def git_checkout(factory, repo, workdir=None, repourl=None, branch=None, section='xenclient'):
    buildp = 'build/'
    if workdir is None:
        workdir = buildp+repo
    if repourl is None:
        repourl = WithProperties('%s/'+section+'/'+repo+'.git', 'git_base')
    workdes = workdir.replace('/', '_')
    assert workdir.startswith(buildp)
    factory.addStep(NecessaryCommand(name='delete_'+workdes, command=['rm', '-rf', workdir[len(buildp):]]))
    factory.addStep(MakeDirectory(name='mkdir_'+workdes, dir=workdir))
    factory.addStep(NecessaryCommand(name='clone_'+repo, command=['git', 'clone', repourl, '.'], workdir=workdir))
    if branch:
        if branch !='master':
            des = 'branch'
            version = branch
        else:
            version = None
    else:
        des = '_tag'
        version = Property('revision')
    if version:
        factory.addStep(NecessaryCommand(name='checkout_'+repo+'_to_'+des, command=['git', 'checkout', version], workdir=workdir))

c['builders'] = []

taglock = MasterLock("handle_tag") # held while polling and creating tags
fpoll = factory.BuildFactory()
class BGit(Git):
    """work around buildbot bug that Git branch is not renderable, i.e. 
    we wouldn't otherwise be able to use properties for the branch"""
    renderables = ['repourl', 'branch']
# XXX this step non-fatally fails to describe with http://trac.buildbot.net/ticket/2539

fpoll.addStep(BGit(
        name='update_build-machines',
        description=['update build-machines to HEAD'],
        repourl=Interpolate('%(prop:git_base)s/infrastructure/build-machines.git'),
        branch=Property('inspection_branch'), alwaysUseLatest=True, progress=True))

def extract_branches(rc, out, err):
    return {'untagged_branches': ' '.join(out.split())}
class SetUntaggedBranches(SetPropertyFromCommand):
    def getText(self, cmd, results):
        untagged = self.property_changes['untagged_branches'].split()
        if untagged == []:
            des = 'no branches'
        else:
            des = ' and '.join(untagged)+ ' branch(es)'
        return 'found '+des+' in build-machines.git / auto_build_branches.txt have untagged commits'
# TODO: reenable SSH? it is a lot slower though
fpoll.addStep(SetUntaggedBranches(name='poll_for_untagged_branches',
                                  description='looking for untagged branches mentioned in build-machines.git / auto_build_branches.txt',
                          command =['./do_tag.py', '-l', 'auto_build_branches.txt',
                                    #'-u', Property('git_ssh_user'),
                                    #'-s', Property('git_ssh_server')
                                    '-v', Interpolate('%(prop:site)s-%(prop:build_type)s-')],
                                  locks = [taglock.access('exclusive')],
                          extract_fn = extract_branches))
class TagTriggers(LoggingBuildStep):
    flunkOnFailure = True
    name='tag triggers'
    def start(self):
        all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
        schedulers = [sch for sch in all_schedulers if sch.name == 'trigger_'+XT_TAG]
        if len(schedulers) != 1:
            self.step_status.setText(['found %d rather than %s scheduler' % (len(schedulers), XT_TAG)])
            self.finished(FAILURE)
        sch = schedulers[0]
        untagged = self.build.getProperties()['untagged_branches']
        log.msg('sch.codebases=%r' % (sch.codebases))
        for branch in untagged.split():
            log.msg('untagged branch', branch)
            sch.trigger({'':{'branch':branch}}).addErrback(log.err, '(ignored) while invoking XT Tag scheduler:')
        self.finished(SUCCESS)    
fpoll.addStep(TagTriggers())
c['builders'].append(BuilderConfig(name=XT_POLL,
                                   mergeRequests=lambda b, x, y: True,
                                   slavenames=LINUX_SLAVES,
                                   factory=fpoll))

ftag = factory.BuildFactory()
ftag.addStep(NecessaryCommand(name="echobuildtag", description=[WithProperties("Making tag for %(branch)s on %(slavename)s")], command=WithProperties("echo %s", "branch")))
# XXX this step non-fatally fails to describe with http://trac.buildbot.net/ticket/2539
ftag.addStep(BGit(
        name='update_build-machines',
        description=['update build-machines'],
        repourl=Interpolate('%(prop:git_base)s/infrastructure/build-machines.git'),
        branch=Property('inspection_branch'), 
        alwaysUseLatest=True, progress=True))
# create a tag. This involves defaulting to a fallback branch for each repository.
# Previously we always defaulted to master and always picked up new repositories.
# However, topic branches may be based off a release branch, and we have some extraneous
# repos, and just because a new repo arrives after a branch is made doesn't mean we 
# should start including it.
# we used to set up the build config and have do_build.sh do this
# however I would like to avoid the whole do_build.sh apparatus at this stage
# and it comes down to one shortish shell pipeline so we do it directly
# also the ID selection code should not be in the branches since an errant branch
# could cause problems
# TODO: remove -I / NEXT_ID / give_next_id code from do_build.sh on master
# TODO: reenable SSH when we need it; it is a lot slower though
ftag.addStep(SetPropertyFromCommand(
        name='create_tag', haltOnFailure=True, description='create tag', 
        locks=[taglock.access('exclusive')],
        command=['sudo', './do_tag.py', '-t', '-v', '-f', '-b', Property('branch'),
#                 '-u', Property('git_ssh_user'),
#                 '-s', Property('git_ssh_server'),
                 Interpolate('%(prop:site)s-%(prop:build_type)s-')],
        extract_fn=lambda rc, out, err: {'tag':out.strip()}))
ftag.addStep(NecessaryCommand(name='mkdir_build_tag_dir', command=['sudo', 'mkdir', '-p', WithProperties('%(central_build_directory)s/%(branch)s/%(tag)s')]))
ftag.addStep(NecessaryCommand(name='chown_build_branch_dir', command=['sudo', 'chown', Property('build_user'), WithProperties('%(central_build_directory)s/%(branch)s')]))
ftag.addStep(NecessaryCommand(name='chown_build_tag_dir', command=['sudo', 'chown', Property('build_user'), WithProperties('%(central_build_directory)s/%(branch)s/%(tag)s')]))
ftag.addStep(NecessaryCommand(name='mkdir_buildoutput_branch', command=['sudo', 'mkdir', '-p', WithProperties('%(windows_build_staging)s-%(build_type)s/%(branch)s')]))
ftag.addStep(NecessaryCommand(name='chmod_buildoutput_branch', command=['sudo', 'chmod', 'a+rwxs', WithProperties('%(windows_build_staging)s-%(build_type)s/%(branch)s')]))
ftag.addStep(Trigger(schedulerNames=['trigger_complete_XT_build'], waitForFinish=False, haltOnFailure=True,
                     sourceStamp={'revision':Property('tag'), 'branch':Property('branch'), 'changeids':[611,612]}))
ftag.addStep(Trigger(schedulerNames=['trigger_XT_debianrepo'], waitForFinish=False, haltOnFailure=True,
                     sourceStamp={'revision':Property('tag'), 'branch':Property('branch'), 'changeids':[611,612]}))
ftag.addStep(NecessaryCommand(name='generate git diff', 
                              command=['./diff_tag.py', Property('tag'),
                                       '-v',
                                       '-j', 'diff.json',
                                       '-p', Interpolate('%(prop:site)s-%(prop:build_type)s-'),
                                       '-s', Interpolate('-%(prop:branch)s')]))
ftag.addStep(NecessaryCommand(name='copy diff',
                              command=['sudo', 'cp', 'diff.json', 
                                       Interpolate('%(prop:central_build_directory)s/%(prop:branch)s/%(prop:tag)s/diff.json')]))

c['builders'].append(BuilderConfig(name=XT_TAG,
                                   slavenames=LINUX_SLAVES, 
                                   mergeRequests=lambda b, x, y: x.source.branch == y.source.branch,
                                   factory=ftag))

def make_tagged_build_factory():
    f = factory.BuildFactory()
    f.addStep(NecessaryCommand(name="echobuild", description=[WithProperties("Running build tag %(revision)s for %(branch)s branch on %(slavename)s")], command=WithProperties("echo %(revision)s")))
    f.addStep(SetPropertyFromCommand(name='xcbuildid', command=['echo', Property('revision')], 
                                     extract_fn=lambda rc,out,err : {'xcbuildid':out.split('-')[2]}))
    f.addStep(SetPropertyFromCommand(name='branch', command=['echo', Property('revision')], 
                                     extract_fn=lambda rc,out,err : {'branch':'-'.join(out.strip().split('-')[3:])}))
    return f

def setup_oe(f):
    git_checkout(f, 'build-machines', branch=Property('inspection_branch'), section='infrastructure')
    f.addStep(NecessaryCommand(name='prepare_scratch', command=['sudo', 'build-machines/prepare_scratch.py', '-u', Property('build_user_name')], timeout=24*60*60))
    git_checkout(f, 'build-scripts', workdir='build/scratch/build-scripts')
    git_checkout(f, 'build-config', workdir='build/scratch/build-config')
    f.addStep(NecessaryCommand(name="copy_config", command=["cp", Interpolate("./scratch/build-config/configs/cam_%(prop:build_type)s"), "scratch/.config"]))
    f.addStep(NecessaryCommand(name='rsync_OE_download_cache_down',
                                    command=['sudo', 'rsync', '-ltvzru', WithProperties('%(master_download_cache)s/'), 'oe-download']))
    f.addStep(NecessaryCommand(name='chown_oe-download', command=['sudo', 'chown', Property('build_user_name'), '-R', 'oe-download']))
    f.addStep(NecessaryCommand(name='mkdir_oe-sstate', command=['mkdir', '-p', 'oe-sstate']))
    f.addStep(NecessaryCommand(name='mkdir_scratch_misc_oe', 
                                    command=['mkdir', '-p', 'scratch/misc/oe']))
    f.addStep(NecessaryCommand(name='symlink_across_oecache', 
                                    command=['ln', '-s', '../../../oe-download', 'scratch/misc/oe/oe-download']))
    f.addStep(NecessaryCommand(name='symlink_across_oestate', 
                                    command=['ln', '-s', '../../../oe-sstate', 'scratch/misc/oe/oe-sstate']))

def sync_back(f):
    f.addStep(NecessaryCommand(name='rsync_OE_download_cache_back', 
                               command=['sudo', 'rsync', '-ltvzru', 'oe-download/', Property('master_download_cache')]))
    
def run_do_build(f, *args):
    f.addStep(XcBBShellCommand(name='run_do_build.sh', command=["./build-scripts/do_build.sh", "-b", Property("branch"), "-S",
                                                                "-d", Property('rsync_build_destination'),
                                                                "-w", WithProperties("rsync://rsync.cam.xci-test.com/xc_buildoutput/xenclient-windows-%(build_type)s/%(branch)s/%(xcbuildid)s/output"),
                                                                "-i", Property("xcbuildid")]+list(args), timeout=24*60*60, workdir='build/scratch'))

finstaller = make_tagged_build_factory()
setup_oe(finstaller)
run_do_build(finstaller)
sync_back(finstaller)
c['builders'].append(BuilderConfig(name=XT_INSTALLER, slavenames=SQUEEZE32_SLAVES, factory=finstaller))
c['schedulers'] += force(XT_INSTALLER)
c['schedulers'].append(basic.Dependent(name='trigger_'+XT_INSTALLER, upstream=complete_trigger, builderNames=[XT_INSTALLER]))

fdebianrepo = make_tagged_build_factory()
setup_oe(fdebianrepo)
run_do_build(fdebianrepo, "-s", "setupoe,debian_repo_xctools,debian_repo_xctools_copy")
sync_back(fdebianrepo)

c['builders'].append(BuilderConfig(name=XT_DEBIANREPO, slavenames=SQUEEZE32_SLAVES, factory=fdebianrepo))
c['schedulers'] += force_and_trigger(XT_DEBIANREPO)

windowsf = make_tagged_build_factory()

# TODO: confirm error handling works if there is mandatory file lock in build-scripts
git_checkout(windowsf, 'build-config')
windowsf.addStep(NecessaryCommand(name="clean", description="clean up working dir", command="IF EXIST build-scripts (rmdir build-scripts /S /Q)"))
# We did try local clone of git repositories since the clones were running slowly, but garbage collection of the git repos helped
# for repo in ['build-scripts', 'gfx-drivers', 'msi-installer', 'win-changelog', 'win-tools', 'xc-windows']:
#     windowsf.addStep(NecessaryCommand(name="clone_or_fetch_to_"+repo+"_mirror", 
#          command="IF NOT EXIST reference\\"+repo+".reference ("+
# 	  "git clone --mirror git://git.xci-test.com/xenclient/"+repo+".git reference/"+repo+".reference) "+
# 	   "ELSE (git --git-dir reference/"+repo+".reference fetch --all)"))
# TODO: get this working
#windowsf.addStep(Git(workdir="build/build-scripts",repourl="../reference/build-scripts.reference", mode="full"))
#  ... but for now do a standard clone
git_checkout(windowsf, 'build-scripts')
windowsf.addStep(NecessaryCommand(name="CopyConfig", workdir="build", command=['copy', 'build-config\\windows\\winbuild-cam-config.xml', 'build-scripts\\windows\\config']))
windowsf.addStep(NecessaryCommand(name="prepare", description="prepare", workdir="build/build-scripts/windows",command=["powershell", "-Command", ".\\winbuild-prepare.ps1","config=winbuild-cam-config.xml",WithProperties("build=%s","xcbuildid"),WithProperties("tag=%(revision)s")],timeout=None, haltOnFailure=True))
# this had haltOnFailure=False but I'd rather halt on problems
windowsf.addStep(Compile(workdir="build/build-scripts/windows",command=["powershell", "-Command", ".\\winbuild-all.ps1","config=winbuild-cam-config.xml",WithProperties("build=%(xcbuildid)s"), WithProperties("tag=%(revision)s")],timeout=None, haltOnFailure=True))
# TODO: use a secure single use rsync account, preferably into the main build tag directory
windowsf.addStep(NecessaryCommand(name="CopyOutput", description="Copy Output", workdir="build/build-scripts/windows", command=["rsync","-vzr","output",WithProperties("rsync://rsync.cam.xci-test.com/xc_buildoutput/xenclient-windows-%(build_type)s/%(branch)s/%(xcbuildid)s/") ]))
windowsf.addStep(NecessaryCommand(name="CopyLogs", description="Copy Logs", workdir="build/build-scripts/windows", command=["rsync","-vzr","logs",WithProperties("rsync://rsync.cam.xci-test.com/xc_buildoutput/xenclient-windows-%(build_type)s/%(branch)s/%(xcbuildid)s/") ]))

c['builders'].append(BuilderConfig(name=XT_WINDOWS, slavenames=WINDOWS_SLAVES,
                                   factory=windowsf))

syncxtf = make_tagged_build_factory()
syncxtf.addStep(SetPropertyFromCommand(property='archive_path', command=WithProperties('echo %(branch)s/%(revision)s/sync')))
for repo in ['build-scripts', 'ctxlicensing', 'sync-database', 'sync-cli', 'sync-server', 'sync-ui-helper', 'xclicensing']:
    git_checkout(syncxtf, repo, workdir='build/src/' + repo)
syncxtf.addStep(NecessaryCommand(
    name = "build Sync XT",
    description = "Run build-scripts.git/do_sync_xt.sh",
    command = ["src/build-scripts/do_sync_xt.sh", "-i", Property('revision'), "-v", "."], 
    haltOnFailure = True))
# the directory structure from rpmbuild, i.e. putting everything directly in an out directory
# is not great with rsync which needs the full directory structure, so we shuffle things around
syncxtf.addStep(NecessaryCommand(name='mkdir_final_output_directory', command=['mkdir', '-p', WithProperties('out/%s', 'archive_path')]))
syncxtf.addStep(NecessaryCommand(name='move_output_into_place', command=['sh', '-c', WithProperties('mv out/*.rpm out/%s', 'archive_path')]))
# the -K hack prevents rsync from deleting the symlink /home/xc_builds/master
# and replacing it with a directory
syncxtf.addStep(NecessaryCommand(name='rsync_to_filer', command=['rsync', '--chmod=Fgo+r,Dgo+rx', '-rvaK', WithProperties('out/%s', 'branch'), WithProperties('%s/', 'rsync_build_destination')]))

c['builders'].append(BuilderConfig(name=XT_SYNCXT, slavenames=CENTOS_SLAVES, factory=syncxtf))

docsf = make_tagged_build_factory()
docsf.addStep(NecessaryCommand(name="build-machine-scripts",description="Checkout build-machine-scripts",command="rm -rf build-machine-scripts && git clone git://git.xci-test.com/xenclient-build/build-machines.git build-machine-scripts"))
docsf.addStep(NecessaryCommand(name="CleanDocsOutput", description="Clean Docs output dir", command="sudo rm -rf ./docs_out && mkdir ./docs_out"))
docsf.addStep(NecessaryCommand(name="echobranch", description=[WithProperties("Running build on branch %s", "branch")], command=WithProperties("echo Running build on branch %s", "branch")))
docsf.addStep(Compile(command=["./build-machine-scripts/docs_build.sh", WithProperties("%s", "buildnumber"), WithProperties("%s", "branch")],timeout=None, haltOnFailure=True))
docsf.addStep(NecessaryCommand(name="CopyOutput", description="Copy Output Docs", command=["rsync", "-rvz", "./docs_out/", WithProperties("root@rsync.cam.xci-test.com:/home/xc_dist/docs/%s/", "branch")]))

c['builders'].append(BuilderConfig(name=XT_DOCS,
                                   slavenames=SQUEEZE32_SLAVES,
                                   factory=docsf))

def myshortrev(templates):
    def filter(rev, repo):
        # TODO: link to compare_builds output?
        return unicode(rev)
    return filter

def gen_sorted_builder_names(self, status, category):
    orig = sorted(status.getBuilderNames())

    # Create list of builders based on value of category
    if category == None:
        sortedBuilderNames = [x for x in orig if x not in ['XT_Tag', 'XT_Poll', 'XT_installer']]
    elif category.lower() == 'windows':
        sortedBuilderNames = [x for x in orig if x not in ['XT_Tag', 'XT_Poll', 'XT_installer', 'XT_debianrepo', 'XT_docs']]
    elif category.lower() == 'docs':
        sortedBuilderNames = [x for x in orig if x not in ['XT_Tag', 'XT_Poll', 'XT_installer', 'XT_debianrepo', 'XT_Windows']]
    elif category.lower() == 'linux':
        sortedBuilderNames = [x for x in orig if x not in ['XT_Tag', 'XT_Poll', 'XT_installer', 'XT_Windows', 'XT_docs']]

    if 'XT_installer' in orig:
        sortedBuilderNames = sortedBuilderNames + ['XT_installer']
    if 'XT_Tag' in orig:
        sortedBuilderNames = ['XT_Tag']+sortedBuilderNames

    return sortedBuilderNames

def gen_stampmap_entry(stampmap, rev, ss, start, end):
    if rev not in stampmap:
        return (ss, start, end)
    else:
        if stampmap[rev][1] > start and stampmap[rev][2] < end:
            return (ss, start, end)
        elif stampmap[rev][1] > start:
            return (ss, start, stampmap[rev][2])
        elif stampmap[rev][2] < end:
            return (ss, stampmap[rev][1], end)
        else:
            return stampmap[rev]

def update_running_builds(stampmap, buildmap, sortedBuilderNames):
    #for all tags
    for ss,_,_ in stampmap.itervalues():
        rev= ss[0].revision
        failed = False
        complete = True
        #for all builders
        for bn in sortedBuilderNames:
            build = buildmap.get( (bn, rev))
            if build is None:
                complete = False
            elif build.isFinished() and build.results not in [SUCCESS, WARNINGS]:
                failed = True
        #If tag's build is not complete and it isn't due to failure
        if not failed and not complete:
            stampmap[rev] = (stampmap[rev][0], stampmap[rev][1], time())

    return stampmap

def get_url_strings(stamps, sortedBuilderNames, build_numbers):
    url_strings = []
    for j in range(0, len(stamps)):
        url_string = ''
        for idx, bn in enumerate(sortedBuilderNames):
            if build_numbers[idx][j] is not None:
                url_string = url_string + '&' + bn + '=' + str(build_numbers[idx][j])
        url_strings.append(url_string)

    return url_strings

class XTGrid(HtmlResource, GridStatusMixin):
    status = None
    changemaster = None
    default_rev_order = "desc"

    @inlineCallbacks
    def content(self, request, cxt):
        """This method builds the transposed grid display.
        That is, build hosts across the top, build stamps down the left side
        """

        # get and process url parameters
        # The number of builds to show on screen
        numBuilds = int(request.args.get("length", [30])[0])

        # Filter out things like Windows, Linux, Android...
        category = request.args.get("category", [None])[0]
        cxt['category'] = category

        # Filter output to one branch
        branch = request.args.get("branch", [ANYBRANCH])[0]
        cxt['branch']= branch
        cxt['ANYBRANCH'] = ANYBRANCH
        
        # How to order the builds?
        rev_order = request.args.get("rev_order", [self.default_rev_order])[0]
        if rev_order not in ["asc", "desc"]:
            rev_order = self.default_rev_order

        # Set how often the page needs to reset in seconds
        cxt['refresh'] = self.get_reload_time(request)

        # Set the length of the page
        cxt['length'] = numBuilds
        #Collect current system status
        status = self.getStatus(request)
        #Create an ordered list of our builders
        sortedBuilderNames = gen_sorted_builder_names(self, status, category)

        stampmap = { } # { ss-tuple : source stamp, earliest time, latest time }
        buildmap = {} # { (bn, revision) : build }
        cutoff = None
        for bn in sortedBuilderNames:
            builder = status.getBuilder(bn)
            last = builder.nextBuildNumber-1

            # For potentially the whole list of builds from n to 0 (Use break logic to exit)
            for num in range(last, -1, -1):
                try:
                    build = builder.getBuildByNumber(num)
                except IndexError:
                    continue

                # Must check if this build is on a branch we care about
                if branch != ANYBRANCH:
                    if build.properties['branch'].lower() != branch:
                        continue

                #Have to generate our own source stamp if under XT_Tag
                if bn != 'XT_Tag':
                    ss = build.getSourceStamps(absolute=True)
                else:
                    try:
                        tag = build.properties['tag']
                    except KeyError:
                        continue
                    else:
                        ss = [SourceStamp(revision=tag, branch =build.properties['branch'])]

                #Get the tag name and build times
                rev = ss[0].revision
                start, end = build.getTimes()
                if not end:
                    end = time()

                #Create/update stampmap values for this revision
                stampmap[rev] = gen_stampmap_entry(stampmap, rev, ss, start, end)

                buildmap.setdefault((bn, rev), build)
                if cutoff is None and len(stampmap) > numBuilds:
                    cutoff = start
                if start < cutoff:
                    break

        # tags with builds unstarted are still "running", so update all end times
        stampmap = update_running_builds(stampmap, buildmap, sortedBuilderNames)

        # Set page values
        cxt['stampdates'] = dict( [(rev, strftime('%a %H:%M', gmtime(stampmap[rev][1]))) for rev in stampmap])
        cxt['stamphours'] = dict( [(rev, unicode('%.1fh') % ((stampmap[rev][2] - stampmap[rev][1])/3600.0)) for rev in stampmap])

        stampstarts = sorted(stampmap.itervalues(), key = lambda stamp: stamp[1])
        cxt['stamps']= stamps = [stamp[0][0] for stamp in stampstarts][-numBuilds:]
        cxt['authors'] = []
        cxt['repolist'] = []
        #For every tag
        for stamp in stamps:
            repos = []
            authors = []

            if stamp.branch is None or stamp.revision is None:
                log.msg('bad stamp branch=%r revision=%r skipped' %(stamp.branch, stamp.revision))
            else:
                #Try to get the JSON file
                try:
                    json_data = open('/home/xc_builds/' + stamp.branch +'/' + stamp.revision + '/diff.json')
                    data = load(json_data)
                except IOError:
                    log.msg('inaccessible diff.json on branch=%r revision=%r skipped' %(stamp.branch, stamp.revision))
                    repos = None
                    authors = None
                    pass
                else:

                    #For every dictionary in the array read from JSON
                    for record in data:
                        try:
                            #Get author name
                            potential_author = record['author']
                            #Get repo name
                            potential_repo = record['repo']

                            #Only add the repo to the list if the name is new
                            try:
                                idx = repos.index(potential_repo)
                                idx = repos.index(potential_repo)

                                #Only add the author to the sublist if name is new
                                try:
                                    authors[idx].index(potential_author)
                                except ValueError:
                                    authors[idx].append(potential_author)
                            except ValueError:
                                repos.append(potential_repo)
                                authors.append([])
                                authors[-1].append(potential_author)

                        except KeyError:
                            continue
                    json_data.close()

            cxt['authors'].append(authors)
            cxt['repolist'].append(repos)

        #Set builder names for table header
        cxt['sorted_builder_names'] = sortedBuilderNames
        #Set the range of indices available in stamps (How many tags were found)
        cxt['range'] = range(len(stamps))
        if rev_order == "desc":
            cxt['range'].reverse()

        #Generate versions of builders & builds suitable for web presentation
        cxt['builder_builds'] = builder_builds = []
        cxt['builders'] = []
        build_numbers = []
        for bn in sortedBuilderNames:
            builder = status.getBuilder(bn)
            b = yield self.builder_cxt(request, builder)
            cxt['builders'].append(b)
            builds = [ buildmap.get((bn, ss.revision)) for ss in stamps]
            append_builds = [self.build_cxt(request, b) for b in builds]
            for idx, b in enumerate(builds):
                if b is not None:
                    start, end = b.getTimes()
                    eta = b.getETA()
                    cur_time = time()
                    if eta is not None:
                        append_builds[idx].update({ 'Prog':int( floor(100 * ( float(cur_time-start)/float((cur_time-start)+eta) )) ) })
                        eta = "%02d:%02d:%02d" % (eta/3600, (eta%3600)/60, eta%60)
                        append_builds[idx].update({'ETA':eta})
                    else:
                        append_builds[idx].update({'Prog':None})
                        append_builds[idx].update({'ETA':None})
            builder_builds.append(append_builds)

            #Generate a list of build numbers for each tag
            numberList = []
            for b in builds:
                if b is None:
                    numberList.append(None)
                else:
                    numberList.append(b.getNumber())
            build_numbers.append(numberList)

        #Generate the URLs to reference each tag's builds
        cxt['build_numbers'] = get_url_strings(stamps, sortedBuilderNames, build_numbers)

        #Finally, generate the page
        self.clearRecentBuildsCache()
        template = request.site.buildbot_service.templates.get_template('xtgrid.html')
        returnValue(template.render(**cxt))

class XTTagSummary(HtmlResource, GridStatusMixin):
    status = None
    changemaster = None
    default_rev_order = "desc"

    @inlineCallbacks
    def content(self, request, cxt):
        """This method builds the build details page which shows the commits
        for each build in detail."""
        # get and process url parameters
        # Filter out things like Windows, Linux, Android...
        category = request.args.get("category", [None])[0]
        cxt['category'] = category

        # Set how often the page needs to reset in seconds
        cxt['refresh'] = self.get_reload_time(request)

        # XTCommit Specifics:
        # Set length to 0 to stop the "show more"
        cxt['length'] = 0
        #How to filter the list of commits
        author_filter = request.args.get("author", [None])[0]
        repo_filter = request.args.get("repo", [None])[0]

        #Collect current system status
        status = self.getStatus(request)
        #Create an ordered list of our builders
        sortedBuilderNames = gen_sorted_builder_names(self, status, category)

        stampmap = { } # { ss-tuple : source stamp, earliest time, latest time }
        buildmap = {} # { (bn, revision) : build }
        cutoff = None
        for bn in sortedBuilderNames:
            builder = status.getBuilder(bn)

            try:
                build_num = request.args.get(bn, [None])[0]
                if build_num is None:
                    continue
                build = builder.getBuildByNumber(int(build_num))
            except IndexError:
                continue

            # TODO: remove duplication with XTGrid
            #Have to generate our own source stamp if under XT_Tag
            if bn != 'XT_Tag':
                ss = build.getSourceStamps(absolute=True)
            else:
                try:
                    tag = build.properties['tag']
                except KeyError:
                    continue
                else:
                    ss = [SourceStamp(revision=tag, branch =build.properties['branch'])]

            #Get the tag name and build times
            rev = ss[0].revision
            start, end = build.getTimes()
            if not end:
                end = time()

            #Create/update stampmap values for this revision
            stampmap[rev] = gen_stampmap_entry(stampmap, rev, ss, start, end)

            buildmap.setdefault((bn, rev), build)

        # tags with builds unstarted are still "running", so update all end times
        stampmap = update_running_builds(stampmap, buildmap, sortedBuilderNames)

        # Set page values
        cxt['stampdates'] = dict( [(rev, strftime('%a %H:%M', gmtime(stampmap[rev][1]))) for rev in stampmap])
        cxt['stamphours'] = dict( [(rev, unicode('%.1fh') % ((stampmap[rev][2] - stampmap[rev][1])/3600.0)) for rev in stampmap])

        stampstarts = sorted(stampmap.itervalues(), key = lambda stamp: stamp[1])
        cxt['stamps']= stamps = [stamp[0][0] for stamp in stampstarts][-1:]
        cxt['authors'] = []
        cxt['repolist'] = []
        cxt['commit_list'] = [] # [[repo, author, hash, subject],...]
        #For every tag
        for stamp in stamps:
            repos = []
            authors = []

            if stamp.branch is None or stamp.revision is None:
                log.msg('bad stamp branch=%r revision=%r skipped' %(stamp.branch, stamp.revision))
            else:
                #Try to get the JSON file
                try:
                    json_data = open('/home/xc_builds/' + stamp.branch +'/' + stamp.revision + '/diff.json')
                    data = load(json_data)
                except IOError:
                    log.msg('inaccessible diff.json on branch=%r revision=%r skipped' %(stamp.branch, stamp.revision))
                    repos = None
                    authors = None
                    pass
                else:
                    #For every dictionary in the array read from JSON
                    for record in data:
                        try:
                            #Get author name
                            potential_author = record['author']
                            #Get repo name
                            potential_repo = record['repo']
                            #Get hash
                            hash = record['subject']
                            #Get hash
                            subject = record['hash']

                            #Only add the repo to the list if the name is new
                            try:
                                idx = repos.index(potential_repo)
                                idx = repos.index(potential_repo)

                                #Only add the author to the sublist if name is new
                                try:
                                    authors[idx].index(potential_author)
                                except ValueError:
                                    authors[idx].append(potential_author)
                            except ValueError:
                                repos.append(potential_repo)
                                authors.append([])
                                authors[-1].append(potential_author)

                            #Create and append entry for web page to display
                            if repo_filter is None and author_filter is None:
                                to_add = [potential_repo, potential_author, hash, subject]
                                cxt['commit_list'].append(to_add)
                            elif repo_filter is None:
                                if author_filter == potential_author:
                                    to_add = [potential_repo, potential_author, hash, subject]
                                    cxt['commit_list'].append(to_add)
                            elif author_filter is None:
                                if repo_filter == potential_repo:
                                    to_add = [potential_repo, potential_author, hash, subject]
                                    cxt['commit_list'].append(to_add)

                        except KeyError:
                            continue
                    json_data.close()

            cxt['authors'].append(authors)
            cxt['repolist'].append(repos)

        #Set builder names for table header
        cxt['sorted_builder_names'] = sortedBuilderNames
        #Set the range of indices available in stamps (How many tags were found)
        cxt['range'] = range(len(stamps))

        #Generate versions of builders & builds suitable for web presentation
        cxt['builder_builds'] = builder_builds = []
        cxt['builders'] = []
        build_numbers = []
        for bn in sortedBuilderNames:
            builder = status.getBuilder(bn)
            b = yield self.builder_cxt(request, builder)
            cxt['builders'].append(b)
            builds = [ buildmap.get((bn, ss.revision)) for ss in stamps]
            append_builds = [self.build_cxt(request, b) for b in builds]
            for idx, b in enumerate(builds):
                if b is not None:
                    start, end = b.getTimes()
                    eta = b.getETA()
                    cur_time = time()
                    if eta is not None:
                        append_builds[idx].update({ 'Prog':int( floor(100 * ( float(cur_time-start)/float((cur_time-start)+eta) )) ) })
                        eta = "%02d:%02d:%02d" % (eta/3600, (eta%3600)/60, eta%60)
                        append_builds[idx].update({'ETA':eta})
                    else:
                        append_builds[idx].update({'Prog':None})
                        append_builds[idx].update({'ETA':None})
            builder_builds.append(append_builds)

            #Generate a list of build numbers for each tag
            numberList = []
            for b in builds:
                if b is None:
                    numberList.append(None)
                else:
                    numberList.append(b.getNumber())
            build_numbers.append(numberList)

        #Generate the URLs to reference each tag's builds
        cxt['build_numbers'] = get_url_strings(stamps, sortedBuilderNames, build_numbers)

        #Finally, generate the page
        self.clearRecentBuildsCache()
        template = request.site.buildbot_service.templates.get_template('xtgrid.html')
        returnValue(template.render(**cxt))

class MyWebStatus(WebStatus):
    def setServiceParent(self, parent):
        WebStatus.setServiceParent(self, parent)
        self.templates.filters['shortrev'] = myshortrev(self.templates)
    def setupUsualPages(self, numbuilds, num_events, num_events_max):
        WebStatus.setupUsualPages(self, numbuilds, num_events, num_events_max)
        self.putChild('xtgrid', XTGrid())
        self.putChild('xttagsummary', XTTagSummary())

c['status'] = []
c['status'].append(MyWebStatus(http_port=8010, 
                               authz=authz.Authz(forceBuild=True, stopBuild=True,
                                                 cancelPendingBuild=True, 
                                                 pingBuilder=True)))
c['status'].append(IRC(host='irc.cam.xci-test.com', 
                       nick=gethostname().split('.')[0].replace('buildbot', 'bb'),
                       password="password",
                       notify_events={'exception': 1,
                                      'started' : 1,
                                      'finished' : 1,
                                      'successToFailure': 1,
                                      'failureToSuccess': 1},
                       allowForce=True,
                       channels=["#build"]))
                       
c['title'] = "XenClient XT"
c['projectName'] = "XenClient XT"
c['projectURL'] = "http://wiki.xci-test.com/"
c['buildbotURL'] = "http://"+getfqdn(gethostname())+"/"
c['logHorizon'] = 300 # only keep logs for this many builds