#====== Copyright 1996-2005, Valve Corporation, All rights reserved. ======= # # Purpose: Syncs, builds and runs tests on the steam code in # the currently active p4 clientspec # #============================================================================= # INSTALLATION INSTRUCTIONS: # # 1. add the vc compiler solution compiler devenv.exe to the path (defaults to being in "C:\\Program Files\\Microsoft Visual Studio .NET 2003\\Common7\\IDE\\devenv.exe") # 2. create a P4 clientspec that contains the steam code, eg. //steam/main/... # 3. make that the default clientspec # 4. sync to it # 5. run this build script from inside that directory import sys, os, string, re, time, smtplib, getopt, P4, SystemHelpers # config options g_bTest = 0 # 1: disables reporting to SRCDEV; only reports to admin g_bRunTests = 0 # 1: enables testing after build is complete g_bSync = 0 # 1: enables syncing to perforce g_bLockMutex = 0 # 1: require mutex to be free before proceeding to build g_bBuild = 0 # 1: build binaries g_bDebug = 0 # 1: enables debug output g_bDev = 0 # 1: script builds immediately upon start, regardless of presence of perforce state g_bShaders = 0 g_bXBOX = 0 g_szEmailAlias = "" # email alias of users interested in build failures g_szAdministrator = "" # address to auto-email if the script fails for some reason, test output g_szMailhost = "" # set to hostname of machine running an SMTP server g_szSenderEmail = "" # email address that the failure mails come from; TODO: Get buildmachine email g_szBuildExe = "" # executable for compilation; devenv: .Net, BuildConsole: IncrediBuild #g_szBuildExe = "devenv" # executable for compilation; devenv: .Net, BuildConsole: IncrediBuild g_szBuildType = "" # build type; /rebuild or /build g_szBuildFlags = "" # should be empty for devenv, /all for buildconsole g_szTestingFile = "" # name of testing file, located in .\Build Machine Tests\ g_szLocalLogFile = "" # name of generated log file; currently not used g_szPublishedErrorsDir = "" # place to copy log file to reference in failure email g_szTestDirectory = "" # directory where the testing files are located g_szBaseDir = os.getcwd() # directory from which the script is launched g_szP4SrcFilesToWatch = "" # path to src files to watch g_szP4ForceFilesToWatch = "" # path to bin files to watch g_szP4SyncFilesToWatch = "" g_szP4Mutex = "" # name of perforce mutex to wait for g_szP4ChangeCounter = "" # perforce counter containg the changelist number we're verifying g_szP4VerifiedCounter = "" # perforce counter the last changelist number we have verified g_nP4MostRecentCheckin = 0 # used to keep track of current checkin g_nP4LastVerifiedCheckin = 0 # used to keep track of last verified checkin g_nSleepTimeBetweenChecks = 10 # number of seconds to wait before checking for a free mutex g_szBuildName = "" # mails off a success checkin def SendSuccessEmail( _szEmail, _szChangeNumber ): cMailport = smtplib.SMTP(g_szMailhost) szTestString = "" if g_bTest: szTestString = "TEST" print "sending success email to: " + _szEmail szMessage = 'From: ' + g_szBuildName + ' Builder ' + ' <' + g_szSenderEmail + '>\n' +\ 'To: ' + szTestString + _szEmail + '\n' +\ 'Subject: [ ' + g_szBuildName + ' build successful ]' +\ '\n' +\ '\n' +\ 'Your checkin:\n\n' +\ _szChangeNumber +\ '\n\n' +\ 'has been successfully built and verified against all tests.\n' if ( g_bTest | g_bDev ): cMailport.sendmail( g_szSenderEmail, g_szAdministrator, szMessage ) else: cMailport.sendmail( g_szSenderEmail, _szEmail, szMessage ) cMailport.quit() # print of debugging messages def dPrint( _szText ): if g_bDebug: print _szText # mails off the current error string def SendFailureEmail( _szErr ): cMailport = smtplib.SMTP( g_szMailhost ) szTestString = "" if g_bTest: szTestString = "TEST" print "sending failure email to: " + g_szEmailAlias print "failure reason: " + _szErr szMessage = 'From: ' + g_szBuildName + ' Builder' + ' <' + g_szSenderEmail + '>\n' +\ 'To: ' + szTestString + g_szEmailAlias + '\n' +\ 'Subject: [ ' + g_szBuildName + ' branch broken ]' +\ '\n' +\ '\n' +\ _szErr +\ '\n' +\ P4.GetUnverifiedCheckins( g_szP4SrcFilesToWatch, g_szP4VerifiedCounter, g_szP4ChangeCounter ) if g_bTest | g_bDev: cMailport.sendmail( g_szSenderEmail, g_szAdministrator, szMessage ) else: cMailport.sendmail( g_szSenderEmail, szTestString + g_szEmailAlias, szMessage ) cMailport.quit() # use to email the admin about any unexpected errors that occur def ComplainToAdmin( _szErr ): cMailport = smtplib.SMTP(g_szMailhost) print "sending script failure email to: " + g_szAdministrator print "failure reason: |" + _szErr + "|" szMessage = 'From: ' + g_szBuildName + " Builder" + ' <' + g_szSenderEmail + '>\n' +\ 'To: ' + g_szAdministrator + '\n' +\ 'Subject: [ ' + g_szBuildName + ' builder build script error ]' +\ '\n' +\ '\n' +\ 'error reason:\n' +\ _szErr +\ '\n' cMailport.sendmail(g_szSenderEmail, g_szAdministrator, szMessage) cMailport.quit() # parses out the reason for failure from a test def GetTestFailureReason( _nBuild ): # start our error message szMsg = "Test log file:\n" # make a directory to copy the error files into, based on the build config and the p4 changelist number nBuildNum = P4.GetCounter(g_szP4ChangeCounter) szErrDir = g_szPublishedErrorsDir + nBuildNum + "\\" + _nBuild os.popen("mkdir " + szErrDir, "r").read() # copy all the files to the errors dir os.popen("copy " + _nBuild + "\\*.*" + " " + szErrDir + " /Y", "r").read() # add the error log to the failure message szMsg += " " + szErrDir + "\\" + g_szLocalLogFile + "\n" # add the minidumps to the failure message rgMinidumps = string.split( os.popen("dir " + _nBuild + "\\*.mdmp /b", "r").read(), "\n" ) if len(rgMinidumps) > 1: szMsg += "\nCrash dump:\n" for szMinidump in rgMinidumps: if len(szMinidump) > 16: szMsg += " " + szErrDir + "\\" + szMinidump + "\n" szMsg += "To view the minidump, click the link above then click \"open\" on the next dialog.\nHit \"F5\" in the now open debugger. When it asks for the source code, change the start of the suggested path from \"c:\\\" to \"\\\\steam3builder\\\".\n" # delete the minidumps, so they don't confuse us next time os.popen("del " + _nBuild + "\\*.mdmp", "r").read() # parse out the error from the logfile szLog = os.popen( "type " + _nBuild + "\\" + szLocalLogFile, "r").read() szLogLines = [] szLogLines = string.split( szLog, "\n" ) bFoundErr = 0 for szLine in szLogLines: aszToken = string.split(szLine, ' ') if len( aszToken ) > 3: if aszToken[0] == "***" and aszToken[1] == "TEST" and aszToken[2] == "FAILED": # probably an error line, add to the list szMsg += szLine szMsg += "\n" bFoundErr = 1 # if we haven't found any error lines, use the last line of the file if bFoundErr == 0 and len(szLogLines) > 0: szMsg += szLogLines[len(szLogLines) - 1] return szMsg # performs a single test run of the specified exe with the specified test parameters def RunTest( _szCommand ): aszParms = string.split( _szCommand, " ", 1) print "running " + _szCommand print os.popen( _szCommand, "r" ).read() # return success return aszParms[0] def RunCompare( _szCommand ): szResult = os.popen( _szCommand, "r" ).read() return szResult def SearchFileForErrors( _szFile, _szError ): print ("Searching " + _szFile + " for occurences of " + _szError + ".\n") bIncludeNext = 0 szErrorResults = "" aszFileLines = string.split( os.popen( "type " + _szFile, "r").read(), "\n" ) for szLine in aszFileLines: if ( bIncludeNext == 1 ): szErrorResults += szLine + "\n" bIncludeNext = 0 aszParms = string.split(szLine, " ", 1) if (aszParms[0] == _szError): #there is a UnitTest error, warning, or assert szErrorResults += "\n" + szLine + "\n" bIncludeNext = 1; return szErrorResults # runs a set of tests as defined by the test script file def RunTestScript( ): print ("Enter Testing") # make sure we're in the right dir SystemHelpers.ChangeDir("..") # load the script szReturn = os.popen( "RunTestScripts.py" ).read() return szReturn # runs a single build, and runs the tests on the build if specified def RunBuild( _szBuild ): # launch devstudio to build the solution szCmd = _szBuild aszToken = string.split(szCmd, " ", 3) if aszToken[0] == "devenv": #we are building this. Find out what it is. if aszToken[2] == "/project": #building a project. Grab it. aszToken = string.split(szCmd, " ", 5) szCmd = g_szBuildExe + " " + aszToken[1] + " /project " + aszToken[3] + " " + g_szBuildType + " " + aszToken[5] else: #not build a specific project. Probably everything under lostcoast. szCmd = g_szBuildExe + " " + aszToken[1] + " " + g_szBuildType + " " + aszToken[3] + " " + g_szBuildFlags else: #didn't see the devenv line; not building this but it isn't an error either. return "" print "building: " + szCmd szOutput = os.popen(szCmd, "r").read() # parse the output for any errors aszOutputLines = string.split(szOutput, "\n") bSuccess = 1 szBuildErr = "\n\n\nError building configuration " + _szBuild + ":\n" for szLine in aszOutputLines: if g_bDebug: print szLine; aszTokens = string.split(string.lstrip(szLine), ' ', 4) if len(aszTokens) > 1: if aszTokens[0] == "--------------------Configuration:": bPrintedProject = 0 szProject = aszTokens[1] if aszTokens[1] == ":" and aszTokens[2] == "error" and aszTokens[3] != "PRJ0019": # probably an error line, add to the list count = string.count( szBuildErr, aszTokens[4]) if ( count == 0 ): if ( bPrintedProject == 0 ): szBuildErr += "Project: " + szProject + "\n" bPrintedProject = 1 szBuildErr += szLine + "\n" #err2 += szLine + "\n" bSuccess = 0 if aszTokens[1] == ":" and aszTokens[2] == "error" and aszTokens[3] == "PRJ0019": # the delete error line, add to the list szBuildErr += "IGNORE: " szBuildErr += szLine + "\n" #err2 += szLine + "\n" aszTokens = string.split(string.lstrip(szLine), ' ') if len(aszTokens) > 4: # can't do this: the weird delete error will trigger this and we need to ignore that # check that the "Rebuild All: x succeeded, y failed, z skipped line says no failures #if szT[0] == "Rebuild" and szT[3] rrorMessages + "The " + szTestName + " test failed\nThe error is " + szCompareLine #if szErrorMessages== "succeeded," and szT[5] == "failed," and not szT[4] == "0": # failure # bSuccess = 0 # check for linker errors if aszTokens[0] == "LINK" and aszTokens[1] == ":" and aszTokens[2] == "fatal" and aszTokens[3] == "error": if ( bPrintedProject == 0 ): szBuildErr += "Project: " + szProject + "\n" bPrintedProject = 1 # linker error line, add to the list szBuildErr += szLine + "\n" #err2 += szLine + "\n" bSuccess = 0 # can't do this either: Delete file problem # check the standard error dealie. # if szT[1] == '-' and szT[3] == "error(s)," and szT[2] != '0': #we have a build error here # err += szLine # err += "\n" #check for fatal error if aszTokens[2] == "fatal" and aszTokens[3] == "error": if ( bPrintedProject == 0 ): szBuildErr += "Project: " + szProject + "\n" bPrintedProject = 1 #fatal error szBuildErr += szLine + "\n" #err2 += szLine + "\n" bSuccess = 0 # return immediately if failed if not bSuccess: print szBuildErr szBuildErr + "\n\n" #ComplainToAdmin(err2) return szBuildErr return "" def RunBuildBatch(): bSuccess = 1 #aszBatchBuildLines = string.split( os.popen( "type " + szBatchFile, "r").read(), "\n" ) #bIsDevLine = 0 szBuildErrs = "" szBuildResult = RunBuild( "devenv everything.sln /build \"debug|win32\" " ) szBuildResult += RunBuild( "devenv everything.sln /build \"release|win32\" " ) szTestResult = RunTestScript() if szBuildResult != "": szBuildErrs += szBuildResult bSuccess = 0 if szTestResult != "\n": szBuildErrs += szTestResult bSuccess = 0 if not bSuccess: SendFailureEmail(szBuildErrs) return bSuccess # runs all the builds def RunAllBuilds(): bSuccess = 1 szBuildErrs = "" SystemHelpers.ChangeDir("\game") if g_bBuild: SystemHelpers.ChangeDir("\src") # build the shadercompiler and all shaders if g_bShaders: print( "Compiling shadercompile" ) os.system( "devenv shadercompile.sln /rebuild release > silence" ) # should check here to make sure the shadercompile worked SystemHelpers.ChangeDir("\\src\\materialsystem\\stdshaders") print( "Building shaders" ) child_stdin, child_stdout, child_stderr = os.popen3( "buildallshaders.bat" ) print( child_stdout.read() ) szSomeShaderErr = child_stderr.read() szShaderErrors = "" aszShaderLines = string.split( szSomeShaderErr, "\n" ) for szLine in aszShaderLines: dPrint( szLine ) nCount = string.count( szLine, 'U1073:' ) if nCount > 0: aszToken = string.split( szLine, " " ) szShaderName = aszToken[9] szShaderErrors = szShaderErrors + "The shader file " + szShaderName + " is missing and failed during buildallshaders.bat.\n" if szShaderErrors: SendFailureEmail( szShaderErrors ) SystemHelpers.ChangeDir("\\src") bSuccess = bSuccess & RunBuildBatch(); #XBOX Section if g_bXBOX: szXBoxOutput = RunBuild( "devenv source_x360.sln /rebuild \"release|xbox 360\"" ) if "\n\n\nError building configuration " + "devenv source_x360.sln /rebuild release" + ":\n" != szXBoxOutput: bSuccess = 0 SendFailureEmail( szXBoxOutput ) #delete tier0.dll if g_bRunTests: szTestErrors = RunTestScript() if szTestErrors != "\n": # success = 0 ComplainToAdmin( szTestErrors ) return bSuccess # builds from a local branch and runs a subset of tests def PerformSourceBuild(): # build and test in each configuration if not RunAllBuilds(): print "Source build failed" return 0 print "Source build: SUCCESS" return 1 # syncs, builds, runs tests def PerformDailyBuild(): print " changes detected, starting daily build" # update the counter to be what we're verifying change = P4.SubmittedChangelist( g_szP4SrcFilesToWatch ) g_nP4MostRecentCheckin = change g_nP4LastVerifiedCheckin = P4.GetCounter(g_szP4VerifiedCounter) if g_nP4MostRecentCheckin and g_nP4LastVerifiedCheckin: print "Most recent checkin is " + g_nP4MostRecentCheckin + "\n" print "Last verified checkin is " + g_nP4LastVerifiedCheckin + "\n" # the p4 command can occasionally fail to deliver a valid changelist number, unclear why # can't update the counter, it just means we'll run twice if change: P4.SetCounter(g_szP4ChangeCounter, change) # sync to the new files if ( g_bSync ): SystemHelpers.ChangeDir("\\src") print( "Cleaning\n" ) os.system("cleanalltargets.bat > silence") SystemHelpers.ChangeDir("\\") print "Synching force files." P4.Sync( g_szP4ForceFilesToWatch, 1 ) print "Synching other files." P4.Sync( g_szP4SyncFilesToWatch, 0 ) print( "Setting up VPC" ) os.system("setupVPC.bat") #P4.UnlockMutex(g_szP4Mutex) # build and test in each configuration if not RunAllBuilds(): print "Daily build failed" return # send a success email, from past the last successful checkin to the current if change: szVerifiedOrig = P4.GetCounter(g_szP4VerifiedCounter) if szVerifiedOrig: szVerifiedPlusOne = str( int( szVerifiedOrig ) + 1 ) changes = P4.GetChangelistRange(szVerifiedPlusOne, change, g_szP4SrcFilesToWatch ); for ch in changes: if len(ch) > 1: szEmail = P4.GetEmailFromChangeLine(ch) SendSuccessEmail(szEmail, ch) #SendSuccessEmail("jason", ch) # remember this change that we've verified P4.SetCounter(g_szP4VerifiedCounter, change) print "Daily build: AN UNEQUIVOCAL SUCCESS" def PrintConfig(): print("Configuration:") print("test = " + str(g_bTest)) print("run_tests = " + str(g_bRunTests)) print("lock_mutex = " + str(g_bLockMutex)) print("build = " + str(g_bBuild)) print("debug = " + str(g_bDebug)) print("dev = " + str(g_bDev)) print("shaders = " + str(g_bShaders)) print("sync = " + str(g_bSync)) print("email_alias = " + g_szEmailAlias) print("admin_email = " + g_szAdministrator) print("mail_host = " + g_szMailhost) print("sender_email = " + g_szSenderEmail) print("build_exe = " + g_szBuildExe) print("build_type = " + g_szBuildType) print("build_flags = " + g_szBuildFlags) print("test_file = " + g_szTestingFile) print("log_file = " + g_szLocalLogFile) print("error_dir = " + g_szPublishedErrorsDir) print("test_dir = " + g_szTestDirectory) print("src_files = " + g_szP4SrcFilesToWatch) print("force_files = " + g_szP4ForceFilesToWatch) print("sync_files = " + g_szP4SyncFilesToWatch) print("mutex = " + g_szP4Mutex) print("change_counter = " + g_szP4ChangeCounter) print("verify_counter = " + g_szP4VerifiedCounter) print("build_name = " + g_szBuildName) def ParseConfigFile(configFileName): aszBatchBuildLines = string.split( os.popen( "type " + configFileName, "r").read(), "\n" ) global g_bTest global g_bRunTests global g_bLockMutex global g_bBuild global g_bDebug global g_bDev global g_bShaders global g_bSync global g_szEmailAlias global g_szAdministrator global g_szMailhost global g_szSenderEmail global g_szBuildExe global g_szBuildType global g_szBuildFlags global g_szTestingFile global g_szLocalLogFile global g_szPublishedErrorsDir global g_szTestDirectory global g_szP4SrcFilesToWatch global g_szP4ForceFilesToWatch global g_szP4SyncFilesToWatch global g_szP4Mutex global g_szP4ChangeCounter global g_szP4VerifiedCounter global g_szBuildName for szLine in aszBatchBuildLines: aszTokens = string.split(string.lstrip(szLine), ' ', 2) firstToken = aszTokens[0] if firstToken == '#' or firstToken == '': continue secondToken = aszTokens[1] if firstToken == "test": g_bTest = int(secondToken) elif firstToken == "run_tests": g_bRunTests = int(secondToken) elif firstToken == "lock_mutex": g_bLockMutex = int(secondToken) elif firstToken == "build": g_bBuild = int(secondToken) elif firstToken == "debug": g_bDebug = int(secondToken) elif firstToken == "dev": g_bDev = int(secondToken) elif firstToken == "shaders": g_bShaders = int(secondToken) elif firstToken == "sync": g_bSync = int(secondToken) elif firstToken == "email_alias": g_szEmailAlias = secondToken elif firstToken == "admin_email": g_szAdministrator = secondToken elif firstToken == "mail_host": g_szMailhost = secondToken elif firstToken == "sender_email": g_szSenderEmail = secondToken elif firstToken == "build_exe": g_szBuildExe = secondToken elif firstToken == "build_type": g_szBuildType = secondToken elif firstToken == "build_flags": g_szBuildFlags = secondToken elif firstToken == "test_file": g_szTestingFile = secondToken elif firstToken == "log_file": g_szLocalLogFile = secondToken elif firstToken == "error_dir": g_szPublishedErrorsDir = secondToken elif firstToken == "test_dir": g_szTestDirectory = secondToken elif firstToken == "src_files": g_szP4SrcFilesToWatch += secondToken elif firstToken == "force_files": g_szP4ForceFilesToWatch += secondToken + ";" elif firstToken == "sync_files": g_szP4SyncFilesToWatch += secondToken + ";" elif firstToken == "mutex": g_szP4Mutex = secondToken elif firstToken == "change_counter": g_szP4ChangeCounter = secondToken elif firstToken == "verify_counter": g_szP4VerifiedCounter = secondToken elif firstToken == "build_name": g_szBuildName = secondToken PrintConfig() #----------------------------------------------------------------------------- # Main #----------------------------------------------------------------------------- if __name__ == '__main__': try: print "----------------------------------------------------" print g_szBuildName + " BUILD SCRIPT STARTED" ParseConfigFile(sys.argv[1]) while 1: if (g_bDev | P4.AnyNewCheckins( g_szP4ChangeCounter, g_szP4SrcFilesToWatch )): print "Changes Detected.\n" if ( (g_bTest & ~g_bLockMutex) | ~g_bLockMutex | P4.Query(g_szP4Mutex) ): PerformDailyBuild() g_bDev = 0 print "" print "------------------------------------------" print "waiting for changes to be detected..." else: time.sleep( g_nSleepTimeBetweenChecks - ( g_bTest * g_nSleepTimeBetweenChecks )) else: time.sleep( g_nSleepTimeBetweenChecks ) except RuntimeError, e: ComplainToAdmin(e)