#========= Copyright 1996-2008, Valve Corporation, All rights reserved. ========== # # Master script for performing automated compiles of bsp's with cubemaps, reslists, # and nodegraphs. Compiles can be triggered manually by adding the vmf's absolute # path to mapbuild\forcelist.txt, or by adding the "autocompile" keyword # to the checkin comments for a vmf. # #================================================================================= use strict; use warnings; use Net::SMTP; my $maxCompiles = 3; # Maximum number of simultaneous compiles. my $running = 0; my @g_CompileList; my @finalizeList; my @g_Finalizing; my @g_CompileThreads; my %g_ToolArgs; my %g_RunningCompiles; my @g_OutputFiles; my $g_FilenameIdx = 0; my @g_MainDirs; my $g_MainDirIdx = 0; my @g_MapsToBuild; my $buildbsp = 1; # Compile the bsp with vbsp, vvis, and vrad my $buildreslists = 1; # Build reslists for the map my $buildnodegraphs = 1; # Build the nodegraph (.ain) for the map my $rootdir = "..\\..\\..\\.."; # Relative path from this script to one level above the "main" directory system( "echo. > finalize.txt" ); $SIG{CHLD}='IGNORE'; # Generate filenames for the compile output files for ( my $ct = 0; $ct < $maxCompiles * 2; ++$ct ) { my $filename = join( '', "$rootdir\\main1\\src\\devtools\\mapbuild\\buildqueue", $g_FilenameIdx++, ".txt" ); push( @g_OutputFiles, $filename ); } # Generate main directory names for each simultaneous compile for ( my $ct = 1; $ct <= $maxCompiles; ++$ct ) { my $filename = join( '', "main", $ct ); push( @g_MainDirs, $filename ); } # Loop for infinity. If changes are made to this script, the process needs to # be killed and restarted for those changes to take effect. Note, any maps that # are compiling or waiting in the queue will NOT be picked up again when the script # is restarted. Any maps that were compiling or waiting to compile when the process # was stopped will need to be started manually by adding the vmf absolute path # to forcelist.txt. A map can be manually added to the queue at any time while the script # is running by adding the vmf absolute path to forcelist.txt. while ( 1 ) { # Generate a list of maps that need to be built. For vmf's that have been changed in # Perforce, the checkin comments are parsed and if the word "autocompile" # is found then that map will be added to the build list. All maps that are listed in # forcelist.txt are also added to the build list. syncChangedMaps(); # Append the maps to the build queue while filtering out duplicates. We only # want to filter duplicates out of the queue - if a map is already compiling # then it's reasonable to add it to the queue again because it must have changed # since the current build began. my $updated = 0; for my $map ( @g_MapsToBuild ) { if ( $map =~ /(\w*.vmf)/ ) { # make sure it's not already in the queue my $duplicate = 0; my $testname = $1; for ( @g_CompileList ) { if ( /$testname/ ) { $duplicate = 1; last; } } system( "echo. >> log.txt" ); if ( $duplicate != 1 ) { print( "Adding to compile list $map\n" ); system( "echo Adding to compile list $map >> log.txt" ); push( @g_CompileList, $map ); $updated = 1; } else { print( "Duplicate map found: $testname\n" ); system( "echo Duplicate map found: $testname >> log.txt" ); } } } splice( @g_MapsToBuild ); # If there is a build slot open, start building the next map StartCompiles(); sleep( 60 ); # Time is arbitrary - just don't spam Perforce # Load in the finalize queue open( INFILE, "; close( INFILE ); system( "echo. > finalize.txt" ); # Finalize the maps (cubemaps, reslist, nodegraph, and checkin). The finalize list # also contains the compile output (times,args,etc.) that need to go in the checkin comments. my $maindir = ""; my $mod = ""; my $mapname = ""; my $strTime = ""; my $vbspargs = ""; my $vvisargs = ""; my $vradargs = ""; my $outfile = ""; for ( @finalizeList ) { if ( /Finalize: (\w*) (\w*)/ ) { $mod = $1; $mapname = $2; next; } elsif ( /maindir: (.*)/ ) { $maindir = $1; next; } elsif ( /vbspargs: (.*)/ ) { $vbspargs = $1; next; } elsif ( /vvisargs: (.*)/ ) { $vvisargs = $1; next; } elsif ( /vradargs: (.*)/ ) { $vradargs = $1; next; } elsif ( /time: (.*)/ ) { $strTime = $1; next; } elsif ( /outfile: (.*)/ ) { # This is the final entry for a map $outfile = $1; } else { next; } print( "\nFinalizing $mod\\$mapname.bsp\n" ); system( "echo Finalizing $mapname.bsp >> log.txt" ); # A compile slot has opened up --$running; push( @g_Finalizing, $mapname ); # Let another compile start in the just-emptied slot. StartCompiles(); my $ldrResult = 0; my $hdrResult = 0; # HACK: Make sure a zombie process can't hold up the system system( "taskkill /im hl2.exe /f" ); chdir( "$rootdir\\$maindir\\src\\devtools\\mapbuild" ); # Get the Perforce client and owner names for the checkin file my $g_client; my $g_owner; my @clientspec = readpipe "p4 client -o"; for ( @clientspec ) { if ( /^Client:[\s*](.*)/ ) { $g_client = $1; print( "Using client $g_client\n" ); } elsif ( /^Owner:[\s*](.*)/ ) { $g_owner = $1; print( "Using owner $g_owner\n" ); } } if ( $buildbsp == 1 ) { # Build cubemaps system( "echo. >> $outfile" ); system( "time /t >> $outfile" ); system( "echo Building cubemaps for $mapname. >> $outfile" ); if ( $vradargs =~ /-hdr|-both/ ) { $hdrResult = system( "$rootdir\\$maindir\\game\\hl2.exe -allowdebug -game $mod -window -w 1152 -h 864 +mat_picmip 0 -dev +mat_hdr_level 2 +sv_cheats 1 +map $mapname -buildcubemaps" ); } if ( $vradargs =~ /-ldr|-both/ ) { $ldrResult = system( "$rootdir\\$maindir\\game\\hl2.exe -allowdebug -game $mod -window -w 1152 -h 864 +mat_picmip 0 -dev +mat_hdr_level 0 +sv_cheats 1 +map $mapname -buildcubemaps" ); } } if ( $buildreslists == 1 ) { system( "echo. >> $outfile" ); system( "time /t >> $outfile" ); system( "echo Building reslists and nodegraph for $mapname. >> $outfile" ); system( "del /s $rootdir\\$maindir\\game\\$mod\\reslists_temp\\*.lst" ); system( "p4 edit $rootdir\\$maindir\\game\\$mod\\reslists_xbox\\$mapname.lst" ); system( "p4 add $rootdir\\$maindir\\game\\$mod\\reslists_xbox\\$mapname.lst" ); system( "del $rootdir\\$maindir\\game\\$mod\\reslists_xbox\\$mapname.lst" ); my $extraflags = ""; # if ( $mod =~ /portal/ ) # { # $extraflags = "-tempcontent"; # } system( "del \/s $rootdir\\$maindir\\game\\modelsounds.cache" ); system( "$rootdir\\$maindir\\game\\hl2.exe -allowdebug -game $mod -window -dev -makereslists makereslists_xbox.txt $extraflags +map $mapname" ); } elsif ( $buildnodegraphs == 1 ) { # system( "p4 edit $rootdir\\$maindir\\game\\$mod\\maps\\graphs\\$mapname.ain >> log.txt" ); # TODO: Build nodegraphs method. Currently we get this for free when building reslists. } # Generate the checkin file system( "echo. >> $outfile" ); system( "time /t >> $outfile" ); system( "echo Submitting to Perforce. >> $outfile" ); open( OUTFILE, ">checkin.txt" ); print( OUTFILE "Change: new\n\n" ); print( OUTFILE "Client: $g_client\n\n" ); print( OUTFILE "User: $g_owner\n\n" ); print( OUTFILE "Status: new\n\n" ); print( OUTFILE "Description: Autobuild for map $mapname.bsp\n" ); print( OUTFILE "\n" ); if ( $hdrResult == 1 ) { print( OUTFILE "\tWARNING! There was an error while building HDR cubemaps.\n" ); print( OUTFILE "\tHDR Cubemaps may need to be rebuilt before using this map.\n\n" ); } if ( $ldrResult == 1 ) { print( OUTFILE "\tWARNING! There was an error while building LDR cubemaps.\n" ); print( OUTFILE "\tLDR Cubemaps may need to be rebuilt before using this map.\n\n" ); } if ( $buildbsp == 1 ) { print( OUTFILE "\tCOMPILE ARGS\n" ); print( OUTFILE "\tvbsp:\t",$vbspargs,"\n" ); print( OUTFILE "\tvvis:\t",$vvisargs,"\n" ); print( OUTFILE "\tvrad:\t",$vradargs,"\n" ); print( OUTFILE "\n" ); print( OUTFILE "\tBuild time: $strTime\n" ); print( OUTFILE "\n" ); } # Add the comments from the last checkin of this map system( "p4 changes -m 1 -s submitted -l $rootdir\\$maindir\\content\\$mod\\maps\\$mapname.vmf > comments.txt" ); open(INFILE, "comments.txt"); my @comments = ; close(INFILE); if ( $buildbsp == 1 ) { print( OUTFILE "\tCheckin Comments: " ); print( OUTFILE @comments,"\n\n" ); print( OUTFILE "\tRebuilt bsp\n" ); } if ( $buildnodegraphs == 1 ) { print( OUTFILE "\n\tRebuilt nodegraph\n" ); } if ( $buildreslists == 1 ) { print( OUTFILE "\n\tRebuilt reslist\n" ); } print( OUTFILE "Files:\n" ); if ( $buildbsp == 1 ) { print( OUTFILE "\t//ValveGames/staging/game/$mod/maps/$mapname.bsp\n" ); } if ( $buildnodegraphs == 1 ) { # Copy up to fileserver system( "copy $rootdir\\$maindir\\game\\$mod\\maps\\graphs\\$mapname.ain \"\\\\fileserver\\user\\xbox\\xbox_orange\\graphs\\$mod\\$mapname.ain\"" ); } if ( $buildreslists == 1 ) { print( OUTFILE "\t//ValveGames/staging/game/$mod/reslists_xbox/$mapname.lst\n" ); } close( OUTFILE ); # Check in the map system( "p4 submit -i < checkin.txt >> log.txt" ); # Send the build output in an email to the submitter my $email = "kerry\@valvesoftware.com"; for ( @comments ) { if ( /Change \d* on \S* by (\w*)@/ ) { $email = join( '@', $1, "valvesoftware.com" ); } } open(INFILE, "$outfile"); my @compileoutput = ; close(INFILE); my$smtp; $smtp = Net::SMTP->new('exchange2.valvesoftware.com'); $smtp->mail("Mapbuilder"); $smtp->to("$email"); $smtp->data(); $smtp->datasend( "To: $email\n" ); $smtp->datasend( "Subject: $mapname has finished compiling.\n" ); $smtp->datasend( @comments ); $smtp->datasend( @compileoutput ); $smtp->dataend(); $smtp->quit; delete( $g_RunningCompiles{$outfile} ); splice( @g_Finalizing ); chdir( "$rootdir\\main1\\src\\devtools\\mapbuild" ); } } # end while(1) #---------------------------------------- # Start a compile if there is an open slot #---------------------------------------- sub StartCompiles { my $idx = 0; print( "Current maplist:\n" ); for ( @g_CompileList ) { print( "$_\n" ); } my $skipCt = 0; while ( @g_CompileList > $skipCt && $running < $maxCompiles ) { my $nextMap = splice( @g_CompileList, 0, 1 ); # Make sure this map isn't already compiling in another slot my $duplicate = 0; for ( values %g_RunningCompiles ) { $nextMap =~ /(\w*).vmf/; if ( /^$1$/ ) { system( "echo $nextMap already compiling - skipping for now. >> log.txt" ); push @g_CompileList, $nextMap; ++$skipCt; $duplicate = 1; last; } } if ( $duplicate == 0 ) { Compile( $nextMap ); ++$running; ++$idx; my $mapct = @g_CompileList; system( "echo Maps waiting: $mapct >> log.txt" ); } } UpdateStats(); } #----------------------------------------------------------- # Compile a single map - this subroutine forks the process #----------------------------------------------------------- sub Compile { # sync the rest of the content and bins. my $maindir = $g_MainDirs[$g_MainDirIdx]; $g_MainDirIdx = ( $g_MainDirIdx + 1 ) % $maxCompiles; chdir( "$rootdir\\$maindir\\src\\devtools\\mapbuild" ); system( "p4 sync $rootdir\\$maindir\\game\\..." ); system( "p4 sync $rootdir\\$maindir\\src\\..." ); # parse the map and mod name from the full path my $fullpath = shift; my $mod; my $mapname; if ( $fullpath =~ /(\w*)\\maps\\(\w*).vmf/ ) { $mod = $1; $mapname = $2; } else { print( "Error getting map name and mod\n" ); exit(); } # Get the next output file. All output from the build tools # (vbsp,vvis,vrad) will be redirected to this file. my $outfile = undef; for ( @g_OutputFiles ) { unless ( defined $g_RunningCompiles{$_} ) { $outfile = $_; last; } } unless ( defined $outfile ) { my $filename = join( '', "buildqueue", $g_FilenameIdx++, ".txt" ); push( @g_OutputFiles, $filename ); $outfile = $filename; } $g_RunningCompiles{$outfile} = $mapname; # Force-sync the vmf and bsp system( "echo Compiling $mod\\$mapname.bsp > $outfile" ); system( "echo. >> log.txt" ); system( "date /t >> log.txt" ); system( "time /t >> log.txt" ); system( "echo Compling $mod\\$mapname.bsp >> log.txt" ); system( "p4 sync -f ..\\..\\..\\content\\$mod\\maps\\$mapname.vmf >> log.txt" ); system( "p4 sync -f $rootdir\\$maindir\\content\\$mod\\maps\\$mapname.vmf >> log.txt" ); system( "p4 sync -f $rootdir\\$maindir\\game\\$mod\\maps\\$mapname.bsp >> log.txt" ); if ( $buildbsp == 1 ) { print "\nCompiling $mod\\$mapname.bsp\n"; # load the map compile args for this mod if( !open( INFILE, "$rootdir\\$maindir\\game\\$mod\\scripts\\mapautocompile.txt") ) { print( "Error opening autocompile.txt\n" ); } my @lines = ; close (INFILE); my $tool; my $rest; my $toolmap; for ( @lines ) { if ( /map: (.*)/ ) { $toolmap = $1; } elsif ( /\s+(.+)/ ) { ($tool, $rest) = split ( /:\s*/, $1, 2 ); $g_ToolArgs{$toolmap}{$tool} = $rest; } } # Open the bsp for edit system( "p4 edit $rootdir\\$maindir\\game\\$mod\\maps\\$mapname.bsp" ); system( "p4 add $rootdir\\$maindir\\game\\$mod\\maps\\$mapname.bsp" ); system( "del $rootdir\\$maindir\\game\\$mod\\maps\\$mapname.bsp" ); } chdir( "$rootdir\\main1\\src\\devtools\\mapbuild" ); #-------------------------------------- # fork this compile to a new process #-------------------------------------- my $pid = fork(); print "fork failed!" unless defined $pid; if ( $pid != 0 ) { # I am the parent return; } my $strTime = ""; my $vbspargs = ""; my $vvisargs = ""; my $vradargs = ""; chdir( "$rootdir\\$maindir\\src\\devtools\\mapbuild" ); if ( $buildbsp == 1 ) { # Compile # delete the bsp to ensure that vbsp succeeded system( "del $rootdir\\$maindir\\content\\$mod\\maps\\$mapname.bsp" ); my $startTime = time; $vbspargs = compileTool( $maindir, $mod, $mapname, "vbsp", $outfile ); $vvisargs = compileTool( $maindir, $mod, $mapname, "vvis", $outfile ); $vradargs = compileTool( $maindir, $mod, $mapname, "vrad", $outfile ); my $totalTime = time - $startTime; my $hours = int($totalTime / 3600); $totalTime -= ($hours * 3600); my $minutes = int( $totalTime / 60 ); $totalTime -= ($minutes * 60); my $seconds = $totalTime; $strTime = sprintf( "%02d:%02d:%02d", $hours, $minutes, $seconds ); } chdir( "$rootdir\\main1\\src\\devtools\\mapbuild" ); # Add the map's name to the finalize list while( !open( OUTFILE, ">>finalize.txt" ) ) {} print OUTFILE "Finalize: $mod $mapname\n"; print OUTFILE "maindir: $maindir\n"; if ( $buildbsp == 1 ) { # Output the compile args and time print OUTFILE "vbspargs: $vbspargs\n"; print OUTFILE "vvisargs: $vvisargs\n"; print OUTFILE "vradargs: $vradargs\n"; print OUTFILE "time: $strTime\n"; } print OUTFILE "outfile: $outfile\n"; close( OUTFILE ); # This child is finished exit(); } #---------------------------------------- # Run a map build tool (vbsp, vvis, vrad) #---------------------------------------- sub compileTool { my $maindir = shift; my $mod = shift; my $map = shift; my $tool = shift; my $outfile = shift; # Load the compile arguments that were parsed from mapautocompile.txt. # If the map has its own custom args, then use those - otherwise, # just use the default set. my $toolArgs = $g_ToolArgs{"default"}{$tool}; if ( exists $g_ToolArgs{$map}{$tool} ) { $toolArgs = $g_ToolArgs{$map}{$tool}; } system( "echo. >> $outfile" ); system( "time /t >> $outfile" ); system( "echo Starting $tool for $map >> $outfile" ); system( "echo. >> $outfile" ); # HACK: There is a crash bug when using -both in vrad. If the -both # flags is present, remove it and do two vrad compiles with -ldr and -hdr. if ( $tool =~ /vrad/ && $toolArgs =~ /-both/ ) { system( "echo Found -both argument. Splitting into -ldr and -hdr >> $outfile" ); my $newArgs = $toolArgs; $newArgs =~ s/-both //; system( "$rootdir\\$maindir\\game\\bin\\$tool -vproject $rootdir\\$maindir\\game\\$mod $newArgs -ldr $rootdir\\$maindir\\content\\$mod\\maps\\$map >> $outfile" ); system( "$rootdir\\$maindir\\game\\bin\\$tool -vproject $rootdir\\$maindir\\game\\$mod $newArgs -hdr $rootdir\\$maindir\\content\\$mod\\maps\\$map >> $outfile" ); } else { system( "$rootdir\\$maindir\\game\\bin\\$tool -vproject $rootdir\\$maindir\\game\\$mod $toolArgs $rootdir\\$maindir\\content\\$mod\\maps\\$map >> $outfile" ); } # copy the new bsp into the maps directory system( "copy /Y $rootdir\\$maindir\\content\\$mod\\maps\\$map.bsp $rootdir\\$maindir\\game\\$mod\\maps" ); system( "echo. >> $outfile" ); system( "echo Finished >> $outfile" ); return $toolArgs; } #----------------------------------- # Check if this map should build now #----------------------------------- sub syncChangedMaps { # TODO: Get these mod names from a file? my @mods = ( "ep2", "portal", "episodic", "hl2", "tf" ); # Query perforce for changed maps by doing a speculative sync. This generates a list # of files that it WOULD sync, without actually doing the sync. # BUG: This doesn't return new files that have been added - don't know why yet. my @maps; for my $mod ( @mods ) { push ( @maps, readpipe "p4 sync -n ..\\..\\..\\content\\$mod\\maps\\*.vmf" ); } # forcelist.txt allows maps to be manually added to the build. open(INFILE, "forcelist.txt"); my @forcemaps = ; close( INFILE ); system( "echo. > forcelist.txt" ); # Run through the list of maps to see if any should be built for( @maps) { if( /updating (.*)|adding (.*)/ ) { # Check if this map should be built now if ( shouldBuild( $1 ) == 0 ) { next; } # Add this map name to the build list system( "p4 sync -f $1" ); push( @g_MapsToBuild, $1 ); system( "echo Adding to the build list: $1 >> log.txt" ); } } for( @forcemaps ) { if ( /\.vmf/ ) { # Add this map name to the build list $_ =~ /(.*)/; system( "p4 sync -f $1" ); push( @g_MapsToBuild, $1 ); system( "echo Forcing add to the build list: $1 >> log.txt" ); } } } #----------------------------------- # Check if this map should build now #----------------------------------- sub shouldBuild { my $map = shift; # if the map line contains the force flag, build the map if ( $map =~ /-forcebuild/ ) { return 1; } # Get the comments from the last checkin of this map # and look for the autocompile keyword my @comments = readpipe "p4 changes -m 1 -s submitted -l $map"; for( @comments ) { if ( /autocompile/i ) { return 1; } } return 0; } #---------------------------------------------------------------------------------- # Hacky little routine to build an html file that contains the current state # of the autocompiler (running, maps building, maps waiting, etc.) and a frameset # that shows the compile output for each map that's currently building. This way # users of the autocompiler can keep an eye on their map's progress. #---------------------------------------------------------------------------------- sub UpdateStats { my $noCompiles = 0; my $noWaiting = 0; if ( keys( %g_RunningCompiles ) == 0 && @g_Finalizing == 0 ) { $noCompiles = 1; } if ( @g_CompileList == 0 ) { $noWaiting = 1; } # Start building the main html page open( OUTFILE, ">buildlist.htm" ); print( OUTFILE "\n" ); print( OUTFILE "\n" ); print( OUTFILE "Mapbuilder state: " ); if ( $noCompiles == 1 && $noWaiting == 1 ) { print( OUTFILE "Ready
\n" ); } else { print( OUTFILE "RUNNING
\n" ); } print( OUTFILE "Max simultaneous compiles: $maxCompiles\n" ); print( OUTFILE "

\n" ); print( OUTFILE "Active:\n

\n" ); # Show a list of maps that are currently compiling my $filect = 0; if ( $noCompiles == 1 ) { print( OUTFILE "None" ); } else { for ( @g_OutputFiles ) { if ( defined( $g_RunningCompiles{$_} ) ) { my $testname = $g_RunningCompiles{$_}; my $filename = $_; if ( $filename =~ /(\w*\.txt)/ ) { $filename = $1; } my $finishing = 0; for ( @g_Finalizing ) { if ( /$testname/ ) { $finishing = 1; } } print( OUTFILE "$testname.bsp" ); ++$filect; if ( $finishing == 1 ) { print( OUTFILE " - Finishing (cubemaps, reslist, nodegraph.)\n
\n" ); } else { print( OUTFILE " - Compiling\n
\n" ); } } } } # Show a list of maps that are waiting to compile print( OUTFILE "

\nWaiting:\n

\n" ); if ( $noWaiting == 1 ) { print( OUTFILE "None" ); } else { for ( @g_CompileList ) { /(\w*).vmf/; print( OUTFILE "$1.bsp\n
\n" ); } } print( OUTFILE "\n" ); close( OUTFILE ); # create the main page open( OUTFILE, ">buildqueue.htm" ); print( OUTFILE "\n" ); print( OUTFILE "\n" ); my $size = 0; if ( $filect != 0 ) { print( OUTFILE "\n" ); print( OUTFILE "\n" ); for ( @g_OutputFiles ) { if ( defined( $g_RunningCompiles{$_} ) ) { my $filename = $_; if ( $filename =~ /(\w*\.txt)/ ) { $filename = $1; } print( OUTFILE "\n" ); } } print( OUTFILE "\n" ); print( OUTFILE "\n" ); close( OUTFILE ); }