diff options
Diffstat (limited to 'build.sh')
| -rwxr-xr-x | build.sh | 370 |
1 files changed, 227 insertions, 143 deletions
@@ -1,9 +1,13 @@ #!/bin/sh -# This is a the master build script - it is intended to be run by parabuild -# It is called by a wrapper script in the shared repository which sets up -# the environment from the various BuildParams files and does all the build -# result post-processing. +# This is the custom build script for the viewer +# +# It must be run by the Linden Lab build farm shared buildscript because +# it relies on the environment that sets up, functions it provides, and +# the build result post-processing it does. +# +# The shared buildscript build.sh invokes this because it is named 'build.sh', +# which is the default custom build script name in buildscripts/hg/BuildParams # # PLEASE NOTE: # @@ -11,9 +15,6 @@ # Cygwin can be tricky.... # * The special style in which python is invoked is intentional to permit # use of a native python install on windows - which requires paths in DOS form -# * This script relies heavily on parameters defined in BuildParams -# * The basic convention is that the build name can be mapped onto a mercurial URL, -# which is also used as the "branch" name. check_for() { @@ -33,52 +34,82 @@ build_dir_Linux() build_dir_CYGWIN() { - echo build-vc100 + echo build-vc120 +} + +viewer_channel_suffix() +{ + local package_name="$1" + local suffix_var="${package_name}_viewer_channel_suffix" + local suffix=$(eval "echo \$${suffix_var}") + if [ "$suffix"x = ""x ] + then + echo "" + else + echo "_$suffix" + fi } installer_Darwin() { - ls -1td "$(build_dir_Darwin ${last_built_variant:-Release})/newview/"*.dmg 2>/dev/null | sed 1q + local package_name="$1" + local package_dir="$(build_dir_Darwin ${last_built_variant:-Release})/newview/" + local pattern=".*$(viewer_channel_suffix ${package_name})_[0-9]+_[0-9]+_[0-9]+_[0-9]+_i386\\.dmg\$" + # since the additional packages are built after the base package, + # sorting oldest first ensures that the unqualified package is returned + # even if someone makes a qualified name that duplicates the last word of the base name + local package=$(ls -1tr "$package_dir" 2>/dev/null | grep -E "$pattern" | head -n 1) + test "$package"x != ""x && echo "$package_dir/$package" } installer_Linux() { - ls -1td "$(build_dir_Linux ${last_built_variant:-Release})/newview/"*.tar.bz2 2>/dev/null | sed 1q + local package_name="$1" + local package_dir="$(build_dir_Linux ${last_built_variant:-Release})/newview/" + local pattern=".*$(viewer_channel_suffix ${package_name})_[0-9]+_[0-9]+_[0-9]+_[0-9]+_i686\\.tar\\.bz2\$" + # since the additional packages are built after the base package, + # sorting oldest first ensures that the unqualified package is returned + # even if someone makes a qualified name that duplicates the last word of the base name + package=$(ls -1tr "$package_dir" 2>/dev/null | grep -E "$pattern" | head -n 1) + test "$package"x != ""x && echo "$package_dir/$package" } installer_CYGWIN() { - v=${last_built_variant:-Release} - d=$(build_dir_CYGWIN $v) - if [ -r "$d/newview/$v/touched.bat" ] + local package_name="$1" + local variant=${last_built_variant:-Release} + local build_dir=$(build_dir_CYGWIN ${variant}) + local package_dir + if [ "$package_name"x = ""x ] then - p=$(sed 's:.*=::' "$d/newview/$v/touched.bat") - echo "$d/newview/$v/$p" + package_dir="${build_dir}/newview/${variant}" + else + package_dir="${build_dir}/newview/${package_name}/${variant}" + fi + if [ -r "${package_dir}/touched.bat" ] + then + local package_file=$(sed 's:.*=::' "${package_dir}/touched.bat") + echo "${package_dir}/${package_file}" fi } pre_build() { local variant="$1" - begin_section "Pre$variant" + begin_section "Configure $variant" [ -n "$master_message_template_checkout" ] \ && [ -r "$master_message_template_checkout/message_template.msg" ] \ && template_verifier_master_url="-DTEMPLATE_VERIFIER_MASTER_URL=file://$master_message_template_checkout/message_template.msg" - check_for "Before 'autobuild configure'" ${build_dir}/packages/dictionaries - - "$AUTOBUILD" configure -c $variant -- \ + "$autobuild" configure -c $variant -- \ -DPACKAGE:BOOL=ON \ -DRELEASE_CRASH_REPORTING:BOOL=ON \ -DVIEWER_CHANNEL:STRING="\"$viewer_channel\"" \ - -DVIEWER_LOGIN_CHANNEL:STRING="\"$viewer_login_channel\"" \ -DGRID:STRING="\"$viewer_grid\"" \ -DLL_TESTS:BOOL="$run_tests" \ -DTEMPLATE_VERIFIER_OPTIONS:STRING="$template_verifier_options" $template_verifier_master_url - check_for "After 'autobuild configure'" ${build_dir}/packages/dictionaries - - end_section "Pre$variant" + end_section "Configure $variant" } package_llphysicsextensions_tpv() @@ -88,16 +119,22 @@ package_llphysicsextensions_tpv() if [ "$variant" = "Release" ] then llpetpvcfg=$build_dir/packages/llphysicsextensions/autobuild-tpv.xml - "$AUTOBUILD" build --verbose --config-file $llpetpvcfg -c Tpv + "$autobuild" build --verbose --config-file $llpetpvcfg -c Tpv # capture the package file name for use in upload later... PKGTMP=`mktemp -t pgktpv.XXXXXX` trap "rm $PKGTMP* 2>/dev/null" 0 - "$AUTOBUILD" package --verbose --config-file $llpetpvcfg > $PKGTMP + "$autobuild" package --verbose --config-file $llpetpvcfg --results-file "$(native_path $PKGTMP)" tpv_status=$? - sed -n -e 's/^wrote *//p' $PKGTMP > $build_dir/llphysicsextensions_package + if [ -r "${PKGTMP}" ] + then + cat "${PKGTMP}" >> "$build_log" + eval $(cat "${PKGTMP}") # sets autobuild_package_{name,filename,md5} + autobuild_package_filename="$(shell_path "${autobuild_package_filename}")" + echo "${autobuild_package_filename}" > $build_dir/llphysicsextensions_package + fi else - echo "Do not provide llphysicsextensions_tpv for $variant" + record_event "Do not provide llphysicsextensions_tpv for $variant" llphysicsextensions_package="" fi end_section "PhysicsExtensions_TPV" @@ -109,39 +146,39 @@ build() local variant="$1" if $build_viewer then - begin_section "Viewer$variant" - check_for "Before 'autobuild build'" ${build_dir}/packages/dictionaries + "$autobuild" build --no-configure -c $variant + build_ok=$? + + # Run build extensions + if [ $build_ok -eq 0 -a -d ${build_dir}/packages/build-extensions ]; then + for extension in ${build_dir}/packages/build-extensions/*.sh; do + begin_section "Extension $extension" + . $extension + end_section "Extension $extension" + if [ $build_ok -ne 0 ]; then + break + fi + done + fi - "$AUTOBUILD" build --no-configure -c $variant - viewer_build_ok=$? - end_section "Viewer$variant" + # *TODO: Make this a build extension. package_llphysicsextensions_tpv tpvlib_build_ok=$? - if [ $viewer_build_ok -eq 0 -a $tpvlib_build_ok -eq 0 ] + if [ $build_ok -eq 0 -a $tpvlib_build_ok -eq 0 ] then echo true >"$build_dir"/build_ok else echo false >"$build_dir"/build_ok fi - check_for "After 'autobuild configure'" ${build_dir}/packages/dictionaries - fi } -# This is called from the branch independent script upon completion of all platform builds. -build_docs() -{ - begin_section Docs if "$AUTOBUILD" build -c Doxygen then echo true >"$build_dir"/build_ok else echo false >"$build_dir"/build_ok fi - end_section Docs -} - - # Check to see if we were invoked from the wrapper, if not, re-exec ourselves from there if [ "x$arch" = x ] then @@ -153,7 +190,7 @@ then cat <<EOF This script, if called in a development environment, requires that the branch independent build script repository be checked out next to this repository. -This repository is located at http://hg.lindenlab.com/parabuild/buildscripts +This repository is located at http://bitbucket.org/lindenlabinternal/sl-buildscripts EOF exit 1 fi @@ -162,65 +199,24 @@ fi # Check to see if we're skipping the platform eval '$build_'"$arch" || pass -# Run the version number update script -# File no longer exists in code-sep branch, so let's make sure it exists in order to use it. -if test -f scripts/update_version_files.py ; then - begin_section UpdateVer - eval $(python scripts/update_version_files.py \ - --channel="$viewer_channel" \ - --server_channel="$server_channel" \ - --revision=$revision \ - --verbose \ - | sed -n -e "s,Setting viewer channel/version: '\([^']*\)' / '\([^']*\)',VIEWER_CHANNEL='\1';VIEWER_VERSION='\2',p")\ - || fail update_version_files.py - echo "{\"Type\":\"viewer\",\"Version\":\"${VIEWER_VERSION}\"}" > summary.json - end_section UpdateVer -fi - -if [ -z "$AUTOBUILD" ] +# ensure AUTOBUILD is in native path form for child processes +AUTOBUILD="$(native_path "$AUTOBUILD")" +# set "$autobuild" to cygwin path form for use locally in this script +autobuild="$(shell_path "$AUTOBUILD")" +if [ ! -x "$autobuild" ] then - export autobuild_dir="$here/../../../autobuild/bin/" - if [ -d "$autobuild_dir" ] - then - export AUTOBUILD="$autobuild_dir"autobuild - if [ -x "$AUTOBUILD" ] - then - # *HACK - bash doesn't know how to pass real pathnames to native windows python - case "$arch" in - CYGWIN) AUTOBUILD=$(cygpath -u $AUTOBUILD.cmd) ;; - esac - else - record_failure "Not executable: $AUTOBUILD" - exit 1 - fi - else - record_failure "Not found: $autobuild_dir" - exit 1 - fi + record_failure "AUTOBUILD not executable: '$autobuild'" + exit 1 fi -# load autbuild provided shell functions and variables -# Merov: going back to the previous code that passes even if it fails catching a failure -# TODO: use the correct code here under and fix the llbase import in python code -#if "$AUTOBUILD" source_environment > source_environment -#then -# . source_environment -#else - # dump environment variables for debugging -# env|sort -# record_failure "autobuild source_environment failed" -# cat source_environment >&3 -# exit 1 -#fi -eval "$("$AUTOBUILD" source_environment)" +# load autobuild provided shell functions and variables +eval "$("$autobuild" source_environment)" # dump environment variables for debugging +begin_section "Environment" env|sort +end_section "Environment" -check_for "Before 'autobuild install'" ${build_dir}/packages/dictionaries - - -check_for "After 'autobuild install'" ${build_dir}/packages/dictionaries # Now run the build succeeded=true build_processes= @@ -233,66 +229,125 @@ do # Only the last built arch is available for upload last_built_variant="$variant" - begin_section "Do$variant" + begin_section "$variant" build_dir=`build_dir_$arch $variant` build_dir_stubs="$build_dir/win_setup/$variant" - begin_section "PreClean" + begin_section "Initialize Build Directory" rm -rf "$build_dir" - end_section "PreClean" - mkdir -p "$build_dir" mkdir -p "$build_dir/tmp" + end_section "Initialize Build Directory" if pre_build "$variant" "$build_dir" >> "$build_log" 2>&1 then - if $build_link_parallel - then - begin_section BuildParallel - ( build "$variant" "$build_dir" > "$build_dir/build.log" 2>&1 ) & - build_processes="$build_processes $!" - end_section BuildParallel - else - begin_section "Build$variant" + begin_section "Build $variant" build "$variant" "$build_dir" 2>&1 | tee -a "$build_log" | sed -n 's/^ *\(##teamcity.*\)/\1/p' if `cat "$build_dir/build_ok"` then - echo so far so good. + if [ "$variant" == "Release" ] + then + if [ -r "$build_dir/autobuild-package.xml" ] + then + begin_section "Autobuild metadata" + upload_item docs "$build_dir/autobuild-package.xml" text/xml + if [ "$arch" != "Linux" ] + then + record_dependencies_graph # defined in buildscripts/hg/bin/build.sh + else + record_event "no dependency graph for linux (probable python version dependency)" 1>&2 + fi + end_section "Autobuild metadata" + else + record_event "no autobuild metadata at '$build_dir/autobuild-package.xml'" + fi + else + record_event "do not record autobuild metadata for $variant" + fi else - record_failure "Build of \"$variant\" failed." + record_failure "Build of \"$variant\" failed." fi - end_section "Build$variant" - fi - else - record_failure "Build Prep for \"$variant\" failed." + end_section "Build $variant" + fi + end_section "$variant" + if ! $succeeded + then + record_event "remaining variants skipped due to $variant failure" + break fi - end_section "Do$variant" done -# If we are building variants in parallel, wait, then collect results. -# This requires that the build dirs are variant specific -if $build_link_parallel && [ x"$build_processes" != x ] +# build debian package +if [ "$arch" == "Linux" ] then - begin_section WaitParallel - wait $build_processes - for variant in $variants - do - eval '$build_'"$variant" || continue - eval '$build_'"$arch"_"$variant" || continue - - begin_section "Build$variant" - build_dir=`build_dir_$arch $variant` - build_dir_stubs="$build_dir/win_setup/$variant" - tee -a $build_log < "$build_dir/build.log" | sed -n 's/^ *\(##teamcity.*\)/\1/p' - if `cat "$build_dir/build_ok"` + if $succeeded + then + if $build_viewer_deb && [ "$last_built_variant" == "Release" ] then - echo so far so good. + begin_section "Build Viewer Debian Package" + have_private_repo=false + # mangle the changelog + dch --force-bad-version \ + --distribution unstable \ + --newversion "${VIEWER_VERSION}" \ + "Automated build #$build_id, repository $branch revision $revision." \ + >> "$build_log" 2>&1 + + # build the debian package + $pkg_default_debuild_command >>"$build_log" 2>&1 || record_failure "\"$pkg_default_debuild_command\" failed." + + # Unmangle the changelog file + hg revert debian/changelog + + end_section "Build Viewer Debian Package" + + # Run debian extensions + if [ -d ${build_dir}/packages/debian-extensions ]; then + for extension in ${build_dir}/packages/debian-extensions/*.sh; do + . $extension + done + fi + # Move any .deb results. + mkdir -p ../packages_public + mkdir -p ../packages_private + mv ${build_dir}/packages/*.deb ../packages_public 2>/dev/null || true + mv ${build_dir}/packages/packages_private/*.deb ../packages_private 2>/dev/null || true + + # upload debian package and create repository + begin_section "Upload Debian Repository" + for deb_file in `/bin/ls ../packages_public/*.deb ../*.deb 2>/dev/null`; do + upload_item debian $deb_file binary/octet-stream + done + for deb_file in `/bin/ls ../packages_private/*.deb 2>/dev/null`; do + upload_item debian_private $deb_file binary/octet-stream + have_private_repo=true + done + + create_deb_repo + + # Rename the local debian_repo* directories so that the master buildscript + # doesn't make a remote repo again. + for debian_repo_type in debian_repo debian_repo_private; do + if [ -d "$build_log_dir/$debian_repo_type" ]; then + mv $build_log_dir/$debian_repo_type $build_log_dir/${debian_repo_type}_pushed + fi + done + + if [ $have_private_repo = true ]; then + eval "$python_command \"$redirect\" '\${private_S3PROXY_URL}${S3PREFIX}repo/$repo/rev/$revision/index.html'"\ + >"$build_log_dir/private.html" || fatal generating redirect + upload_item global_redirect "$build_log_dir/private.html" text/html + + fi + + end_section "Upload Debian Repository" + else - record_failure "Parallel build of \"$variant\" failed." + echo debian build not enabled fi - end_section "Build$variant" - done - end_section WaitParallel + else + echo skipping debian build due to failed build. + fi fi # check status and upload results to S3 @@ -300,18 +355,31 @@ if $succeeded then if $build_viewer then - begin_section Upload - # Upload installer - note that ONLY THE FIRST ITEM uploaded as "installer" - # will appear in the version manager. + begin_section Upload Installer + # Upload installer package=$(installer_$arch) if [ x"$package" = x ] || test -d "$package" then # Coverity doesn't package, so it's ok, anything else is fail succeeded=$build_coverity else + # Upload base package. upload_item installer "$package" binary/octet-stream upload_item quicklink "$package" binary/octet-stream - [ -f summary.json ] && upload_item installer summary.json text/plain + [ -f $build_dir/summary.json ] && upload_item installer $build_dir/summary.json text/plain + + # Upload additional packages. + for package_id in $additional_packages + do + package=$(installer_$arch "$package_id") + if [ x"$package" != x ] + then + upload_item installer "$package" binary/octet-stream + upload_item quicklink "$package" binary/octet-stream + else + record_failure "Failed to find additional package for '$package_id'." + fi + done case "$last_built_variant" in Release) @@ -320,8 +388,15 @@ then do upload_item symbolfile "$build_dir/$symbolfile" binary/octet-stream done - + + # Upload the actual dependencies used + if [ -r "$build_dir/packages/installed-packages.xml" ] + then + upload_item installer "$build_dir/packages/installed-packages.xml" text/xml + fi + # Upload the llphysicsextensions_tpv package, if one was produced + # *TODO: Make this an upload-extension if [ -r "$build_dir/llphysicsextensions_package" ] then llphysicsextensions_package=$(cat $build_dir/llphysicsextensions_package) @@ -335,15 +410,24 @@ then ;; esac + # Run upload extensions + if [ -d ${build_dir}/packages/upload-extensions ]; then + for extension in ${build_dir}/packages/upload-extensions/*.sh; do + begin_section "Upload Extenstion $extension" + . $extension + end_section "Upload Extenstion $extension" + done + fi + # Upload stub installers upload_stub_installers "$build_dir_stubs" fi - end_section Upload + end_section Upload Installer else - echo skipping viewer + echo skipping upload of installer fi else - echo skipping upload of build results due to failed build. + echo skipping upload of installer due to failed build. fi # The branch independent build.sh script invoking this script will finish processing |