diff options
Diffstat (limited to '.github')
| -rw-r--r-- | .github/release.yaml | 18 | ||||
| -rw-r--r-- | .github/workflows/build.yaml | 285 | ||||
| -rwxr-xr-x | .github/workflows/flatten_files.py | 180 |
3 files changed, 456 insertions, 27 deletions
diff --git a/.github/release.yaml b/.github/release.yaml new file mode 100644 index 0000000000..0f4884c944 --- /dev/null +++ b/.github/release.yaml @@ -0,0 +1,18 @@ +changelog:
+ exclude:
+ labels:
+ - ignore-for-release
+ authors:
+ - dependabot
+ categories:
+ - title: Breaking Changes ðŸ›
+ labels:
+ - semver-major
+ - breaking-change
+ - title: New Features 🎉
+ labels:
+ - semver-minor
+ - enhancement
+ - title: Other Changes
+ labels:
+ - '*'
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a110c018ec..69a7e4b6a6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,35 +4,67 @@ on: workflow_dispatch: pull_request: push: - branches: [main, contribute] + branches: [main, "actions*"] tags: ["*"] jobs: build: strategy: matrix: - runner: [windows-large] - configuration: [ReleaseOS] - addrsize: [64] + runner: [windows-large, macos-12-xl] + configuration: [Release, ReleaseOS] + python-version: ["3.11"] include: - - runner: windows-large + - runner: macos-12-xl + developer_dir: "/Applications/Xcode_14.0.1.app/Contents/Developer" + exclude: + - runner: macos-12-xl configuration: ReleaseOS - addrsize: 32 runs-on: ${{ matrix.runner }} + outputs: + viewer_channel: ${{ steps.build.outputs.viewer_channel }} + viewer_version: ${{ steps.build.outputs.viewer_version }} env: + AUTOBUILD_ADDRSIZE: 64 + AUTOBUILD_BUILD_ID: ${{ github.run_id }} AUTOBUILD_CONFIGURATION: ${{ matrix.configuration }} - AUTOBUILD_ADDRSIZE: ${{ matrix.addrsize }} + # authorizes fetching private constituent packages + AUTOBUILD_GITHUB_TOKEN: ${{ secrets.SHARED_AUTOBUILD_GITHUB_TOKEN }} AUTOBUILD_INSTALLABLE_CACHE: ${{ github.workspace }}/.autobuild-installables AUTOBUILD_VARIABLES_FILE: ${{ github.workspace }}/.build-variables/variables - AUTOBUILD_VSVER: "170" # vs2k22 - LOGFAIL: debug # Show details when tests fail + AUTOBUILD_VSVER: "170" + DEVELOPER_DIR: ${{ matrix.developer_dir }} + # Ensure that viewer builds engage Bugsplat. + BUGSPLAT_DB: "SecondLife_Viewer_2018" + BUGSPLAT_PASS: ${{ secrets.BUGSPLAT_PASS }} + BUGSPLAT_USER: ${{ secrets.BUGSPLAT_USER }} + build_coverity: false + build_log_dir: ${{ github.workspace }}/.logs + build_viewer: true + BUILDSCRIPTS_SHARED: ${{ github.workspace }}/.shared + # extracted and committed to viewer repo + BUILDSCRIPTS_SUPPORT_FUNCTIONS: ${{ github.workspace }}/buildscripts_support_functions GIT_REF: ${{ github.head_ref || github.ref }} + LL_SKIP_REQUIRE_SYSROOT: 1 + # Setting this variable directs Linden's TUT test driver code to capture + # test-program log output at the specified level, but to display it only if + # the individual test fails. + LOGFAIL: DEBUG + master_message_template_checkout: ${{ github.workspace }}/.master-message-template + # Only set variants to the one configuration: don't let build.sh loop + # over variants, let GitHub distribute variants over multiple hosts. + variants: ${{ matrix.configuration }} steps: - name: Checkout code uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Checkout build variables uses: actions/checkout@v3 with: @@ -40,8 +72,14 @@ jobs: ref: viewer path: .build-variables + - name: Checkout master-message-template + uses: actions/checkout@v3 + with: + repository: secondlife/master-message-template + path: .master-message-template + - name: Install autobuild and python dependencies - run: pip3 install autobuild llbase + run: pip3 install autobuild llsd - name: Cache autobuild packages uses: actions/cache@v3 @@ -63,31 +101,224 @@ jobs: env: RUNNER_OS: ${{ runner.os }} run: | + # set up things the viewer's build.sh script expects + set -x + mkdir -p "$build_log_dir" + mkdir -p "$BUILDSCRIPTS_SHARED/packages/lib/python" + source "$BUILDSCRIPTS_SUPPORT_FUNCTIONS" + if [[ "$OSTYPE" =~ cygwin|msys ]] + then + native_path() { cygpath --windows "$1"; } + shell_path() { cygpath --unix "$1"; } + else + native_path() { echo "$1"; } + shell_path() { echo "$1"; } + fi + finalize() + { + case "$1" in + true|0) + record_success "Build Succeeded" + ;; + *) + record_failure "Build Failed with $1" + ;; + esac + } + initialize_build() + { + echo "initialize_build" + } + initialize_version() + { + export revision="$AUTOBUILD_BUILD_ID" + } + python_cmd() + { + if [[ "x${1:0:1}" == "x-" ]] # -m, -c, etc. + then # if $1 is a switch, don't try to twiddle paths + "$(shell_path "$PYTHON_COMMAND")" "$@" + elif [[ "$(basename "$1")" == "codeticket.py" ]] + then # ignore any attempt to contact codeticket + echo "## $@" + else # running a script at an explicit path: fix path for Python + local script="$1" + shift + "$(shell_path "$PYTHON_COMMAND")" "$(native_path "$script")" "$@" + fi + } + repo_branch() + { + git -C "$1" branch | grep '^* ' | cut -c 3- + } + record_dependencies_graph() + { + echo "TODO: generate and post dependency graph" + } + # Since we're not uploading to codeticket, DO NOT sleep for minutes. + sleep() + { + echo "Not sleeping for $1 seconds" + } + export -f native_path shell_path finalize initialize_build initialize_version + export -f python_cmd repo_branch record_dependencies_graph sleep + ## Useful for diagnosing Windows LLProcess/LLLeap test failures + ##export APR_LOG="${RUNNER_TEMP}/apr.log" + export arch=$(uname | cut -b-6) + # Surprise! GH Windows runner's MINGW6 is a $arch value we've never + # seen before, so numerous tests don't know about it. + [[ "$arch" == "MINGW6" ]] && arch=CYGWIN + export AUTOBUILD="$(which autobuild)" + # Build with a tag like "Second_Life_Project_Shiny#abcdef0" to get a + # viewer channel "Second Life Project Shiny" (ignoring "#hash", + # needed to disambiguate tags). + if [[ "$GITHUB_REF_TYPE" == "tag" && "${GITHUB_REF_NAME:0:12}" == "Second_Life_" ]] + then viewer_channel="${GITHUB_REF_NAME%#*}" + export viewer_channel="${viewer_channel//_/ }" + else export viewer_channel="Second Life Test" + fi + echo "viewer_channel=$viewer_channel" >> "$GITHUB_OUTPUT" + # On windows we need to point the build to the correct python # as neither CMake's FindPython nor our custom Python.cmake module # will resolve the correct interpreter location. if [[ "$RUNNER_OS" == "Windows" ]]; then - export PYTHON="$(cygpath -m "$(which python)")" + export PYTHON="$(native_path "$(which python)")" echo "Python location: $PYTHON" + export PYTHON_COMMAND="$PYTHON" + else + export PYTHON_COMMAND="python3" fi - - autobuild configure -- -DVIEWER_CHANNEL="Second Life Test ${GIT_REF##*/}" - autobuild build --no-configure + export PYTHON_COMMAND_NATIVE="$(native_path "$PYTHON_COMMAND")" - # Find artifacts - if [[ "$RUNNER_OS" == "Windows" ]]; then - installer_path=$(find ./build-*/newview/ | grep '_Setup\.exe') - installer_name="$(basename $installer_path)" - elif [[ "$RUNNER_OS" == "macOS" ]]; then - installer_path=$(find ./build-*/newview/ | grep '\.dmg') - installer_name="$(basename $installer_path)" - fi + ./build.sh + + # Each artifact is downloaded as a distinct .zip file. Multiple jobs + # (per the matrix above) writing the same filepath to the same + # artifact name will *overwrite* that file. Moreover, they can + # interfere with each other, causing the upload to fail. + # https://github.com/actions/upload-artifact#uploading-to-the-same-artifact + # Given the size of our installers, and the fact that we typically + # only want to download just one instead of a single zip containing + # several, generate a distinct artifact name for each installer. + # If the matrix above can run multiple builds on the same + # platform, we must disambiguate on more than the platform name. + # e.g. if we were still running Windows 32-bit builds, we'd need to + # qualify the artifact with bit width. + echo "artifact=$RUNNER_OS" >> $GITHUB_OUTPUT - echo "installer_path=$installer_path" >> $GITHUB_OUTPUT - echo "installer_name=$installer_name" >> $GITHUB_OUTPUT - - name: Upload installer uses: actions/upload-artifact@v3 with: - name: ${{ steps.build.outputs.installer_name }} - path: ${{ steps.build.outputs.installer_path }} + name: "${{ steps.build.outputs.artifact }}-installer" + # emitted by build.sh, possibly multiple lines + path: | + ${{ steps.build.outputs.installer }} + + - name: Upload executable + if: steps.build.outputs.viewer_exe + uses: actions/upload-artifact@v3 + with: + name: "${{ steps.build.outputs.artifact }}-exe" + path: ${{ steps.build.outputs.viewer_exe }} + + # The other upload of nontrivial size is the symbol file. Use a distinct + # artifact for that too. + - name: Upload symbol file + uses: actions/upload-artifact@v3 + with: + name: "${{ steps.build.outputs.artifact }}-symbols" + path: | + ${{ steps.build.outputs.symbolfile }} + + - name: Upload metadata + uses: actions/upload-artifact@v3 + with: + # Call this artifact just "Windows" or "macOS" because it's the only + # artifact in which we expect files from both platforms with + # colliding names (e.g. autobuild-package.xml). Our flatten_files.py + # (see release step) resolves collisions by prepending the artifact + # name, so when we anticipate collisions, it's good to keep the + # artifact name short and sweet. + name: "${{ steps.build.outputs.artifact }}" + # emitted by build.sh, possibly multiple lines + path: | + ${{ steps.build.outputs.metadata }} + + - name: Upload physics package + uses: actions/upload-artifact@v3 + # should only be set for viewer-private + if: steps.build.outputs.physicstpv + with: + name: "${{ steps.build.outputs.artifact }}-physics" + # emitted by build.sh, zero or one lines + path: | + ${{ steps.build.outputs.physicstpv }} + + post-windows-symbols: + needs: build + runs-on: windows + steps: + - name: Post Windows symbols + uses: secondlife/viewer-post-bugsplat-windows@main + with: + username: ${{ secrets.BUGSPLAT_USER }} + password: ${{ secrets.BUGSPLAT_PASS }} + database: "SecondLife_Viewer_2018" + channel: ${{ needs.build.outputs.viewer_channel }} + version: ${{ needs.build.outputs.viewer_version }} + + post-mac-symbols: + needs: build + runs-on: macos-latest + steps: + - name: Post Mac symbols + uses: secondlife/viewer-post-bugsplat-mac@main + with: + username: ${{ secrets.BUGSPLAT_USER }} + password: ${{ secrets.BUGSPLAT_PASS }} + database: "SecondLife_Viewer_2018" + channel: ${{ needs.build.outputs.viewer_channel }} + version: ${{ needs.build.outputs.viewer_version }} + + release: + needs: build + runs-on: ubuntu-latest + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'Second_Life_') + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} + + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + +## - name: Install PyGithub +## run: pip3 install PyGithub +## +## - name: Unpack artifacts +## env: +## BUILD: ${{ toJSON(needs.build) }} +## run: .github/workflows/post_artifacts.py + + - uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Show what we downloaded + run: ls -R artifacts + + - name: Reshuffle artifact files + run: .github/workflows/flatten_files.py assets artifacts + + # forked from softprops/action-gh-release + - uses: secondlife-3p/action-gh-release@v1 + with: + prerelease: true + generate_release_notes: true + # the only reason we generate a GH release is to post build products + fail_on_unmatched_files: true + files: "assets/*" diff --git a/.github/workflows/flatten_files.py b/.github/workflows/flatten_files.py new file mode 100755 index 0000000000..542fa0206b --- /dev/null +++ b/.github/workflows/flatten_files.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""\ +@file flatten_files.py +@author Nat Goodspeed +@date 2023-08-18 +@brief From an input directory tree, populate a single flat output directory. + +$LicenseInfo:firstyear=2023&license=viewerlgpl$ +Copyright (c) 2023, Linden Research, Inc. +$/LicenseInfo$ +""" + +DESCRIPTION = """\ +From an input directory tree, populate a single flat output directory. + +For files with colliding names, rename them to unambiguous names derived from +their relative pathname within the input tree. + +This is useful when downloading GitHub build artifacts from multiple platforms +to post them all as release assets without collisions. +""" + +from collections import defaultdict +from contextlib import suppress +import filecmp +import os +from pathlib import Path +import sys + +class Error(Exception): + pass + +def flatten(output, input='.', dry_run=False): + try: + in_stat = os.stat(input) + except FileNotFoundError as err: + raise Error(f'{input} does not exist') from err + + try: + out_stat = os.stat(output) + except FileNotFoundError: + # output doesn't yet exist - at this point that's okay + out_stat = None + + # use samestat() to avoid being fooled by different ways of expressing the + # same path + if out_stat and os.path.samestat(out_stat, in_stat): + # output directory same as input: in this case, don't prune output + # directory from input tree walk because we'd prune everything + out_stat = None + elif out_stat: + # distinct existing output directory (potentially containing, or + # contained by, the input directory) + outfiles = [f for f in Path(output).rglob('*') if f.is_file()] + if outfiles: + print(f'Warning: {output} already contains {len(outfiles)} files:', file=sys.stderr) + for f in sorted(outfiles): + print(' ', f.relative_to(output), file=sys.stderr) + + # Use os.walk() instead of Path.rglob() so we can prune unwanted + # directories. + infiles = [] + for parent, dirs, files in os.walk(input): + infiles.extend(Path(parent, f) for f in files) + # Prune directories: because we must modify the dirs list in-place, + # and because we're using indexes, traverse backwards so deletion + # won't affect subsequent iterations. Yes we really must subtract 1 + # that many times. + for idx in range(len(dirs)-1, -1, -1): + if dirs[idx].startswith('.'): + # skip dot-named directories + print(f'ignoring {dirs[idx]}', file=sys.stderr) + del dirs[idx] + elif out_stat and os.path.samestat(os.stat(os.path.join(parent, dirs[idx])), out_stat): + # output directory lives under input directory: ignore any + # previous contents + print(f'ignoring nested output directory {os.path.join(parent, dirs[idx])}', + file=sys.stderr) + del dirs[idx] + + # Now that we've traversed the input tree, create the output directory if + # needed. + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + + # group files by basename to identify collisions + basenames = defaultdict(list) + for f in infiles: + basenames[f.name].append(f) + + # output names: populate it right away with unique basenames + outnames = { name: files[0] for name, files in basenames.items() + if len(files) == 1 } + + # now focus on the collisions + for name, files in basenames.items(): + if len(files) <= 1: + continue + + # Special case: are these colliding files equal? e.g. viewer_version.txt + # Pass shallow=False so we actually read the files in question. Even + # if they're identical, they've been downloaded from different + # artifacts and have different timestamps (which would fool the default + # shallow=True). This could be time-consuming if we were comparing two + # of our very large files, but (a) our very large files already have + # distinct names and so don't reach this call and (b) if we somehow do + # wind up in that situation, it would be even more important to post + # only a single copy. + if all(filecmp.cmp(files[0], f, shallow=False) for f in files[1:]): + # pick only one of them and use its simple basename + outnames[name] = files[0] + continue + + # Because of our intended use for GitHub Actions build artifacts, we + # assume the top-level artifact names are descriptive. We'd still like + # to eliminate mid-level directory names that don't help disambiguate, + # so for instance, given: + # Windows metadata/newview/viewer_version.txt + # macOS metadata/newview/viewer_version.txt + # we see no reason to retain the 'newview' pathname component. Try + # longer and longer prefixes of the pathname parents. (But don't + # forget to trim off the original input directory pathname.) + filepairs = [(f, f.relative_to(input)) for f in files] + partslen = max(len(rel.parts) for f, rel in filepairs) + # skip the basename itself, we'll append that explicitly + for prefixlen in range(partslen - 1): + # Consider these relative names (shouldn't occur, but...): + # parent/autobuild-package.xml + # parent/newview/autobuild-package.xml + # Unless these are in fact identical, they'll collide, meaning + # we'll see them here. But beware their unequal numbers of parts. + # partslen will be 3, so prefixlen will be 0, 1 -- but unless we + # constrain it with min(), for prefixlen == 1 we'd construct: + # ('parent', 'autobuild-package.xml', 'autobuild-package.xml') + # ('parent', 'newview', 'autobuild-package.xml') + # whereas of course the correct answer would be: + # ('parent', 'autobuild-package.xml') + # ('parent', 'newview', 'autobuild-package.xml') + # Since we already know the basename is identical for every f in + # files, though, we can omit it from our uniqueness testing. + trynames = { rel.parts[:min(prefixlen+1, len(rel.parts)-1)]: f + for f, rel in filepairs } + if len(trynames) == len(files): + # Found a prefix without collisions -- note that we're + # guaranteed to get here eventually since the full paths are + # distinct in the filesystem, we just want to try to shorten. + # Path.parts is specifically documented to be a tuple. Join + # the key tuple with some delimiter acceptable to the + # filesystem. + outnames.update(('-'.join(nametuple + (name,)), f) + for nametuple, f in trynames.items()) + # stop considering longer prefixlens + break + + # at this point outnames should have distinct keys -- move to the output + # directory + for name, f in outnames.items(): + newname = output / name + if (not dry_run) and newname != f: + newname = f.rename(newname) + print(f'{f} => {newname}') + +def main(*raw_args): + from argparse import ArgumentParser + parser = ArgumentParser(description=DESCRIPTION) + parser.add_argument('-n', '--dry-run', action='store_true', default=False, + help="""show what would happen without moving files""") + parser.add_argument('output', metavar='OUTDIR', + help="""populate OUTDIR with (possibly renamed) files""") + parser.add_argument('input', metavar='INDIR', nargs='?', default='.', + help="""recursively read files under INDIR tree""") + + args = parser.parse_args(raw_args) + flatten(args.output, args.input, dry_run=args.dry_run) + +if __name__ == "__main__": + try: + sys.exit(main(*sys.argv[1:])) + except Error as err: + sys.exit(str(err)) |