diff options
author | Bryan O'Sullivan <bos@lindenlab.com> | 2008-06-02 21:14:31 +0000 |
---|---|---|
committer | Bryan O'Sullivan <bos@lindenlab.com> | 2008-06-02 21:14:31 +0000 |
commit | 9db949eec327df4173fde3de934a87bedb0db13c (patch) | |
tree | aeffa0f0e68b1d2ceb74d460cbbd22652c9cd159 /scripts | |
parent | 419e13d0acaabf5e1e02e9b64a07648bce822b2f (diff) |
svn merge -r88066:88786 svn+ssh://svn.lindenlab.com/svn/linden/branches/cmake-9-merge
dataserver-is-deprecated
for-fucks-sake-whats-with-these-commit-markers
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/build_version.py | 54 | ||||
-rwxr-xr-x | scripts/install.py | 700 | ||||
-rwxr-xr-x | scripts/update_version_files.py | 2 |
3 files changed, 755 insertions, 1 deletions
diff --git a/scripts/build_version.py b/scripts/build_version.py new file mode 100755 index 0000000000..4bef290b7d --- /dev/null +++ b/scripts/build_version.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# Print the build information embedded in a header file. +# +# Expects to be invoked from the command line with a file name and a +# list of directories to search. The file name will be one of the +# following: +# +# llversionserver.h +# llversionviewer.h +# +# The directory list that follows will include indra/llcommon, where +# these files live. + +import errno, os, re + +def get_version(filename): + fp = open(filename) + data = fp.read() + fp.close() + + vals = {} + m = re.search('const S32 LL_VERSION_MAJOR = (\d+);', data) + vals['major'] = m.group(1) + m = re.search('const S32 LL_VERSION_MINOR = (\d+);', data) + vals['minor'] = m.group(1) + m = re.search('const S32 LL_VERSION_PATCH = (\d+);', data) + vals['patch'] = m.group(1) + m = re.search('const S32 LL_VERSION_BUILD = (\d+);', data) + vals['build'] = m.group(1) + + return "%(major)s.%(minor)s.%(patch)s.%(build)s" % vals + +if __name__ == '__main__': + import sys + + try: + for path in sys.argv[2:]: + name = os.path.join(path, sys.argv[1]) + try: + print get_version(name) + break + except OSError, err: + if err.errno != errno.ENOENT: + raise + else: + print >> sys.stderr, 'File not found:', sys.argv[1] + sys.exit(1) + except AttributeError: + print >> sys.stderr, 'Error: malformatted file: ', name + sys.exit(1) + except IndexError: + print >> sys.stderr, ('Usage: %s llversion[...].h [directories]' % + sys.argv[0]) diff --git a/scripts/install.py b/scripts/install.py new file mode 100755 index 0000000000..7421d697f6 --- /dev/null +++ b/scripts/install.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python +"""\ +@file install.py +@author Phoenix +@date 2008-01-27 +@brief Install files into an indra checkout. + +Install files as specified by: +https://wiki.lindenlab.com/wiki/User:Phoenix/Library_Installation + + +$LicenseInfo:firstyear=2007&license=mit$ + +Copyright (c) 2007, Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +$/LicenseInfo$ +""" + +import copy +import errno +import md5 +import optparse +import os +import pprint +import sys +import tarfile +import urllib +import urlparse + +from sets import Set as set, ImmutableSet as frozenset + +# Locate -our- python library relative to our install location. +from os.path import realpath, dirname, join + +# Walk back to checkout base directory +base_dir = dirname(dirname(realpath(__file__))) +# Walk in to libraries directory +lib_dir = join(join(join(base_dir, 'indra'), 'lib'), 'python') + +if lib_dir not in sys.path: + sys.path.insert(0, lib_dir) + +from indra.base import llsd +from indra.util import helpformatter + +class InstallFile(object): + "This is just a handy way to throw around details on a file in memory." + def __init__(self, pkgname, url, md5sum, cache_dir, platform_path): + self.pkgname = pkgname + self.url = url + self.md5sum = md5sum + filename = urlparse.urlparse(url)[2].split('/')[-1] + self.filename = os.path.join(cache_dir, filename) + self.platform_path = platform_path + + def __str__(self): + return "ifile{%s:%s}" % (self.pkgname, self.url) + + def _is_md5_match(self): + hasher = md5.new(file(self.filename).read()) + if hasher.hexdigest() == self.md5sum: + return True + return False + + def is_match(self, platform): + """@brief Test to see if this ifile is part of platform +@param platform The target platform. Eg, win32 or linux/i686/gcc/3.3 +@return Returns True if the ifile is in the platform. + """ + if self.platform_path == 'common': + return True + req_platform_path = platform.split('/') + #print "platform:",req_platform_path + #print "path:",self.platform_path + # to match, every path part much match + match_count = min(len(req_platform_path), len(self.platform_path)) + for ii in range(0, match_count): + if req_platform_path[ii] != self.platform_path[ii]: + return False + #print "match!" + return True + + + def fetch_local(self): + print "Looking for:",self.filename + if not os.path.exists(self.filename): + print "not there -- fetching" + elif self.md5sum and not self._is_md5_match(): + print "Found, but md5 does not match." + os.remove(self.filename) + else: + print "Found matching package" + return + print "Downloading",self.url,"to local file",self.filename + urllib.urlretrieve(self.url, self.filename) + if self.md5sum and not self._is_md5_match(): + raise RuntimeError("Error matching md5 for %s" % self.url) + +class LicenseDefinition(object): + def __init__(self, definition): + #probably looks like: + # { text : ..., + # url : ... + # blessed : ... + # } + self._definition = definition + + +class BinaryDefinition(object): + def __init__(self, definition): + #probably looks like: + # { packages : {platform...}, + # copyright : ... + # license : ... + # description: ... + # } + self._definition = definition + + def _ifiles_from(self, tree, pkgname, cache_dir): + return self._ifiles_from_path(tree, pkgname, cache_dir, []) + + def _ifiles_from_path(self, tree, pkgname, cache_dir, path): + ifiles = [] + if tree.has_key('url'): + ifiles.append(InstallFile( + pkgname, + tree['url'], + tree.get('md5sum', None), + cache_dir, + path)) + else: + for key in tree: + platform_path = copy.copy(path) + platform_path.append(key) + ifiles.extend( + self._ifiles_from_path( + tree[key], + pkgname, + cache_dir, + platform_path)) + return ifiles + + def ifiles(self, pkgname, platform, cache_dir): + """@brief return a list of appropriate InstallFile instances to install +@param pkgname The name of the package to be installed, eg 'tut' +@param platform The target platform. Eg, win32 or linux/i686/gcc/3.3 +@param cache_dir The directory to cache downloads. +@return Returns a list of InstallFiles which are part of this install + """ + if 'packages' not in self._definition: + return [] + all_ifiles = self._ifiles_from( + self._definition['packages'], + pkgname, + cache_dir) + if platform == 'all': + return all_ifiles + return [ifile for ifile in all_ifiles if ifile.is_match(platform)] + +class InstalledPackage(object): + def __init__(self, definition): + # looks like: + # { url1 : [file1,file2,...], + # url2 : [file1,file2,...],... + # } + self._installed = {} + for url in definition: + self._installed[url] = definition[url] + + def urls(self): + return self._installed.keys() + + def files_in(self, url): + return self._installed[url] + + def remove(self, url): + self._installed.pop(url) + + def add_files(self, url, files): + self._installed[url] = files + +class Installer(object): + def __init__(self, install_filename, installed_filename, dryrun): + self._install_filename = install_filename + self._install_changed = False + self._installed_filename = installed_filename + self._installed_changed = False + self._dryrun = dryrun + self._binaries = {} + self._licenses = {} + self._installed = {} + self.load() + + def load(self): + if os.path.exists(self._install_filename): + install = llsd.parse(file(self._install_filename).read()) + try: + for name in install['binaries']: + self._binaries[name] = BinaryDefinition( + install['binaries'][name]) + except KeyError: + pass + try: + for name in install['licenses']: + self._licenses[name] = LicenseDefinition(install['licenses'][name]) + except KeyError: + pass + if os.path.exists(self._installed_filename): + installed = llsd.parse(file(self._installed_filename).read()) + try: + bins = installed['binaries'] + for name in bins: + self._installed[name] = InstalledPackage(bins[name]) + except KeyError: + pass + + def _write(self, filename, state): + print "Writing state to",filename + if not self._dryrun: + file(filename, 'w').write(llsd.format_xml(state)) + + def save(self): + if self._install_changed: + state = {} + state['licenses'] = {} + for name in self._licenses: + state['licenses'][name] = self._licenses[name]._definition + #print "self._binaries:",self._binaries + state['binaries'] = {} + for name in self._binaries: + state['binaries'][name] = self._binaries[name]._definition + self._write(self._install_filename, state) + if self._installed_changed: + state = {} + state['binaries'] = {} + bin = state['binaries'] + for name in self._installed: + #print "installed:",name,self._installed[name]._installed + bin[name] = self._installed[name]._installed + self._write(self._installed_filename, state) + + def is_license_info_valid(self): + valid = True + for bin in self._binaries: + binary = self._binaries[bin]._definition + if not binary.has_key('license'): + valid = False + print >>sys.stderr, "No license info for binary", bin + '.' + continue + if binary['license'] not in self._licenses: + valid = False + lic = binary['license'] + print >>sys.stderr, "Missing license info for '" + lic + "'", + print >>sys.stderr, 'in binary', bin + '.' + return valid + + def detail_binary(self, name): + "Return a binary definition detail" + try: + detail = self._binaries[name]._definition + return detail + except KeyError: + return None + + def _update_field(self, binary, field): + """Given a block and a field name, add or update it. + @param binary[in,out] a dict containing all the details about a binary. + @param field the name of the field to update. + """ + if binary.has_key(field): + print "Update value for '" + field + "'" + print "(Leave blank to keep current value)" + print "Current Value: '" + binary[field] + "'" + else: + print "Specify value for '" + field + "'" + value = raw_input("Enter New Value: ") + if binary.has_key(field) and not value: + pass + elif value: + binary[field] = value + + def _add_package(self, binary): + """Add an url for a platform path to the binary. + @param binary[in,out] a dict containing all the details about a binary.""" + print """\ +Please enter a new package location and url. Some examples: +common -- specify a package for all platforms +linux -- specify a package for all arch and compilers on linux +darwin/universal -- specify a mac os x universal +win32/i686/vs/2003 -- specify a windows visual studio 2003 package""" + target = raw_input("Package path: ") + url = raw_input("Package URL: ") + md5sum = raw_input("Package md5: ") + path = target.split('/') + if not binary.has_key('packages'): + binary['packages'] = {} + update = binary['packages'] + for child in path: + if not update.has_key(child): + update[child] = {} + parent = update + update = update[child] + parent[child]['url'] = llsd.uri(url) + parent[child]['md5sum'] = md5sum + + def adopt_binary(self, name): + "Interactively pull a new binary into the install" + if not self._binaries.has_key(name): + print "Adding binary '" + name + "'." + self._binaries[name] = BinaryDefinition({}) + else: + print "Updating binary '" + name + "'." + binary = self._binaries[name]._definition + for field in ('copyright', 'license', 'description'): + self._update_field(binary, field) + self._add_package(binary) + print "Adopted binary '" + name + "':" + pprint.pprint(self._binaries[name]) + self._install_changed = True + return True + + def orphan_binary(self, name): + self._binaries.pop(name) + self._install_changed = True + + def add_license(self, name, text, url): + if self._licenses.has_key(name): + print "License '" + name + "' being overwritten." + definition = {} + if url: + definition['url'] = url + if not url and text is None: + print "Please enter license text. End input with EOF (^D)." + text = sys.stdin.read() + definition['text'] = text + self._licenses[name] = LicenseDefinition(definition) + self._install_changed = True + return True + + def remove_license(self, name): + self._licenses.pop(name) + self._install_changed = True + + def _determine_install_set(self, ifiles): + """@brief determine what to install +@param ifiles A list of InstallFile instances which are necessary for this install +@return Returns the tuple (ifiles to install, ifiles to remove)""" + installed_list = [] + for package in self._installed: + installed_list.extend(self._installed[package].urls()) + installed_set = set(installed_list) + #print "installed_set:",installed_set + install_list = [ifile.url for ifile in ifiles] + install_set = set(install_list) + #print "install_set:",install_set + remove_set = installed_set.difference(install_set) + to_remove = [ifile for ifile in ifiles if ifile.url in remove_set] + #print "to_remove:",to_remove + install_set = install_set.difference(installed_set) + to_install = [ifile for ifile in ifiles if ifile.url in install_set] + #print "to_install:",to_install + return to_install, to_remove + + def _build_ifiles(self, platform, cache_dir): + """@brief determine what files to install and remove +@param platform The target platform. Eg, win32 or linux/i686/gcc/3.3 +@param cache_dir The directory to cache downloads. +@return Returns the tuple (ifiles to install, ifiles to remove)""" + ifiles = [] + for bin in self._binaries: + ifiles.extend(self._binaries[bin].ifiles(bin, platform, cache_dir)) + return self._determine_install_set(ifiles) + + def _remove(self, to_remove): + remove_file_list = [] + for ifile in to_remove: + remove_file_list.extend( + self._installed[ifile.pkgname].files_in(ifile.url)) + self._installed[ifile.pkgname].remove(ifile.url) + self._installed_changed = True + for filename in remove_file_list: + print "rm",filename + if not self._dryrun: + os.remove(filename) + + def _install(self, to_install, install_dir): + for ifile in to_install: + tar = tarfile.open(ifile.filename, 'r') + print "Extracting",ifile.filename,"to destination",install_dir + if not self._dryrun: + # *NOTE: try to call extractall, which first appears + # in python 2.5. Phoenix 2008-01-28 + try: + tar.extractall(path=install_dir) + except AttributeError: + _extractall(tar, path=install_dir) + if self._installed.has_key(ifile.pkgname): + self._installed[ifile.pkgname].add_files(ifile.url, tar.getnames()) + else: + # *HACK: this understands the installed package syntax. + definition = { ifile.url : tar.getnames() } + self._installed[ifile.pkgname] = InstalledPackage(definition) + self._installed_changed = True + + def do_install(self, platform, install_dir, cache_dir): + """@brief Do the installation for for the platform. +@param platform The target platform. Eg, win32 or linux/i686/gcc/3.3 +@param install_dir The root directory to install into. Created if missing. +@param cache_dir The directory to cache downloads. Created if missing.""" + if not self._binaries: + raise RuntimeError("No binaries to install. Please add them.") + _mkdir(install_dir) + _mkdir(cache_dir) + to_install, to_remove = self._build_ifiles(platform, cache_dir) + + # we do this in multiple steps reduce the likelyhood to have a + # bad install. + for ifile in to_install: + ifile.fetch_local() + self._remove(to_remove) + self._install(to_install, install_dir) + + +# +# *NOTE: PULLED FROM PYTHON 2.5 tarfile.py Phoenix 2008-01-28 +# +def _extractall(tar, path=".", members=None): + """Extract all members from the archive to the current working + directory and set owner, modification time and permissions on + directories afterwards. `path' specifies a different directory + to extract to. `members' is optional and must be a subset of the + list returned by getmembers(). + """ + directories = [] + + if members is None: + members = tar + + for tarinfo in members: + if tarinfo.isdir(): + # Extract directory with a safe mode, so that + # all files below can be extracted as well. + try: + os.makedirs(os.path.join(path, tarinfo.name), 0777) + except EnvironmentError: + pass + directories.append(tarinfo) + else: + tar.extract(tarinfo, path) + + # Reverse sort directories. + directories.sort(lambda a, b: cmp(a.name, b.name)) + directories.reverse() + + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + path = os.path.join(path, tarinfo.name) + try: + tar.chown(tarinfo, path) + tar.utime(tarinfo, path) + tar.chmod(tarinfo, path) + except tarfile.ExtractError, e: + if tar.errorlevel > 1: + raise + else: + tar._dbg(1, "tarfile: %s" % e) + + +def _mkdir(directory): + "Safe, repeatable way to make a directory." + try: + os.makedirs(directory) + except OSError, e: + if e[0] != errno.EEXIST: + raise + +def _get_platform(): + "Return appropriate platform packages for the environment." + platform_map = { + 'darwin': 'darwin', + 'linux2': 'linux', + 'win32' : 'win32', + 'cygwin' : 'win32', + 'solaris' : 'solaris' + } + return platform_map[sys.platform] + +def main(): + parser = optparse.OptionParser( + usage="usage: %prog [options]", + formatter = helpformatter.Formatter(), + description="""This script fetches and installs binary packages. + +The process is to open and read an install manifest file which specifies +what files should be installed. For each file in the manifest: + * make sure it has a license + * check the installed version + ** if not installed and needs to be, download and install + ** if installed version differs, download & install + +When specifying a platform, you can specify 'all' to install all +packages, or any platform of the form: + +OS[/arch[/compiler[/compiler_version]]] + +Where the supported values for each are: +OS: darwin, linux, win32, solaris +arch: i686, x86_64, ppc, universal +compiler: vs, gcc +compiler_version: 2003, 2005, 2008, 3.3, 3.4, 4.0, etc. + +No checks are made to ensure a valid combination of platform +parts. Some exmples of valid platforms: + +win32 +win32/i686/vs/2005 +linux/x86_64/gcc/3.3 +linux/x86_64/gcc/4.0 +darwin/universal/gcc/4.0 +""") + parser.add_option( + '--dry-run', + action='store_true', + default=False, + dest='dryrun', + help='Do not actually install files. Downloads will still happen.') + parser.add_option( + '--install-manifest', + type='string', + default=join(base_dir, 'install.xml'), + dest='install_filename', + help='The file used to describe what should be installed.') + parser.add_option( + '--installed-manifest', + type='string', + default=join(base_dir, 'installed.xml'), + dest='installed_filename', + help='The file used to record what is installed.') + parser.add_option( + '-p', '--platform', + type='string', + default=_get_platform(), + dest='platform', + help="""Override the automatically determined platform. \ +You can specify 'all' to do a complete installation of all binaries.""") + parser.add_option( + '--cache-dir', + type='string', + default=join(base_dir, '.install.cache'), + dest='cache_dir', + help='Where to download files.') + parser.add_option( + '--install-dir', + type='string', + default=base_dir, + dest='install_dir', + help='Where to unpack the installed files.') + parser.add_option( + '--skip-license-check', + action='store_false', + default=True, + dest='check_license', + help="Do not perform the license check.") + parser.add_option( + '--add-license', + type='string', + default=None, + dest='new_license', + help="""Add a license to the install file. Argument is the name of \ +license. Specify --license-url if the license is remote or specify \ +--license-text, otherwse the license text will be read from standard \ +input.""") + parser.add_option( + '--remove-license', + type='string', + default=None, + dest='remove_license', + help="Remove a named license.") + parser.add_option( + '--license-url', + type='string', + default=None, + dest='license_url', + help="""Put the specified url into an added license. \ +Ignored if --add-license is not specified.""") + parser.add_option( + '--license-text', + type='string', + default=None, + dest='license_text', + help="""Put the text into an added license. \ +Ignored if --add-license is not specified.""") + parser.add_option( + '--orphan', + type='string', + default=None, + dest='orphan', + help="Remove a binary from the install file.") + parser.add_option( + '--adopt', + type='string', + default=None, + dest='adopt', + help="""Add a binary into the install file. Argument is the name of \ +the binary to add.""") + parser.add_option( + '--list', + action='store_true', + default=False, + dest='list_binaries', + help="List the binaries in the install manifest") + parser.add_option( + '--details', + type='string', + default=None, + dest='detail_binary', + help="Get detailed information on specified binary.") + options, args = parser.parse_args() + + + installer = Installer( + options.install_filename, + options.installed_filename, + options.dryrun) + + # + # Handle the queries for information + # + if options.list_binaries: + print "binary list:",installer._binaries.keys() + return 0 + if options.detail_binary: + detail = installer.detail_binary(options.detail_binary) + if detail: + print "Detail on binary",options.detail_binary+":" + pprint.pprint(detail) + else: + print "Bianry '"+options.detail_binary+"' not found in", + print "install file." + return 0 + + # + # Handle updates -- can only do one of these + # *TODO: should this change the command line syntax? + # + if options.new_license: + if not installer.add_license( + options.new_license, + options.license_text, + options.license_url): + return 1 + elif options.remove_license: + installer.remove_license(options.remove_license) + elif options.orphan: + installer.orphan_binary(options.orphan) + elif options.adopt: + if not installer.adopt_binary(options.adopt): + return 1 + else: + if options.check_license: + if not installer.is_license_info_valid(): + print >>sys.stderr, 'Please add or correct the license', + print >>sys.stderr, 'information in', + print >>sys.stderr, options.install_filename + '.' + print >>sys.stderr, "You can also use the --add-license", + print >>sys.stderr, "option. See", sys.argv[0], "--help" + return 1 + + # *TODO: check against a list of 'known good' licenses. + # *TODO: check for urls which conflict -- will lead to + # problems. + + installer.do_install( + options.platform, + options.install_dir, + options.cache_dir) + + # save out any changes + installer.save() + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/update_version_files.py b/scripts/update_version_files.py index 077120127b..95ff19c65b 100755 --- a/scripts/update_version_files.py +++ b/scripts/update_version_files.py @@ -100,7 +100,7 @@ re_map['indra/llcommon/llversionserver.h'] = \ 'const S32 LL_VERSION_BUILD = %(SERVER_VER_BUILD)s;'), ('const char \* const LL_CHANNEL = "(.+)";', 'const char * const LL_CHANNEL = "%(SERVER_CHANNEL)s";')) -re_map['indra/newview/res/newViewRes.rc'] = \ +re_map['indra/newview/res/viewerRes.rc'] = \ (('FILEVERSION [0-9,]+', 'FILEVERSION %(VER_MAJOR)s,%(VER_MINOR)s,%(VER_PATCH)s,%(VER_BUILD)s'), ('PRODUCTVERSION [0-9,]+', |