From 2cfa661c279b4c74b33a86bba600d02df5d2706c Mon Sep 17 00:00:00 2001 From: brad kittenbrink Date: Thu, 27 May 2010 14:49:15 -0700 Subject: WIP VPLAT-248 - added a generate_breakpad_symbols target to the build and info to upload symbol files to s3 after the build. --- indra/newview/generate_breakpad_symbols.py | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 indra/newview/generate_breakpad_symbols.py (limited to 'indra/newview/generate_breakpad_symbols.py') diff --git a/indra/newview/generate_breakpad_symbols.py b/indra/newview/generate_breakpad_symbols.py new file mode 100644 index 0000000000..884dd06f84 --- /dev/null +++ b/indra/newview/generate_breakpad_symbols.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# @file generate_breakpad_symbols.py +# @author Brad Kittenbrink +# @brief Simple tool for generating google_breakpad symbol information +# for the crash reporter. +# +# $LicenseInfo:firstyear=2010&license=viewergpl$ +# +# Copyright (c) 2010-2010, Linden Research, Inc. +# +# Second Life Viewer Source Code +# The source code in this file ("Source Code") is provided by Linden Lab +# to you under the terms of the GNU General Public License, version 2.0 +# ("GPL"), unless you have obtained a separate licensing agreement +# ("Other License"), formally executed by you and Linden Lab. Terms of +# the GPL can be found in doc/GPL-license.txt in this distribution, or +# online at http://secondlifegrid.net/programs/open_source/licensing/gplv2 +# +# There are special exceptions to the terms and conditions of the GPL as +# it is applied to this Source Code. View the full text of the exception +# in the file doc/FLOSS-exception.txt in this software distribution, or +# online at +# http://secondlifegrid.net/programs/open_source/licensing/flossexception +# +# By copying, modifying or distributing this software, you acknowledge +# that you have read and understood your obligations described above, +# and agree to abide by those obligations. +# +# ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO +# WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY, +# COMPLETENESS OR PERFORMANCE. +# $/LicenseInfo$ + + +import collections +import fnmatch +import itertools +import os +import sys +import subprocess +import bz2 + +def usage(): + print >>sys.stderr, "usage: %s viewer_dir viewer_exes libs_suffix dump_syms_tool viewer_symbol_file" % sys.argv[0] + +def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file): + print "generate_breakpad_symbols: %s" % str((viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file)) + + def matches(f): + return f in viewer_exes or fnmatch.fnmatch(f, libs_suffix) + + def list_files(): + for (dirname, subdirs, filenames) in os.walk(viewer_dir): + #print "scanning '%s' for modules..." % dirname + for f in itertools.ifilter(matches, filenames): + yield os.path.join(dirname, f) + + def dump_module(m): + print "dumping module '%s' with '%s'..." % (m, dump_syms_tool) + child = subprocess.Popen([dump_syms_tool, m] , stdout=subprocess.PIPE) + out, err = child.communicate() + return out + + out = bz2.BZ2File(viewer_symbol_file, 'w') + + for symbols in map(dump_module, list_files()): + out.writelines(symbols) + + out.close() + + return 0 + +if __name__ == "__main__": + if len(sys.argv) != 6: + usage() + sys.exit(1) + sys.exit(main(*sys.argv[1:])) + -- cgit v1.2.3 From 3729983908d365843bf8cf85178b61f719448374 Mon Sep 17 00:00:00 2001 From: brad kittenbrink Date: Thu, 27 May 2010 19:13:51 -0700 Subject: WIP VPLAT-248 cleanups for google_breakpad symbol file generation. --- indra/newview/generate_breakpad_symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/newview/generate_breakpad_symbols.py') diff --git a/indra/newview/generate_breakpad_symbols.py b/indra/newview/generate_breakpad_symbols.py index 884dd06f84..f281846918 100644 --- a/indra/newview/generate_breakpad_symbols.py +++ b/indra/newview/generate_breakpad_symbols.py @@ -44,7 +44,7 @@ def usage(): print >>sys.stderr, "usage: %s viewer_dir viewer_exes libs_suffix dump_syms_tool viewer_symbol_file" % sys.argv[0] def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file): - print "generate_breakpad_symbols: %s" % str((viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file)) + # print "generate_breakpad_symbols: %s" % str((viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file)) def matches(f): return f in viewer_exes or fnmatch.fnmatch(f, libs_suffix) -- cgit v1.2.3 From 29c068afc3779f58fa39f16ef9ddd7a9da29988b Mon Sep 17 00:00:00 2001 From: brad kittenbrink Date: Wed, 2 Jun 2010 18:55:43 -0700 Subject: Finished task VPLAT-296: symbol files are packed in a tarball now instead of concatenated. --- indra/newview/generate_breakpad_symbols.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'indra/newview/generate_breakpad_symbols.py') diff --git a/indra/newview/generate_breakpad_symbols.py b/indra/newview/generate_breakpad_symbols.py index f281846918..3a5f34a5c3 100644 --- a/indra/newview/generate_breakpad_symbols.py +++ b/indra/newview/generate_breakpad_symbols.py @@ -38,7 +38,8 @@ import itertools import os import sys import subprocess -import bz2 +import tarfile +import StringIO def usage(): print >>sys.stderr, "usage: %s viewer_dir viewer_exes libs_suffix dump_syms_tool viewer_symbol_file" % sys.argv[0] @@ -59,12 +60,26 @@ def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_fil print "dumping module '%s' with '%s'..." % (m, dump_syms_tool) child = subprocess.Popen([dump_syms_tool, m] , stdout=subprocess.PIPE) out, err = child.communicate() - return out + return (m,child.returncode, out, err) - out = bz2.BZ2File(viewer_symbol_file, 'w') + out = tarfile.open(viewer_symbol_file, 'w:bz2') - for symbols in map(dump_module, list_files()): - out.writelines(symbols) + for (filename,status,symbols,err) in itertools.imap(dump_module, list_files()): + if status == 0: + module_line = symbols[:symbols.index('\n')] + module_line = module_line.split() + hash_id = module_line[3] + module = ' '.join(module_line[4:]) + if sys.platform in ['win32', 'cygwin']: + mod_name = module[:module.rindex('.pdb')] + else: + mod_name = module + symbolfile = StringIO.StringIO(symbols) + info = tarfile.TarInfo("%(module)s/%(hash_id)s/%(mod_name)s.sym" % dict(module=module, hash_id=hash_id, mod_name=mod_name)) + info.size = symbolfile.len + out.addfile(info, symbolfile) + else: + print >>sys.stderr, "warning: failed to dump symbols for '%s': %s" % (filename, err) out.close() -- cgit v1.2.3 From 792b9d38cbf5fc5d03339c255a13912fa85bee51 Mon Sep 17 00:00:00 2001 From: brad kittenbrink Date: Thu, 3 Jun 2010 13:55:03 -0700 Subject: Added extra error checking to generate_breakpad_symbols.py so the failure in copy_l_viewer_manifest (fixed in changeset 4ae8723c7d33) won't be so silent in the future. --- indra/newview/generate_breakpad_symbols.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'indra/newview/generate_breakpad_symbols.py') diff --git a/indra/newview/generate_breakpad_symbols.py b/indra/newview/generate_breakpad_symbols.py index 3a5f34a5c3..21db9ce152 100644 --- a/indra/newview/generate_breakpad_symbols.py +++ b/indra/newview/generate_breakpad_symbols.py @@ -37,6 +37,7 @@ import fnmatch import itertools import os import sys +import shlex import subprocess import tarfile import StringIO @@ -44,11 +45,27 @@ import StringIO def usage(): print >>sys.stderr, "usage: %s viewer_dir viewer_exes libs_suffix dump_syms_tool viewer_symbol_file" % sys.argv[0] +class MissingModuleError(Exception): + def __init__(self, modules): + Exception.__init__(self, "Failed to find required modules: %r" % modules) + self.modules = modules + def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file): # print "generate_breakpad_symbols: %s" % str((viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file)) + # split up list of viewer_exes + # "'Second Life' SLPlugin" becomes ['Second Life', 'SLPlugin'] + viewer_exes = shlex.split(viewer_exes) + + found_required = dict() + for required in viewer_exes: + found_required[required] = False + def matches(f): - return f in viewer_exes or fnmatch.fnmatch(f, libs_suffix) + if f in viewer_exes: + found_required[f] = True + return True + return fnmatch.fnmatch(f, libs_suffix) def list_files(): for (dirname, subdirs, filenames) in os.walk(viewer_dir): @@ -81,6 +98,12 @@ def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_fil else: print >>sys.stderr, "warning: failed to dump symbols for '%s': %s" % (filename, err) + missing_modules = [m for (m,_) in + itertools.ifilter(lambda (k,v): not v, found_required.iteritems()) + ] + if missing_modules: + raise MissingModuleError(missing_modules) + out.close() return 0 -- cgit v1.2.3 From edd224dc7b08ad72c0f36c5b0f24912fdf66ab1c Mon Sep 17 00:00:00 2001 From: brad kittenbrink Date: Fri, 4 Jun 2010 14:13:33 -0700 Subject: Yet more error checking for linux symbol file generation. --- indra/newview/generate_breakpad_symbols.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'indra/newview/generate_breakpad_symbols.py') diff --git a/indra/newview/generate_breakpad_symbols.py b/indra/newview/generate_breakpad_symbols.py index 21db9ce152..d94301d62e 100644 --- a/indra/newview/generate_breakpad_symbols.py +++ b/indra/newview/generate_breakpad_symbols.py @@ -35,6 +35,7 @@ import collections import fnmatch import itertools +import operator import os import sys import shlex @@ -51,15 +52,13 @@ class MissingModuleError(Exception): self.modules = modules def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file): - # print "generate_breakpad_symbols: %s" % str((viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file)) + print "generate_breakpad_symbols run with args: %s" % str((viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_file)) # split up list of viewer_exes # "'Second Life' SLPlugin" becomes ['Second Life', 'SLPlugin'] viewer_exes = shlex.split(viewer_exes) - found_required = dict() - for required in viewer_exes: - found_required[required] = False + found_required = dict([(module, False) for module in viewer_exes]) def matches(f): if f in viewer_exes: @@ -98,13 +97,28 @@ def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_fil else: print >>sys.stderr, "warning: failed to dump symbols for '%s': %s" % (filename, err) + out.close() + missing_modules = [m for (m,_) in itertools.ifilter(lambda (k,v): not v, found_required.iteritems()) ] if missing_modules: + print >> sys.stderr, "failed to generate %s" % viewer_symbol_file + os.remove(viewer_symbol_file) raise MissingModuleError(missing_modules) - out.close() + symbols = tarfile.open(viewer_symbol_file, 'r:bz2') + tarfile_members = symbols.getnames() + def match_module_basename(m): + return os.path.splitext(required_module)[0] == os.path.splitext(os.path.basename(m))[0] + for required_module in viewer_exes: + # there must be at least one .sym file in tarfile_members that matches each required module (ignoring file extensions) + if not reduce(operator.or_, itertools.imap(match_module_basename, tarfile_members)): + print >> sys.stderr, "failed to find required %s in generated %s" % (required_module, viewer_symbol_file) + os.remove(viewer_symbol_file) + raise MissingModuleError([required_module]) + + print "successfully generated %s including required modules '%s'" % (viewer_symbol_file, viewer_exes) return 0 -- cgit v1.2.3 From 4bbca7c086724da1af4f8cb6b22e6de8320a0b1b Mon Sep 17 00:00:00 2001 From: brad kittenbrink Date: Tue, 8 Jun 2010 11:01:44 -0700 Subject: Fix for false positive in error checking symbol file generation on windows (MissingModuleError). --- indra/newview/generate_breakpad_symbols.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'indra/newview/generate_breakpad_symbols.py') diff --git a/indra/newview/generate_breakpad_symbols.py b/indra/newview/generate_breakpad_symbols.py index d94301d62e..1f42004bb7 100644 --- a/indra/newview/generate_breakpad_symbols.py +++ b/indra/newview/generate_breakpad_symbols.py @@ -109,12 +109,17 @@ def main(viewer_dir, viewer_exes, libs_suffix, dump_syms_tool, viewer_symbol_fil symbols = tarfile.open(viewer_symbol_file, 'r:bz2') tarfile_members = symbols.getnames() - def match_module_basename(m): - return os.path.splitext(required_module)[0] == os.path.splitext(os.path.basename(m))[0] + symbols.close() + for required_module in viewer_exes: - # there must be at least one .sym file in tarfile_members that matches each required module (ignoring file extensions) + def match_module_basename(m): + return os.path.splitext(required_module)[0].lower() \ + == os.path.splitext(os.path.basename(m))[0].lower() + # there must be at least one .sym file in tarfile_members that matches + # each required module (ignoring file extensions) if not reduce(operator.or_, itertools.imap(match_module_basename, tarfile_members)): - print >> sys.stderr, "failed to find required %s in generated %s" % (required_module, viewer_symbol_file) + print >> sys.stderr, "failed to find required %s in generated %s" \ + % (required_module, viewer_symbol_file) os.remove(viewer_symbol_file) raise MissingModuleError([required_module]) -- cgit v1.2.3