From 277ee6830f68030ece6f469a86a49009a9c1450a Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 12 Sep 2024 12:28:05 -0400 Subject: Add a JSON frame profile stats file pretty-printer script. (cherry picked from commit ab3083819793a30911354670a7929b0d3f7c104c) --- scripts/perf/profile_pretty.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 scripts/perf/profile_pretty.py (limited to 'scripts/perf') diff --git a/scripts/perf/profile_pretty.py b/scripts/perf/profile_pretty.py new file mode 100644 index 0000000000..ca52fe366a --- /dev/null +++ b/scripts/perf/profile_pretty.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""\ +@file profile_pretty.py +@author Nat Goodspeed +@date 2024-09-12 +@brief Pretty-print a JSON file from Develop -> Render Tests -> Frame Profile + +$LicenseInfo:firstyear=2024&license=viewerlgpl$ +Copyright (c) 2024, Linden Research, Inc. +$/LicenseInfo$ +""" + +import logsdir +import json +from pathlib import Path +import sys + +class Error(Exception): + pass + +def pretty(path=None): + if not path: + logs = logsdir.logsdir() + profiles = Path(logs).glob('profile.*.json') + sort = [(p.stat().st_mtime, p) for p in profiles] + sort.sort(reverse=True) + try: + path = sort[0][1] + except IndexError: + raise Error(f'No profile.*.json files in {logs}') + # print path to sys.stderr in case user is redirecting stdout + print(path, file=sys.stderr) + + with open(path) as inf: + data = json.load(inf) + json.dump(data, sys.stdout, indent=4) + +def main(*raw_args): + from argparse import ArgumentParser + parser = ArgumentParser(description=""" +%(prog)s pretty-prints a JSON file from Develop -> Render Tests -> Frame Profile. +The file produced by the viewer is a single dense line of JSON. +""") + parser.add_argument('path', nargs='?', + help="""profile filename to pretty-print (default is most recent)""") + + args = parser.parse_args(raw_args) + pretty(args.path) + +if __name__ == "__main__": + try: + sys.exit(main(*sys.argv[1:])) + except (Error, OSError, json.JSONDecodeError) as err: + sys.exit(str(err)) -- cgit v1.2.3 From ee1b0061c36c36ab019438c2a722696801de04f9 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 12 Sep 2024 13:43:36 -0400 Subject: Add script to convert frame profile JSON file to CSV. Also slightly refactor profile_pretty.py. (cherry picked from commit d60b1f92213ace6a8ab6a4a60cb01a43f45d3955) --- scripts/perf/profile_csv.py | 72 ++++++++++++++++++++++++++++++++++++++++++ scripts/perf/profile_pretty.py | 26 +++++++-------- 2 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 scripts/perf/profile_csv.py (limited to 'scripts/perf') diff --git a/scripts/perf/profile_csv.py b/scripts/perf/profile_csv.py new file mode 100644 index 0000000000..273e3b7434 --- /dev/null +++ b/scripts/perf/profile_csv.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""\ +@file profile_csv.py +@author Nat Goodspeed +@date 2024-09-12 +@brief Convert a JSON file from Develop -> Render Tests -> Frame Profile to CSV + +$LicenseInfo:firstyear=2024&license=viewerlgpl$ +Copyright (c) 2024, Linden Research, Inc. +$/LicenseInfo$ +""" + +import logsdir +import json +from pathlib import Path +import sys + +class Error(Exception): + pass + +def convert(path, totals=True, unused=True, file=sys.stdout): + with open(path) as inf: + data = json.load(inf) + print('"name", "file1", "file2", "time", "binds", "samples", "triangles"', file=file) + + if totals: + t = data['totals'] + print(f'"totals", "", "", {t["time"]}, {t["binds"]}, {t["samples"]}, {t["triangles"]}', + file=file) + + for sh in data['shaders']: + print(f'"{sh["name"]}", "{sh["files"][0]}", "{sh["files"][1]}", ' + f'{sh["time"]}, {sh["binds"]}, {sh["samples"]}, {sh["triangles"]}', file=file) + + if unused: + for u in data['unused']: + print(f'"{u}", "", "", 0, 0, 0, 0', file=file) + +def main(*raw_args): + from argparse import ArgumentParser + parser = ArgumentParser(description=""" +%(prog)s converts a JSON file from Develop -> Render Tests -> Frame Profile to +a more-or-less equivalent CSV file. It expands the totals stats and unused +shaders list to full shaders lines. +""") + parser.add_argument('-t', '--totals', action='store_false', default=True, + help="""omit totals from CSV file""") + parser.add_argument('-u', '--unused', action='store_false', default=True, + help="""omit unused shaders from CSV file""") + parser.add_argument('path', nargs='?', + help="""profile filename to convert (default is most recent)""") + + args = parser.parse_args(raw_args) + if not args.path: + logs = logsdir.logsdir() + profiles = Path(logs).glob('profile.*.json') + sort = [(p.stat().st_mtime, p) for p in profiles] + sort.sort(reverse=True) + try: + args.path = sort[0][1] + except IndexError: + raise Error(f'No profile.*.json files in {logs}') + # print path to sys.stderr in case user is redirecting stdout + print(args.path, file=sys.stderr) + + convert(args.path, totals=args.totals, unused=args.unused) + +if __name__ == "__main__": + try: + sys.exit(main(*sys.argv[1:])) + except (Error, OSError, json.JSONDecodeError) as err: + sys.exit(str(err)) diff --git a/scripts/perf/profile_pretty.py b/scripts/perf/profile_pretty.py index ca52fe366a..15b6efd94d 100644 --- a/scripts/perf/profile_pretty.py +++ b/scripts/perf/profile_pretty.py @@ -18,19 +18,7 @@ import sys class Error(Exception): pass -def pretty(path=None): - if not path: - logs = logsdir.logsdir() - profiles = Path(logs).glob('profile.*.json') - sort = [(p.stat().st_mtime, p) for p in profiles] - sort.sort(reverse=True) - try: - path = sort[0][1] - except IndexError: - raise Error(f'No profile.*.json files in {logs}') - # print path to sys.stderr in case user is redirecting stdout - print(path, file=sys.stderr) - +def pretty(path): with open(path) as inf: data = json.load(inf) json.dump(data, sys.stdout, indent=4) @@ -45,6 +33,18 @@ The file produced by the viewer is a single dense line of JSON. help="""profile filename to pretty-print (default is most recent)""") args = parser.parse_args(raw_args) + if not args.path: + logs = logsdir.logsdir() + profiles = Path(logs).glob('profile.*.json') + sort = [(p.stat().st_mtime, p) for p in profiles] + sort.sort(reverse=True) + try: + args.path = sort[0][1] + except IndexError: + raise Error(f'No profile.*.json files in {logs}') + # print path to sys.stderr in case user is redirecting stdout + print(args.path, file=sys.stderr) + pretty(args.path) if __name__ == "__main__": -- cgit v1.2.3 From 705ec153c5ee3f6d1781647c1bbbfcd7c398c987 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 13 Sep 2024 16:32:04 -0400 Subject: Add script to compare a Frame Profile JSON stats file vs. baseline. Extract `latest_file()` logic replicated in profile_pretty.py and profile_csv.py out to logsdir.py, and use for new profile_cmp.py. (cherry picked from commit 439cfc97a81f221daaf8ba13aa5daa87e8511047) --- scripts/perf/logsdir.py | 46 ++++++++++++++++++ scripts/perf/profile_cmp.py | 104 +++++++++++++++++++++++++++++++++++++++++ scripts/perf/profile_csv.py | 24 +++------- scripts/perf/profile_pretty.py | 22 ++------- 4 files changed, 160 insertions(+), 36 deletions(-) create mode 100644 scripts/perf/logsdir.py create mode 100644 scripts/perf/profile_cmp.py (limited to 'scripts/perf') diff --git a/scripts/perf/logsdir.py b/scripts/perf/logsdir.py new file mode 100644 index 0000000000..c8b498cf78 --- /dev/null +++ b/scripts/perf/logsdir.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""\ +@file logsdir.py +@author Nat Goodspeed +@date 2024-09-12 +@brief Locate the Second Life logs directory for the current user on the + current platform. + +$LicenseInfo:firstyear=2024&license=viewerlgpl$ +Copyright (c) 2024, Linden Research, Inc. +$/LicenseInfo$ +""" + +import os +from pathlib import Path +import platform + +class Error(Exception): + pass + +# logic used by SLVersionChecker +def logsdir(): + app = 'SecondLife' + system = platform.system() + if (system == 'Darwin'): + base_dir = os.path.join(os.path.expanduser('~'), + 'Library','Application Support',app) + elif (system == 'Linux'): + base_dir = os.path.join(os.path.expanduser('~'), + '.' + app.lower()) + elif (system == 'Windows'): + appdata = os.getenv('APPDATA') + base_dir = os.path.join(appdata, app) + else: + raise ValueError("Unsupported platform '%s'" % system) + + return os.path.join(base_dir, 'logs') + +def latest_file(dirpath, pattern): + files = Path(dirpath).glob(pattern) + sort = [(p.stat().st_mtime, p) for p in files if p.is_file()] + sort.sort(reverse=True) + try: + return sort[0][1] + except IndexError: + raise Error(f'No {pattern} files in {dirpath}') diff --git a/scripts/perf/profile_cmp.py b/scripts/perf/profile_cmp.py new file mode 100644 index 0000000000..9dbfa3145b --- /dev/null +++ b/scripts/perf/profile_cmp.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""\ +@file profile_cmp.py +@author Nat Goodspeed +@date 2024-09-13 +@brief Compare a frame profile stats file with a similar baseline file. + +$LicenseInfo:firstyear=2024&license=viewerlgpl$ +Copyright (c) 2024, Linden Research, Inc. +$/LicenseInfo$ +""" + +from datetime import datetime +import json +from logsdir import Error, latest_file, logsdir +from pathlib import Path +import sys + +# variance that's ignorable +DEFAULT_EPSILON = 0.03 # 3% + +def compare(baseline, test, epsilon=DEFAULT_EPSILON): + if Path(baseline).samefile(test): + print(f'{baseline} same as\n{test}\nAnalysis moot.') + return + + with open(baseline) as inf: + bdata = json.load(inf) + with open(test) as inf: + tdata = json.load(inf) + print(f'baseline {baseline}\ntestfile {test}') + + for k, tv in tdata['context'].items(): + bv = bdata['context'].get(k) + if bv != tv: + print(f'baseline {k}={bv} vs.\ntestfile {k}={tv}') + + btime = bdata['context'].get('time') + ttime = tdata['context'].get('time') + if btime and ttime: + print('testfile newer by', + datetime.fromisoformat(ttime) - datetime.fromisoformat(btime)) + + # The following ignores totals and unused shaders, except to the extent + # that some shaders were used in the baseline but not in the recent test + # or vice-versa. While the viewer considers that a shader has been used if + # 'binds' is nonzero, we exclude any whose 'time' is zero to avoid zero + # division. + bshaders = {s['name']: s for s in bdata['shaders'] if s['time'] and s['samples']} + tshaders = {s['name']: s for s in tdata['shaders'] if s['time']} + + bothshaders = set(bshaders).intersection(tshaders) + deltas = [] + for shader in bothshaders: + bshader = bshaders[shader] + tshader = tshaders[shader] + bthruput = bshader['samples']/bshader['time'] + tthruput = tshader['samples']/tshader['time'] + delta = (tthruput - bthruput)/bthruput + if abs(delta) > epsilon: + deltas.append((delta, shader, bthruput, tthruput)) + + # descending order of performance gain + deltas.sort(reverse=True) + print(f'{len(deltas)} shaders showed nontrivial performance differences ' + '(millon samples/sec):') + namelen = max(len(s[1]) for s in deltas) if deltas else 0 + for delta, shader, bthruput, tthruput in deltas: + print(f' {shader.rjust(namelen)} {delta*100:6.1f}% ' + f'{bthruput/1000000:8.2f} -> {tthruput/1000000:8.2f}') + + tunused = set(bshaders).difference(tshaders) + print(f'{len(tunused)} baseline shaders not used in test:') + for s in tunused: + print(f' {s}') + bunused = set(tshaders).difference(bshaders) + print(f'{len(bunused)} shaders newly used in test:') + for s in bunused: + print(f' {s}') + +def main(*raw_args): + from argparse import ArgumentParser + parser = ArgumentParser(description=""" +%(prog)s compares a baseline JSON file from Develop -> Render Tests -> Frame +Profile to another such file from a more recent test. It identifies shaders +that have gained and lost in throughput. +""") + parser.add_argument('-e', '--epsilon', type=float, default=int(DEFAULT_EPSILON*100), + help="""percent variance considered ignorable (default %(default)s%%)""") + parser.add_argument('baseline', + help="""baseline profile filename to compare against""") + parser.add_argument('test', nargs='?', + help="""test profile filename to compare + (default is most recent)""") + args = parser.parse_args(raw_args) + compare(args.baseline, + args.test or latest_file(logsdir(), 'profile.*.json'), + epsilon=(args.epsilon / 100.)) + +if __name__ == "__main__": + try: + sys.exit(main(*sys.argv[1:])) + except (Error, OSError, json.JSONDecodeError) as err: + sys.exit(str(err)) diff --git a/scripts/perf/profile_csv.py b/scripts/perf/profile_csv.py index 273e3b7434..7a6b2b338e 100644 --- a/scripts/perf/profile_csv.py +++ b/scripts/perf/profile_csv.py @@ -10,17 +10,16 @@ Copyright (c) 2024, Linden Research, Inc. $/LicenseInfo$ """ -import logsdir import json -from pathlib import Path +from logsdir import Error, latest_file, logsdir import sys -class Error(Exception): - pass - def convert(path, totals=True, unused=True, file=sys.stdout): with open(path) as inf: data = json.load(inf) + # print path to sys.stderr in case user is redirecting stdout + print(path, file=sys.stderr) + print('"name", "file1", "file2", "time", "binds", "samples", "triangles"', file=file) if totals: @@ -51,19 +50,8 @@ shaders list to full shaders lines. help="""profile filename to convert (default is most recent)""") args = parser.parse_args(raw_args) - if not args.path: - logs = logsdir.logsdir() - profiles = Path(logs).glob('profile.*.json') - sort = [(p.stat().st_mtime, p) for p in profiles] - sort.sort(reverse=True) - try: - args.path = sort[0][1] - except IndexError: - raise Error(f'No profile.*.json files in {logs}') - # print path to sys.stderr in case user is redirecting stdout - print(args.path, file=sys.stderr) - - convert(args.path, totals=args.totals, unused=args.unused) + convert(args.path or latest_file(logsdir(), 'profile.*.json'), + totals=args.totals, unused=args.unused) if __name__ == "__main__": try: diff --git a/scripts/perf/profile_pretty.py b/scripts/perf/profile_pretty.py index 15b6efd94d..405b14b373 100644 --- a/scripts/perf/profile_pretty.py +++ b/scripts/perf/profile_pretty.py @@ -10,17 +10,15 @@ Copyright (c) 2024, Linden Research, Inc. $/LicenseInfo$ """ -import logsdir import json -from pathlib import Path +from logsdir import Error, latest_file, logsdir import sys -class Error(Exception): - pass - def pretty(path): with open(path) as inf: data = json.load(inf) + # print path to sys.stderr in case user is redirecting stdout + print(path, file=sys.stderr) json.dump(data, sys.stdout, indent=4) def main(*raw_args): @@ -33,19 +31,7 @@ The file produced by the viewer is a single dense line of JSON. help="""profile filename to pretty-print (default is most recent)""") args = parser.parse_args(raw_args) - if not args.path: - logs = logsdir.logsdir() - profiles = Path(logs).glob('profile.*.json') - sort = [(p.stat().st_mtime, p) for p in profiles] - sort.sort(reverse=True) - try: - args.path = sort[0][1] - except IndexError: - raise Error(f'No profile.*.json files in {logs}') - # print path to sys.stderr in case user is redirecting stdout - print(args.path, file=sys.stderr) - - pretty(args.path) + pretty(args.path or latest_file(logsdir(), 'profile.*.json')) if __name__ == "__main__": try: -- cgit v1.2.3 From be40936881a747893d03c5c003914efb3867ccd1 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 19 Sep 2024 12:40:56 -0400 Subject: trailing spaces from other branches --- scripts/perf/logsdir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/perf') diff --git a/scripts/perf/logsdir.py b/scripts/perf/logsdir.py index c8b498cf78..5ab45a28b6 100644 --- a/scripts/perf/logsdir.py +++ b/scripts/perf/logsdir.py @@ -25,7 +25,7 @@ def logsdir(): if (system == 'Darwin'): base_dir = os.path.join(os.path.expanduser('~'), 'Library','Application Support',app) - elif (system == 'Linux'): + elif (system == 'Linux'): base_dir = os.path.join(os.path.expanduser('~'), '.' + app.lower()) elif (system == 'Windows'): -- cgit v1.2.3 From 4fc8f2ed98b5ea80f763e1b6910b7bc3843ee7e2 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 20 Sep 2024 17:12:00 -0400 Subject: Reverse the sort order for profile_cmp.py to put the biggest performance hits at the top of the list. --- scripts/perf/profile_cmp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts/perf') diff --git a/scripts/perf/profile_cmp.py b/scripts/perf/profile_cmp.py index 9dbfa3145b..34281b8d01 100644 --- a/scripts/perf/profile_cmp.py +++ b/scripts/perf/profile_cmp.py @@ -60,8 +60,9 @@ def compare(baseline, test, epsilon=DEFAULT_EPSILON): if abs(delta) > epsilon: deltas.append((delta, shader, bthruput, tthruput)) - # descending order of performance gain - deltas.sort(reverse=True) + # ascending order of performance gain: put the most egregious performance + # hits at the top of the list + deltas.sort() print(f'{len(deltas)} shaders showed nontrivial performance differences ' '(millon samples/sec):') namelen = max(len(s[1]) for s in deltas) if deltas else 0 -- cgit v1.2.3