Makefile.in | 11 bin/symbolstore.py | 644 +++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/symstore.sh | 85 ++++++ 3 files changed, 740 insertions(+)
New commits: commit 454ba146fed00a447a18c71190dc2904576f7a2e Author: Samuel Mehrbrodt <[email protected]> AuthorDate: Fri Jan 27 16:57:49 2017 +0100 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:41:32 2018 +0100 Symstore: Also add .exe and .dlls to symstore These are needed when analyzing the minidump. Change-Id: Ife296c298e3b2f1ca8a47dcbaaf1947e6aefdc81 Reviewed-on: https://gerrit.libreoffice.org/33631 Reviewed-by: Thorsten Behrens <[email protected]> Tested-by: Thorsten Behrens <[email protected]> (cherry picked from commit 8a0416be440180d0a6cedd449307f6a9bde22eaa) diff --git a/bin/symstore.sh b/bin/symstore.sh index 56260c0b3906..b368eb3e6715 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -6,13 +6,17 @@ add_pdb() type=$2 list=$3 for file in $(find "${INSTDIR}/" -name "*.${extension}"); do + # store dll/exe itself (needed for minidumps) + if [ -f "$file" ]; then + cygpath -w "$file" >> "$list" + fi + # store pdb file filename=$(basename "$file" ".${extension}") pdb="${WORKDIR}/LinkTarget/${type}/${filename}.pdb" if [ -f "$pdb" ]; then cygpath -w "$pdb" >> "$list" fi done - } # check preconditions commit dec2d0066e5df2d3967c754b1590e20e746d5403 Author: Thorsten Behrens <[email protected]> AuthorDate: Sun Jan 15 20:51:22 2017 +0100 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:41:23 2018 +0100 shellcheck: cleanup symstore.sh Change-Id: I8eb53c06892934c21d55b69d12e3c8ff09696295 (cherry picked from commit a1784cc9a73c5f43d084f31ac37b78b3642d46d3) diff --git a/bin/symstore.sh b/bin/symstore.sh index ca5bd306a842..56260c0b3906 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -5,22 +5,22 @@ add_pdb() extension=$1 type=$2 list=$3 - for file in `find ${INSTDIR}/ -name *.${extension}`; do - filename=`basename $file .${extension}` - pdb=`echo ${WORKDIR}/LinkTarget/${type}/${filename}.pdb` + for file in $(find "${INSTDIR}/" -name "*.${extension}"); do + filename=$(basename "$file" ".${extension}") + pdb="${WORKDIR}/LinkTarget/${type}/${filename}.pdb" if [ -f "$pdb" ]; then - echo `cygpath -w $pdb` >>$list + cygpath -w "$pdb" >> "$list" fi done } # check preconditions -if [ -z ${INSTDIR} -o -z ${WORKDIR} ]; then +if [ -z "${INSTDIR}" ] || [ -z "${WORKDIR}" ]; then echo "INSTDIR or WORKDIR not set - script expects calling inside buildenv" exit 1 fi -if [ ! -d ${INSTDIR} -o ! -d ${WORKDIR} ]; then +if [ ! -d "${INSTDIR}" ] || [ ! -d "${WORKDIR}" ]; then echo "INSTDIR or WORKDIR not present - script expects calling after full build" exit 1 fi @@ -47,35 +47,35 @@ do case "$1" in -k|--keep) MAX_KEEP="$2"; shift 2;; -p|--path) SYM_PATH="$2"; shift 2;; - -h|--help) printf "$USAGE"; exit 0; shift;; - -*) echo "$USAGE" >&2; exit 1;; + -h|--help) echo "${USAGE}"; exit 0; shift;; + -*) echo "${USAGE}" >&2; exit 1;; *) break;; esac done if [ $# -gt 0 ]; then - echo $usage >&2 + echo "${USAGE}" >&2 exit 1 fi # populate symbol store from here -TMPFILE=`mktemp` || exit 1 -trap "{ rm -f $TMPFILE; }" EXIT +TMPFILE=$(mktemp) || exit 1 +trap '{ rm -f ${TMPFILE}; }' EXIT # add dlls and executables -add_pdb dll Library $TMPFILE -add_pdb exe Executable $TMPFILE +add_pdb dll Library "${TMPFILE}" +add_pdb exe Executable "${TMPFILE}" # stick all of it into symbol store -symstore.exe add /compress /f @${TMPFILE} /s $SYM_PATH /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" -rm -f $TMPFILE +symstore.exe add /compress /f "@$(cygpath -w "${TMPFILE}")" /s "$(cygpath -w "${SYM_PATH}")" /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" +rm -f "${TMPFILE}" # Cleanup symstore, older revisions will be removed. Unless the # .dll/.exe changes, the .pdb should be shared, so with incremental # tinderbox several revisions should not be that space-demanding. -if [ $MAX_KEEP -gt 0 -a -d ${SYM_PATH}/000Admin ]; then - to_remove=`ls -1 ${SYM_PATH}/000Admin | grep -v '\.txt' | grep -v '\.deleted' | sort | head -n -${MAX_KEEP}` +if [ "${MAX_KEEP}" -gt 0 ] && [ -d "${SYM_PATH}/000Admin" ]; then + to_remove=$(ls -1 "${SYM_PATH}/000Admin" | grep -v '\.txt' | grep -v '\.deleted' | sort | head -n "-${MAX_KEEP}") for revision in $to_remove; do - symstore.exe del /i ${revision} /s `cygpath -w $SYM_PATH` + symstore.exe del /i "${revision}" /s "$(cygpath -w "${SYM_PATH}")" done fi commit 9959ec3d922789fca35d4d1c4f7429947ba91830 Author: Thorsten Behrens <[email protected]> AuthorDate: Sun Jan 15 11:50:27 2017 +0100 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:41:16 2018 +0100 gbuild: populate local symstore on Windows Script based on Lubos' tb master script from http://nabble.documentfoundation.org/Daily-Win32-debug-builds-td4067279.html Change-Id: I7f3247367a63078881f3cf51cf3e2cad59ad67b5 Reviewed-on: https://gerrit.libreoffice.org/33088 Reviewed-by: Thorsten Behrens <[email protected]> Tested-by: Thorsten Behrens <[email protected]> (cherry picked from commit 17e9a5bf94eb08f88f8c78c9982dd0ce48a5e2d9) diff --git a/Makefile.in b/Makefile.in index 940c56eae49e..3c3a77db8b94 100644 --- a/Makefile.in +++ b/Makefile.in @@ -325,6 +325,7 @@ symbols: mkdir -p $(WORKDIR)/symbols/ ifeq ($(OS),WNT) $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/windows/binaries/dump_syms.exe $(WORKDIR)/symbols/ $(INSTDIR)/program/* + $(SRCDIR)/bin/symstore.sh else $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/linux/dump_syms/dump_syms $(WORKDIR)/symbols/ $(INSTDIR)/program/* endif diff --git a/bin/symstore.sh b/bin/symstore.sh new file mode 100755 index 000000000000..ca5bd306a842 --- /dev/null +++ b/bin/symstore.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +add_pdb() +{ + extension=$1 + type=$2 + list=$3 + for file in `find ${INSTDIR}/ -name *.${extension}`; do + filename=`basename $file .${extension}` + pdb=`echo ${WORKDIR}/LinkTarget/${type}/${filename}.pdb` + if [ -f "$pdb" ]; then + echo `cygpath -w $pdb` >>$list + fi + done + +} + +# check preconditions +if [ -z ${INSTDIR} -o -z ${WORKDIR} ]; then + echo "INSTDIR or WORKDIR not set - script expects calling inside buildenv" + exit 1 +fi +if [ ! -d ${INSTDIR} -o ! -d ${WORKDIR} ]; then + echo "INSTDIR or WORKDIR not present - script expects calling after full build" + exit 1 +fi +which symstore.exe > /dev/null 2>&1 || { + echo "symstore.exe is expected in the PATH" + exit 1 +} + +# defaults +MAX_KEEP=5 +SYM_PATH=${WORKDIR}/symstore + +USAGE="Usage: $0 [-h|-k <keep_num_versions>|-p <symbol_store_path>] + -h: this cruft + -k <int>: keep this number of old symbol versions around + (default: ${MAX_KEEP}. Set to 0 for unlimited) + -p <path>: specify full path to symbol store tree +If no path is specified, defaults to ${SYM_PATH}. +" + +# process args +while : +do + case "$1" in + -k|--keep) MAX_KEEP="$2"; shift 2;; + -p|--path) SYM_PATH="$2"; shift 2;; + -h|--help) printf "$USAGE"; exit 0; shift;; + -*) echo "$USAGE" >&2; exit 1;; + *) break;; + esac +done + +if [ $# -gt 0 ]; then + echo $usage >&2 + exit 1 +fi + +# populate symbol store from here +TMPFILE=`mktemp` || exit 1 +trap "{ rm -f $TMPFILE; }" EXIT + +# add dlls and executables +add_pdb dll Library $TMPFILE +add_pdb exe Executable $TMPFILE + +# stick all of it into symbol store +symstore.exe add /compress /f @${TMPFILE} /s $SYM_PATH /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" +rm -f $TMPFILE + +# Cleanup symstore, older revisions will be removed. Unless the +# .dll/.exe changes, the .pdb should be shared, so with incremental +# tinderbox several revisions should not be that space-demanding. +if [ $MAX_KEEP -gt 0 -a -d ${SYM_PATH}/000Admin ]; then + to_remove=`ls -1 ${SYM_PATH}/000Admin | grep -v '\.txt' | grep -v '\.deleted' | sort | head -n -${MAX_KEEP}` + for revision in $to_remove; do + symstore.exe del /i ${revision} /s `cygpath -w $SYM_PATH` + done +fi commit ebc4d6fdefbd4ead9f73845c555b55b8a8ce4d0c Author: Markus Mohrhard <[email protected]> AuthorDate: Tue May 24 09:12:41 2016 +0200 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:33:29 2018 +0100 make the symbol generation also work correctly on windows Change-Id: I2fc5e8fb2535ac076f045435c828126a52ea5bbd (cherry picked from commit 5314e2a2e0d06805a4517f2e947611b1f308f9bd) diff --git a/Makefile.in b/Makefile.in index 5fe8d52668eb..940c56eae49e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -323,7 +323,12 @@ findunusedheaders: symbols: rm -fr $(WORKDIR)/symbols/ mkdir -p $(WORKDIR)/symbols/ +ifeq ($(OS),WNT) + $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/windows/binaries/dump_syms.exe $(WORKDIR)/symbols/ $(INSTDIR)/program/* +else $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/linux/dump_syms/dump_syms $(WORKDIR)/symbols/ $(INSTDIR)/program/* +endif + cd $(WORKDIR)/symbols/ && zip -r $(WORKDIR)/symbols.zip * dump-deps: commit a97a42fd59d33f16f5d32772f1063b2d035be4d8 Author: Markus Mohrhard <[email protected]> AuthorDate: Tue May 24 00:33:32 2016 +0200 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:33:10 2018 +0100 also create the symbols.zip file Change-Id: I4c96dbecd90de83fe9ac76a93ec0520ffb6932d6 (cherry picked from commit 44aef59fa37b280d128159761d157e9b1d053634) diff --git a/Makefile.in b/Makefile.in index 167cc715442b..5fe8d52668eb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -324,6 +324,7 @@ symbols: rm -fr $(WORKDIR)/symbols/ mkdir -p $(WORKDIR)/symbols/ $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/linux/dump_syms/dump_syms $(WORKDIR)/symbols/ $(INSTDIR)/program/* + cd $(WORKDIR)/symbols/ && zip -r $(WORKDIR)/symbols.zip * dump-deps: @$(SRCDIR)/bin/module-deps.pl $(GNUMAKE) $(SRCDIR)/Makefile.gbuild commit d86bf0170ac0e2a78e31c71800daf219b1a66840 Author: Markus Mohrhard <[email protected]> AuthorDate: Tue May 24 00:33:10 2016 +0200 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:33:02 2018 +0100 remove old symbols directory Change-Id: Id6bd2dfd05f1c13d928dbf26414b01086740df74 (cherry picked from commit 9dc090ca54601aab957a41312d9f92aee7e42d0e) diff --git a/Makefile.in b/Makefile.in index 8026231a023f..167cc715442b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -321,6 +321,7 @@ findunusedheaders: $(SRCDIR)/bin/find-unusedheaders.pl symbols: + rm -fr $(WORKDIR)/symbols/ mkdir -p $(WORKDIR)/symbols/ $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/linux/dump_syms/dump_syms $(WORKDIR)/symbols/ $(INSTDIR)/program/* commit c876a2c01cfc20eb66c67dd3821fb1eb148972ca Author: Markus Mohrhard <[email protected]> AuthorDate: Mon May 23 22:57:56 2016 +0200 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:32:52 2018 +0100 add a build system target to generate the symbol files Change-Id: Ib690eb05deaec5d8ce91f6b76daadf427d7ad964 (cherry picked from commit 322d2dcfb65b04d72469a9825a74df226b63eac2) diff --git a/Makefile.in b/Makefile.in index 206f5edbb400..8026231a023f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -320,6 +320,9 @@ findunusedcode: findunusedheaders: $(SRCDIR)/bin/find-unusedheaders.pl +symbols: + mkdir -p $(WORKDIR)/symbols/ + $(SRCDIR)/bin/symbolstore.py $(WORKDIR)/UnpackedTarball/breakpad/src/tools/linux/dump_syms/dump_syms $(WORKDIR)/symbols/ $(INSTDIR)/program/* dump-deps: @$(SRCDIR)/bin/module-deps.pl $(GNUMAKE) $(SRCDIR)/Makefile.gbuild commit 154ef115cd25f83e24a3603cc9c2a85763648ad0 Author: Markus Mohrhard <[email protected]> AuthorDate: Mon May 23 22:46:28 2016 +0200 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:32:45 2018 +0100 also handle .bin files in the windows symbol code Change-Id: I85b0490c515987d56e04d0e5b42111c52bbabbc3 (cherry picked from commit 441bec249e8eeea9f3449c1aa507b23e68554b6c) diff --git a/bin/symbolstore.py b/bin/symbolstore.py index 941f44802174..823be626cce6 100755 --- a/bin/symbolstore.py +++ b/bin/symbolstore.py @@ -485,7 +485,7 @@ class Dumper_Win32(Dumper): or exe files with the same base name next to them.""" if file.endswith(".pdb"): (path,ext) = os.path.splitext(file) - if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll"): + if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll") or os.path.isfile(path + ".bin"): return True return False commit aab8352dc8d2c9847e4d0eb17afccd67e8fb628f Author: Markus Mohrhard <[email protected]> AuthorDate: Tue Dec 15 14:01:27 2015 +0100 Commit: Samuel Mehrbrodt <[email protected]> CommitDate: Mon Dec 3 14:32:25 2018 +0100 add script to generate symbols Change-Id: Icae707709307bc86360676692e55780b9ec89639 (cherry picked from commit c295c06e680feeb26aacba8f5b21c093025844a8) diff --git a/bin/symbolstore.py b/bin/symbolstore.py new file mode 100755 index 000000000000..941f44802174 --- /dev/null +++ b/bin/symbolstore.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# The Mozilla Foundation +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Ted Mielczarek <[email protected]> +# Ben Turner <[email protected]> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +# +# Usage: symbolstore.py <params> <dump_syms path> <symbol store path> +# <debug info files or dirs> +# Runs dump_syms on each debug info file specified on the command line, +# then places the resulting symbol file in the proper directory +# structure in the symbol store path. Accepts multiple files +# on the command line, so can be called as part of a pipe using +# find <dir> | xargs symbolstore.pl <dump_syms> <storepath> +# But really, you might just want to pass it <dir>. +# +# Parameters accepted: +# -c : Copy debug info files to the same directory structure +# as sym files +# -a "<archs>" : Run dump_syms -a <arch> for each space separated +# cpu architecture in <archs> (only on OS X) +# -s <srcdir> : Use <srcdir> as the top source directory to +# generate relative filenames. + +import sys +import os +import re +import shutil +from optparse import OptionParser + +# Utility classes + +class VCSFileInfo: + """ A base class for version-controlled file information. Ensures that the + following attributes are generated only once (successfully): + + self.root + self.clean_root + self.revision + self.filename + + The attributes are generated by a single call to the GetRoot, + GetRevision, and GetFilename methods. Those methods are explicitly not + implemented here and must be implemented in derived classes. """ + + def __init__(self, file): + if not file: + raise ValueError + self.file = file + + def __getattr__(self, name): + """ __getattr__ is only called for attributes that are not set on self, + so setting self.[attr] will prevent future calls to the GetRoot, + GetRevision, and GetFilename methods. We don't set the values on + failure on the off chance that a future call might succeed. """ + + if name == "root": + root = self.GetRoot() + if root: + self.root = root + return root + + elif name == "clean_root": + clean_root = self.GetCleanRoot() + if clean_root: + self.clean_root = clean_root + return clean_root + + elif name == "revision": + revision = self.GetRevision() + if revision: + self.revision = revision + return revision + + elif name == "filename": + filename = self.GetFilename() + if filename: + self.filename = filename + return filename + + raise AttributeError + + def GetRoot(self): + """ This method should return the unmodified root for the file or 'None' + on failure. """ + raise NotImplementedError + + def GetCleanRoot(self): + """ This method should return the repository root for the file or 'None' + on failure. """ + raise NotImplementedErrors + + def GetRevision(self): + """ This method should return the revision number for the file or 'None' + on failure. """ + raise NotImplementedError + + def GetFilename(self): + """ This method should return the repository-specific filename for the + file or 'None' on failure. """ + raise NotImplementedError + +class CVSFileInfo(VCSFileInfo): + """ A class to maintiain version information for files in a CVS repository. + Derived from VCSFileInfo. """ + + def __init__(self, file, srcdir): + VCSFileInfo.__init__(self, file) + self.srcdir = srcdir + + def GetRoot(self): + (path, filename) = os.path.split(self.file) + root = os.path.join(path, "CVS", "Root") + if not os.path.isfile(root): + return None + f = open(root, "r") + root_name = f.readline().strip() + f.close() + if root_name: + return root_name + print >> sys.stderr, "Failed to get CVS Root for %s" % filename + return None + + def GetCleanRoot(self): + parts = self.root.split('@') + if len(parts) > 1: + # we don't want the extra colon + return parts[1].replace(":","") + print >> sys.stderr, "Failed to get CVS Root for %s" % filename + return None + + def GetRevision(self): + (path, filename) = os.path.split(self.file) + entries = os.path.join(path, "CVS", "Entries") + if not os.path.isfile(entries): + return None + f = open(entries, "r") + for line in f: + parts = line.split("/") + if len(parts) > 1 and parts[1] == filename: + return parts[2] + print >> sys.stderr, "Failed to get CVS Revision for %s" % filename + return None + + def GetFilename(self): + file = self.file + if self.revision and self.clean_root: + if self.srcdir: + # strip the base path off + # but we actually want the last dir in srcdir + file = os.path.normpath(file) + # the lower() is to handle win32+vc8, where + # the source filenames come out all lowercase, + # but the srcdir can be mixed case + if file.lower().startswith(self.srcdir.lower()): + file = file[len(self.srcdir):] + (head, tail) = os.path.split(self.srcdir) + if tail == "": + tail = os.path.basename(head) + file = tail + file + return "cvs:%s:%s:%s" % (self.clean_root, file, self.revision) + return file + +class SVNFileInfo(VCSFileInfo): + url = None + repo = None + svndata = {} + + # This regex separates protocol and optional username/password from a url. + # For instance, all the following urls will be transformed into + # 'foo.com/bar': + # + # http://foo.com/bar + # svn+ssh://[email protected]/bar + # svn+ssh://user:[email protected]/bar + # + rootRegex = re.compile(r'^\S+?:/+(?:[^\s/]*@)?(\S+)$') + + def __init__(self, file): + """ We only want to run subversion's info tool once so pull all the data + here. """ + + VCSFileInfo.__init__(self, file) + + if os.path.isfile(file): + command = os.popen("svn info %s" % file, "r") + for line in command: + # The last line of the output is usually '\n' + if line.strip() == '': + continue + # Split into a key/value pair on the first colon + key, value = line.split(':', 1) + if key in ["Repository Root", "Revision", "URL"]: + self.svndata[key] = value.strip() + + exitStatus = command.close() + if exitStatus: + print >> sys.stderr, "Failed to get SVN info for %s" % file + + def GetRoot(self): + key = "Repository Root" + if key in self.svndata: + match = self.rootRegex.match(self.svndata[key]) + if match: + return match.group(1) + print >> sys.stderr, "Failed to get SVN Root for %s" % self.file + return None + + # File bug to get this teased out from the current GetRoot, this is temporary + def GetCleanRoot(self): + return self.root + + def GetRevision(self): + key = "Revision" + if key in self.svndata: + return self.svndata[key] + print >> sys.stderr, "Failed to get SVN Revision for %s" % self.file + return None + + def GetFilename(self): + if self.root and self.revision: + if "URL" in self.svndata and "Repository Root" in self.svndata: + url, repo = self.svndata["URL"], self.svndata["Repository Root"] + file = url[len(repo) + 1:] + return "svn:%s:%s:%s" % (self.root, file, self.revision) + print >> sys.stderr, "Failed to get SVN Filename for %s" % self.file + return self.file + +# Utility functions + +# A cache of files for which VCS info has already been determined. Used to +# prevent extra filesystem activity or process launching. +vcsFileInfoCache = {} + +def GetVCSFilename(file, srcdir): + """Given a full path to a file, and the top source directory, + look for version control information about this file, and return + a tuple containing + 1) a specially formatted filename that contains the VCS type, + VCS location, relative filename, and revision number, formatted like: + vcs:vcs location:filename:revision + For example: + cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36 + 2) the unmodified root information if it exists""" + (path, filename) = os.path.split(file) + if path == '' or filename == '': + return (file, None) + + fileInfo = None + root = '' + if file in vcsFileInfoCache: + # Already cached this info, use it. + fileInfo = vcsFileInfoCache[file] + else: + if os.path.isdir(os.path.join(path, "CVS")): + fileInfo = CVSFileInfo(file, srcdir) + if fileInfo: + root = fileInfo.root + elif os.path.isdir(os.path.join(path, ".svn")) or \ + os.path.isdir(os.path.join(path, "_svn")): + fileInfo = SVNFileInfo(file); + vcsFileInfoCache[file] = fileInfo + + if fileInfo: + file = fileInfo.filename + + # we want forward slashes on win32 paths + return (file.replace("\\", "/"), root) + +def GetPlatformSpecificDumper(**kwargs): + """This function simply returns a instance of a subclass of Dumper + that is appropriate for the current platform.""" + return {'win32': Dumper_Win32, + 'cygwin': Dumper_Win32, + 'linux2': Dumper_Linux, + 'sunos5': Dumper_Solaris, + 'darwin': Dumper_Mac}[sys.platform](**kwargs) + +def SourceIndex(fileStream, outputPath, cvs_root): + """Takes a list of files, writes info to a data block in a .stream file""" + # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing + # Create the srcsrv data block that indexes the pdb file + result = True + pdbStreamFile = open(outputPath, "w") + pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=1\r\nSRCSRV: variables ------------------------------------------\r\nCVS_EXTRACT_CMD=%fnchdir%(%targ%)cvs.exe -d %fnvar%(%var2%) checkout -r %var4% -d %var4% -N %var3%\r\nMYSERVER=''') + pdbStreamFile.write(cvs_root) + pdbStreamFile.write('''\r\nSRCSRVTRG=%targ%\%var4%\%fnbksl%(%var3%)\r\nSRCSRVCMD=%CVS_EXTRACT_CMD%\r\nSRCSRV: source files ---------------------------------------\r\n''') + pdbStreamFile.write(fileStream) # can't do string interpolation because the source server also uses this and so there are % in the above + pdbStreamFile.write("SRCSRV: end ------------------------------------------------\r\n\n") + pdbStreamFile.close() + return result + +class Dumper: + """This class can dump symbols from a file with debug info, and + store the output in a directory structure that is valid for use as + a Breakpad symbol server. Requires a path to a dump_syms binary-- + |dump_syms| and a directory to store symbols in--|symbol_path|. + Optionally takes a list of processor architectures to process from + each debug file--|archs|, the full path to the top source + directory--|srcdir|, for generating relative source file names, + and an option to copy debug info files alongside the dumped + symbol files--|copy_debug|, mostly useful for creating a + Microsoft Symbol Server from the resulting output. + + You don't want to use this directly if you intend to call + ProcessDir. Instead, call GetPlatformSpecificDumper to + get an instance of a subclass.""" + def __init__(self, dump_syms, symbol_path, + archs=None, srcdir=None, copy_debug=False, vcsinfo=False, srcsrv=False): + # popen likes absolute paths, at least on windows + self.dump_syms = os.path.abspath(dump_syms) + self.symbol_path = symbol_path + if archs is None: + # makes the loop logic simpler + self.archs = [''] + else: + self.archs = ['-a %s' % a for a in archs.split()] + if srcdir is not None: + self.srcdir = os.path.normpath(srcdir) + else: + self.srcdir = None + self.copy_debug = copy_debug + self.vcsinfo = vcsinfo + self.srcsrv = srcsrv + + # subclasses override this + def ShouldProcess(self, file): + return False + + def RunFileCommand(self, file): + """Utility function, returns the output of file(1)""" + try: + # we use -L to read the targets of symlinks, + # and -b to print just the content, not the filename + return os.popen("file -Lb " + file).read() + except: + return "" + + # This is a no-op except on Win32 + def FixFilenameCase(self, file): + return file + + # This is a no-op except on Win32 + def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root): + return "" + + # subclasses override this if they want to support this + def CopyDebug(self, file, debug_file, guid): + pass + + def Process(self, file_or_dir): + "Process a file or all the (valid) files in a directory." + if os.path.isdir(file_or_dir): + return self.ProcessDir(file_or_dir) + elif os.path.isfile(file_or_dir): + return self.ProcessFile(file_or_dir) + # maybe it doesn't exist? + return False + + def ProcessDir(self, dir): + """Process all the valid files in this directory. Valid files + are determined by calling ShouldProcess.""" + result = True + for root, dirs, files in os.walk(dir): + for f in files: + fullpath = os.path.join(root, f) + if self.ShouldProcess(fullpath): + if not self.ProcessFile(fullpath): + result = False + return result + + def ProcessFile(self, file): + """Dump symbols from this file into a symbol file, stored + in the proper directory structure in |symbol_path|.""" + result = False + sourceFileStream = '' + # tries to get cvsroot from the .mozconfig first - if it's not set + # the tinderbox cvs_path will be assigned further down + cvs_root = os.environ.get("SRCSRV_ROOT") + for arch in self.archs: + try: + cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r") + module_line = cmd.next() + if module_line.startswith("MODULE"): + # MODULE os cpu guid debug_file + (guid, debug_file) = (module_line.split())[3:5] + # strip off .pdb extensions, and append .sym + sym_file = re.sub("\.pdb$", "", debug_file) + ".sym" + # we do want forward slashes here + rel_path = os.path.join(debug_file, + guid, + sym_file).replace("\\", "/") + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + try: + os.makedirs(os.path.dirname(full_path)) + except OSError: # already exists + pass + f = open(full_path, "w") + f.write(module_line) + # now process the rest of the output + for line in cmd: + if line.startswith("FILE"): + # FILE index filename + (x, index, filename) = line.split(None, 2) + if sys.platform == "sunos5": + start = filename.find(self.srcdir) + if start == -1: + start = 0 + filename = filename[start:] + filename = self.FixFilenameCase(filename.rstrip()) + sourcepath = filename + if self.vcsinfo: + (filename, rootname) = GetVCSFilename(filename, self.srcdir) + # sets cvs_root in case the loop through files were to end on an empty rootname + if cvs_root is None: + if rootname: + cvs_root = rootname + # gather up files with cvs for indexing + if filename.startswith("cvs"): + (ver, checkout, source_file, revision) = filename.split(":", 3) + sourceFileStream += sourcepath + "*MYSERVER*" + source_file + '*' + revision + "\r\n" + f.write("FILE %s %s\n" % (index, filename)) + else: + # pass through all other lines unchanged + f.write(line) + f.close() + cmd.close() + # we output relative paths so callers can get a list of what + # was generated + print rel_path + if self.copy_debug: + self.CopyDebug(file, debug_file, guid) + if self.srcsrv: + # Call on SourceServerIndexing + result = self.SourceServerIndexing(debug_file, guid, sourceFileStream, cvs_root) + result = True + except StopIteration: + pass + except: + print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0] + raise + return result + +# Platform-specific subclasses. For the most part, these just have +# logic to determine what files to extract symbols from. + +class Dumper_Win32(Dumper): + fixedFilenameCaseCache = {} + + def ShouldProcess(self, file): + """This function will allow processing of pdb files that have dll + or exe files with the same base name next to them.""" + if file.endswith(".pdb"): + (path,ext) = os.path.splitext(file) + if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll"): + return True + return False + + def FixFilenameCase(self, file): + """Recent versions of Visual C++ put filenames into + PDB files as all lowercase. If the file exists + on the local filesystem, fix it.""" + + # Use a cached version if we have one. + if file in self.fixedFilenameCaseCache: + return self.fixedFilenameCaseCache[file] + + result = file + + (path, filename) = os.path.split(file) + if os.path.isdir(path): + lc_filename = filename.lower() + for f in os.listdir(path): + if f.lower() == lc_filename: + result = os.path.join(path, f) + break + + # Cache the corrected version to avoid future filesystem hits. + self.fixedFilenameCaseCache[file] = result + return result + + def CopyDebug(self, file, debug_file, guid): + rel_path = os.path.join(debug_file, + guid, + debug_file).replace("\\", "/") + print rel_path + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + shutil.copyfile(file, full_path) + pass + + def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root): + # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing + cwd = os.getcwd() + streamFilename = debug_file + ".stream" + stream_output_path = os.path.join(cwd, streamFilename) + # Call SourceIndex to create the .stream file + result = SourceIndex(sourceFileStream, stream_output_path, cvs_root) + + if self.copy_debug: + pdbstr_path = os.environ.get("PDBSTR_PATH") + pdbstr = os.path.normpath(pdbstr_path) + pdb_rel_path = os.path.join(debug_file, guid, debug_file) + pdb_filename = os.path.normpath(os.path.join(self.symbol_path, pdb_rel_path)) + # move to the dir with the stream files to call pdbstr + os.chdir(os.path.dirname(stream_output_path)) + os.spawnv(os.P_WAIT, pdbstr, [pdbstr, "-w", "-p:" + pdb_filename, "-i:" + streamFilename, "-s:srcsrv"]) + # clean up all the .stream files when done + os.remove(stream_output_path) + return result + +class Dumper_Linux(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or file.endswith(".bin") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("ELF") + return False + + def CopyDebug(self, file, debug_file, guid): + # We want to strip out the debug info, and add a + # .gnu_debuglink section to the object, so the debugger can + # actually load our debug info later. + file_dbg = file + ".dbg" + os.system("objcopy --only-keep-debug %s %s" % (file, file_dbg)) + os.system("objcopy --add-gnu-debuglink=%s %s" % (file_dbg, file)) + + rel_path = os.path.join(debug_file, + guid, + debug_file + ".dbg") + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + shutil.copyfile(file_dbg, full_path) + # gzip the shipped debug files + os.system("gzip %s" % full_path) + print rel_path + ".gz" + +class Dumper_Solaris(Dumper): + def RunFileCommand(self, file): + """Utility function, returns the output of file(1)""" + try: + output = os.popen("file " + file).read() + return output.split('\t')[1]; + except: + return "" + + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("ELF") + return False + +class Dumper_Mac(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .dylib extension, and additionally + file(1) reports as being Mach-O files. It expects to find the file + command in PATH.""" + if file.endswith(".dylib") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("Mach-O") + return False + +# Entry point if called as a standalone program +def main(): + parser = OptionParser(usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>") + parser.add_option("-c", "--copy", + action="store_true", dest="copy_debug", default=False, + help="Copy debug info files into the same directory structure as symbol files") + parser.add_option("-a", "--archs", + action="store", dest="archs", + help="Run dump_syms -a <arch> for each space separated cpu architecture in ARCHS (only on OS X)") + parser.add_option("-s", "--srcdir", + action="store", dest="srcdir", + help="Use SRCDIR to determine relative paths to source files") + parser.add_option("-v", "--vcs-info", + action="store_true", dest="vcsinfo", + help="Try to retrieve VCS info for each FILE listed in the output") + parser.add_option("-i", "--source-index", + action="store_true", dest="srcsrv", default=False, + help="Add source index information to debug files, making them suitable for use in a source server.") + (options, args) = parser.parse_args() + + #check to see if the pdbstr.exe exists + if options.srcsrv: + pdbstr = os.environ.get("PDBSTR_PATH") + if not os.path.exists(pdbstr): + print >> sys.stderr, "Invalid path to pdbstr.exe - please set/check PDBSTR_PATH.\n" + sys.exit(1) + + if len(args) < 3: + parser.error("not enough arguments") + exit(1) + + dumper = GetPlatformSpecificDumper(dump_syms=args[0], + symbol_path=args[1], + copy_debug=options.copy_debug, + archs=options.archs, + srcdir=options.srcdir, + vcsinfo=options.vcsinfo, + srcsrv=options.srcsrv) + for arg in args[2:]: + dumper.Process(arg) + +# run main if run directly +if __name__ == "__main__": + main() _______________________________________________ Libreoffice-commits mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
