esc-reporting/esc-analyze.py | 438 ++++++++++++++++++------------------------- 1 file changed, 186 insertions(+), 252 deletions(-)
New commits: commit 0decbf85a6b30881c4437fa5114f6baabfa0608c Author: jan Iversen <[email protected]> Date: Mon Dec 12 12:45:43 2016 +0100 Update to use gitdm-config information and general cleanup diff --git a/esc-reporting/esc-analyze.py b/esc-reporting/esc-analyze.py index 28af252..da7018a 100755 --- a/esc-reporting/esc-analyze.py +++ b/esc-reporting/esc-analyze.py @@ -66,6 +66,34 @@ def util_load_file(fileName): return rawData +def util_load_csv(fileName, split): + global statList + rawData = {} + with open(fileName, 'r') as fp: + fp.readline() + fp.readline() + for line in fp: + line = line[:-1] + x = line.split(split) + if split == ';' and len(x) != 3: + raise Exception('misformed entry ' + line + ' in filename ' + fileName) + if split == ' ' and len(x) != 2: + y = line.split('"') + if len(y) != 3: + raise Exception('misformed entry ' + line + ' in filename ' + fileName) + x[0] = y[1] + x[1] = y[2].split()[0] + del x[2:] + + if x[0] in rawData: + raise Exception('duplicate entry ' + x[0] + ' in filename ' + fileName) + elif len(x) == 3: + rawData[x[0]] = {'name': x[1], 'license': x[2]} + else: + rawData[x[0]] = x[1] + return rawData + + def util_dump_file(fileName, rawList): try: @@ -79,22 +107,23 @@ def util_dump_file(fileName, rawList): -def util_build_period_stat(cfg, statList, xDate, email, status, pstatus, base = 'gerrit'): +def util_build_period_stat(xDate, email, status, pstatus, base = 'gerrit'): + global cfg, statList for i in '1year', '3month', '1month', '1week': - if xDate >= cfg[i + 'Date']: - if email is not None: + if xDate > cfg[i + 'Date']: + if not email is None: statList['people'][email][base][i][pstatus] += 1 statList['people'][email][base][i]['total'] += 1 - if status: - if base != 'gerrit' : - statList['data'][base][i][status] += 1 - statList['data'][base][i]['total'] += 1 - elif statList['people'][email]['isCommitter']: - statList['data'][base]['committer'][i][status] += 1 - statList['data'][base]['committer'][i]['total'] += 1 - else: - statList['data'][base]['contributor'][i]['total'] += 1 - statList['data'][base]['contributor'][i][status] += 1 + statList['people'][email][base]['total'] += 1 + if not base == 'gerrit' : + statList['data'][base][i][status] += 1 + statList['data'][base]['total'] += 1 + elif statList['people'][email]['isCommitter']: + statList['data'][base]['committer'][i][status] += 1 + statList['data']['gerrit']['committer']['total'] += 1 + else: + statList['data']['gerrit']['contributor']['total'] += 1 + statList['data'][base]['contributor'][i][status] += 1 @@ -109,83 +138,76 @@ def util_load_data_file(fileName): def util_create_person_gerrit(person, email): return { 'name': person, 'email': email, - 'commits': {'1year': {'merged': 0, 'reviewMerged': 0}, - '3month': {'merged': 0, 'reviewMerged': 0}, - '1month': {'merged': 0, 'reviewMerged': 0}, - '1week': {'merged': 0, 'reviewMerged': 0}}, - 'gerrit': {'1year': {'owner': 0, 'reviewer': 0, 'total': 0}, - '3month': {'owner': 0, 'reviewer': 0, 'total': 0}, - '1month': {'owner': 0, 'reviewer': 0, 'total': 0}, - '1week': {'owner': 0, 'reviewer': 0, 'total': 0}, + 'commits': {'1year': {'merged': 0, 'reviewMerged': 0, 'total': 0}, + '3month': {'merged': 0, 'reviewMerged': 0, 'total': 0}, + '1month': {'merged': 0, 'reviewMerged': 0, 'total': 0}, + '1week': {'merged': 0, 'reviewMerged': 0, 'total': 0}, + 'total': 0}, + 'gerrit': {'1year': {'owner': 0, 'reviewer': 0, 'total': 0}, + '3month': {'owner': 0, 'reviewer': 0, 'total': 0}, + '1month': {'owner': 0, 'reviewer': 0, 'total': 0}, + '1week': {'owner': 0, 'reviewer': 0, 'total': 0}, + 'total': 0, 'userName': '*DUMMY*'}, - 'ui': {'1year': {'owner': 0, 'reviewer': 0, 'total': 0}, - '3month': {'owner': 0, 'reviewer': 0, 'total': 0}, - '1month': {'owner': 0, 'reviewer': 0, 'total': 0}, - '1week': {'owner': 0, 'reviewer': 0, 'total': 0}}, - 'qa': {'1year': {'owner': 0, 'reviewer': 0, 'regression': 0, 'bibisected': 0, - 'bisected': 0, 'backtrace': 0, 'fixed': 0, 'total': 0}, - '3month': {'owner': 0, 'reviewer': 0, 'regression': 0, 'bibisected': 0, - 'bisected': 0, 'backtrace': 0, 'fixed': 0, 'total': 0}, - '1month': {'owner': 0, 'reviewer': 0, 'regression': 0, 'bibisected': 0, - 'bisected': 0, 'backtrace': 0, 'fixed': 0, 'total': 0}, - '1week': {'owner': 0, 'reviewer': 0, 'regression': 0, 'bibisected': 0, - 'bisected': 0, 'backtrace': 0, 'fixed': 0, 'total': 0}}, + 'ui': {'1year': {'owner': 0, 'reviewer': 0, 'total': 0}, + '3month': {'owner': 0, 'reviewer': 0, 'total': 0}, + '1month': {'owner': 0, 'reviewer': 0, 'total': 0}, + '1week': {'owner': 0, 'reviewer': 0, 'total': 0}, + 'total': 0}, 'isCommitter': False, 'isContributor': False, - 'hasLicense': False, + 'licenseOK': False, + 'licenseText': '', 'newestCommit' : datetime.datetime(2001, 1, 1), 'prevCommit': datetime.datetime(2001, 1, 1)} def util_create_statList(): - return {'data': {'commits': {'committer': {'1year': {'#': 0}, '3month': {'#': 0}, '1month': {'#': 0}, '1week': {'#': 0}}, - 'contributor': {'1year': {'#': 0}, '3month': {'#': 0}, '1month': {'#': 0}, '1week': {'#': 0}}}, + return {'data': {'commits': {'committer': {'1year': {'merged': 0}, + '3month': {'merged': 0}, + '1month': {'merged': 0}, + '1week': {'merged': 0}, + 'total': 0}, + 'contributor': {'1year': {'merged': 0}, + '3month': {'merged': 0}, + '1month': {'merged': 0}, + '1week': {'merged': 0}, + 'total': 0}}, 'openhub': {'lines_of_code': 0, 'total_commits': 0, 'total_contributors': 0, 'year_commits': 0, 'year_contributors': 0}, - 'gerrit': {'contributor': {'1year': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}, - '3month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}, - '1month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}, - '1week': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}}, - 'committer': {'1year': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}, - '3month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}, - '1month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}, - '1week': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0, 'total': 0}}}, - 'trendCommitter': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}}, - 'trendContributor': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}}, - 'trendQA': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}}, - 'trendUI': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}, - '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0, 'total': 0}}, - 'ui': {'1year': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0, 'total': 0}, - '3month': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0, 'total': 0}, - '1month': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0, 'total': 0}, - '1week': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0, 'total': 0}, + 'gerrit': {'contributor': {'1year': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + '3month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + '1month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + '1week': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + 'total': 0}, + 'committer': {'1year': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + '3month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + '1month': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + '1week': {'ABANDONED': 0, 'MERGED': 0, 'NEW': 0, 'reviewed': 0}, + 'total': 0}}, + 'trendCommitter': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}}, + 'trendContributor': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}}, + 'trendUI': {'1year': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '3month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '1month': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}, + '1week': {'1-5': 0, '6-25': 0, '26-50': 0, '51-100': 0, '100+': 0}}, + 'ui': {'1year': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0}, + '3month': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0}, + '1month': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0}, + '1week': {'added': 0, 'removed': 0, 'commented': 0, 'resolved': 0}, 'needsUXEval' : 0, + 'total' : 0, 'topicUI': 0}, - 'qa': {'1year': {'UNCONFIRMED': 0, 'NEW': 0, 'ASSIGNED': 0, 'REOPENED': 0, 'RESOLVED': 0, - 'VERIFIED': 0, 'CLOSED': 0, 'NEEDINFO': 0, 'PLEASETEST': 0, 'commented': 0, 'total': 0}, - '3month': {'UNCONFIRMED': 0, 'NEW': 0, 'ASSIGNED': 0, 'REOPENED': 0, 'RESOLVED': 0, - 'VERIFIED': 0, 'CLOSED': 0, 'NEEDINFO': 0, 'PLEASETEST': 0, 'commented': 0, 'total': 0}, - '1month': {'UNCONFIRMED': 0, 'NEW': 0, 'ASSIGNED': 0, 'REOPENED': 0, 'RESOLVED': 0, - 'VERIFIED': 0, 'CLOSED': 0, 'NEEDINFO': 0, 'PLEASETEST': 0, 'commented': 0, 'total': 0}, - '1week': {'UNCONFIRMED': 0, 'NEW': 0, 'ASSIGNED': 0, 'REOPENED': 0, 'RESOLVED': 0, - 'VERIFIED': 0, 'CLOSED': 0, 'NEEDINFO': 0, 'PLEASETEST': 0, 'commented': 0, 'total': 0}, - 'unconfirmed': {'count': 0, 'enhancement' : 0, 'needsUXEval' : 0, - 'haveBacktrace' : 0, 'needsDevAdvice' : 0}}, 'easyhacks' : {'needsDevEval': 0, 'needsUXEval': 0, 'cleanup_comments': 0, 'total': 0, 'assigned': 0, 'open': 0}}, 'stat': {'openhub_last_analyse': "2001-01-01"}, @@ -194,17 +216,13 @@ def util_create_statList(): -def util_check_mail(name, mail, statList, combineMail): - if mail in combineMail: - mail = combineMail[mail] +def util_check_mail(name, mail): + global cfg, statList + if mail in cfg['aliases']: + mail = cfg['aliases'][mail] if not mail in statList['people']: - if not name: - name = '*UNKNOWN*' statList['people'][mail] = util_create_person_gerrit(name, mail) - elif name and name != '*UNKNOWN*': - statList['people'][mail]['name'] = name - return mail @@ -225,7 +243,9 @@ def util_build_diff(newList, oldList): -def analyze_mentoring(statList, openhubData, gerritData, gitData, bugzillaData, cfg): +def analyze_mentoring(): + global cfg, statList, openhubData, bugzillaData, gerritData, gitData + print("mentoring: analyze openhub", end="", flush=True) statList['data']['openhub']['lines_of_code'] = int(openhubData['project']['analysis']['total_code_lines']) statList['data']['openhub']['total_commits'] = int(openhubData['project']['analysis']['total_commit_count']) @@ -239,43 +259,40 @@ def analyze_mentoring(statList, openhubData, gerritData, gitData, bugzillaData, print("mentoring: analyze gerrit", end="", flush=True) for row in gerritData['committers']: - mail = util_check_mail(row['name'], row['email'], statList, cfg['contributor']['combine-email']) + mail = util_check_mail(row['name'], row['email']) statList['people'][mail]['gerrit']['userName'] = row['username'] statList['people'][mail]['isCommitter'] = True statList['people'][mail]['isContributor'] = True statNewDate = cfg['1yearDate'] statOldDate = cfg['nowDate'] - for key in gerritData['patch']: - row = gerritData['patch'][key] - if row['status'] == 'SUBMITTED' or row['status'] == 'DRAFT': - row['status'] = 'NEW' + for key, row in gerritData['patch'].items(): xDate = datetime.datetime.strptime(row['updated'], '%Y-%m-%d %H:%M:%S.%f000') if xDate > cfg['cutDate']: continue - ownerEmail = util_check_mail(row['owner']['name'], row['owner']['email'], statList, cfg['contributor']['combine-email']) - statList['people'][ownerEmail]['gerrit']['userName'] = row['owner']['username'] - util_build_period_stat(cfg, statList, xDate, ownerEmail, row['status'], 'owner') - if ownerEmail in cfg['contributor']['contributors'] or ownerEmail in cfg['contributor']['license-pending']: - statList['people'][ownerEmail]['hasLicense'] = True if xDate < statOldDate: statOldDate = xDate if xDate > statNewDate: statNewDate = xDate + if row['status'] == 'SUBMITTED' or row['status'] == 'DRAFT': + row['status'] = 'NEW' + ownerEmail = util_check_mail(row['owner']['name'], row['owner']['email']) + statList['people'][ownerEmail]['gerrit']['userName'] = row['owner']['username'] + statList['people'][ownerEmail]['isContributor'] = True + util_build_period_stat(xDate, ownerEmail, row['status'], 'owner') for i in 'Verified', 'Code-Review': for x in row['labels'][i]['all']: - xEmail = util_check_mail(x['name'], x['email'], statList, cfg['contributor']['combine-email']) + xEmail = util_check_mail(x['name'], x['email']) if xEmail != ownerEmail: - util_build_period_stat(cfg, statList, xDate, xEmail, 'reviewed', 'reviewer') + util_build_period_stat(xDate, xEmail, 'reviewed', 'reviewer') print(" from " + statOldDate.strftime('%Y-%m-%d') + " to " + statNewDate.strftime('%Y-%m-%d')) print("mentoring: analyze git", end="", flush=True) statNewDate = cfg['1yearDate'] statOldDate = cfg['nowDate'] - for key in gitData['commits']: - row = gitData['commits'][key] + for key, row in gitData['commits'].items(): xDate = datetime.datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S") if xDate > cfg['cutDate']: continue @@ -283,14 +300,11 @@ def analyze_mentoring(statList, openhubData, gerritData, gitData, bugzillaData, statOldDate = xDate if xDate > statNewDate: statNewDate = xDate - author = util_check_mail(row['author'], row['author-email'], statList, cfg['contributor']['combine-email']) - committer = util_check_mail(row['committer'], row['committer-email'], statList, cfg['contributor']['combine-email']) + author = util_check_mail(row['author'], row['author-email']) + committer = util_check_mail(row['committer'], row['committer-email']) statList['people'][author]['isContributor'] = True statList['people'][committer]['isContributor'] = True - if author in cfg['contributor']['contributors'] or author in cfg['contributor']['license-pending']: - statList['people'][author]['hasLicense'] = True - if committer in cfg['contributor']['contributors'] or committer in cfg['contributor']['license-pending']: - statList['people'][committer]['hasLicense'] = True + statList['people'][committer]['isCommitter'] = True for i in author, committer: if xDate > statList['people'][i]['newestCommit']: @@ -305,10 +319,12 @@ def analyze_mentoring(statList, openhubData, gerritData, gitData, bugzillaData, if author != committer: statList['people'][author]['commits'][i]['merged'] += 1 statList['people'][committer]['commits'][i]['reviewMerged'] += 1 - statList['data']['commits']['contributor'][i]['#'] += 1 + statList['data']['commits']['contributor'][i]['merged'] += 1 + statList['data']['commits']['contributor']['total'] += 1 else: statList['people'][author]['commits'][i]['merged'] += 1 - statList['data']['commits']['committer'][i]['#'] += 1 + statList['data']['commits']['committer'][i]['merged'] += 1 + statList['data']['commits']['committer']['total'] += 1 print(" from " + statOldDate.strftime("%Y-%m-%d") + " to " + statNewDate.strftime("%Y-%m-%d")) print("mentoring: analyze easyhacks", end="", flush=True) @@ -350,11 +366,12 @@ def analyze_mentoring(statList, openhubData, gerritData, gitData, bugzillaData, -def analyze_ui(statList, openhubData, gerritData, gitData, bugzillaData, cfg): +def analyze_ui(): + global cfg, statList, bugzillaData + print("ui: analyze bugzilla", flush=True) for key, row in bugzillaData['bugs'].items(): - xDate = datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") if xDate > cfg['cutDate']: continue @@ -363,7 +380,7 @@ def analyze_ui(statList, openhubData, gerritData, gitData, bugzillaData, cfg): continue if row['status'] == 'RESOLVED' or row['status'] == 'CLOSED' or row['status'] == 'VERIFIED': - util_build_period_stat(cfg, statList, xDate, None, 'resolved', 'reviewer', base='ui') + util_build_period_stat(xDate, None, 'resolved', 'reviewer', base='ui') continue if 'needsUXEval' in row['keywords']: @@ -373,75 +390,27 @@ def analyze_ui(statList, openhubData, gerritData, gitData, bugzillaData, cfg): statList['data']['ui']['topicUI'] += 1 for change in row['comments']: - email = util_check_mail('*UNKNOWN*', change['creator'], statList, cfg['contributor']['combine-email']) + email = util_check_mail('*UNKNOWN*', change['creator']) xDate = datetime.datetime.strptime(change['creation_time'], "%Y-%m-%dT%H:%M:%SZ") - util_build_period_stat(cfg, statList, xDate, email, 'commented', 'reviewer', base='ui') + util_build_period_stat(xDate, email, 'commented', 'reviewer', base='ui') for change in row['history']: - email = util_check_mail('*UNKNOWN*', change['who'], statList, cfg['contributor']['combine-email']) + email = util_check_mail('*UNKNOWN*', change['who']) xDate = datetime.datetime.strptime(change['when'], "%Y-%m-%dT%H:%M:%SZ") for entry in change['changes']: keywordsAdded = entry['added'].split(", ") for keyword in keywordsAdded: if keyword == 'needsUXEval': - util_build_period_stat(cfg, statList, xDate, email, 'added', 'reviewer', base='ui') - + util_build_period_stat(xDate, email, 'added', 'reviewer', base='ui') keywordsRemoved = entry['removed'].split(", ") for keyword in keywordsRemoved: if keyword == 'needsUXEval': - util_build_period_stat(cfg, statList, xDate, email, 'removed', 'reviewer', base='ui') + util_build_period_stat(xDate, email, 'removed', 'reviewer', base='ui') -def analyze_qa(statList, openhubData, gerritData, gitData, bugzillaData, cfg): - print("qa: analyze bugzilla", flush=True) - for key, row in bugzillaData['bugs'].items(): - email = util_check_mail(row['creator_detail']['real_name'], row['creator'], statList, cfg['contributor']['combine-email']) - xDate = datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") - creationDate = datetime.datetime.strptime(row['creation_time'], "%Y-%m-%dT%H:%M:%SZ") - if xDate > cfg['cutDate']: - continue - - if row['status'] == 'UNCONFIRMED': - statList['data']['qa']['unconfirmed']['count'] += 1 - if 'needsUXEval' in row['keywords']: - statList['data']['qa']['unconfirmed']['needsUXEval'] += 1 - if 'needsDevAdvice' in row['keywords']: - statList['data']['qa']['unconfirmed']['needsDevAdvice'] += 1 - if 'haveBacktrace' in row['keywords']: - statList['data']['qa']['unconfirmed']['haveBacktrace'] += 1 - if row['severity'] == 'enhancement': - statList['data']['qa']['unconfirmed']['enhancement'] += 1 +def analyze_myfunc(): + global cfg, statList, openhubData, bugzillaData, gerritData, gitData, aliasData, licenceCompanyData, licencePersonalData - util_build_period_stat(cfg, statList, creationDate, email, row['status'], 'owner', base='qa') - - for change in row['history']: - email = util_check_mail('*UNKNOWN*', change['who'], statList, cfg['contributor']['combine-email']) - xDate = datetime.datetime.strptime(change['when'], "%Y-%m-%dT%H:%M:%SZ") - for entry in change['changes']: - if entry['field_name'] == 'keywords': - keywordsAdded = entry['added'].split(", ") - for keyword in keywordsAdded: - if keyword == 'bisected': - util_build_period_stat(cfg, statList, xDate, email, '', 'bisected', base='qa') - if keyword == 'bibisected': - util_build_period_stat(cfg, statList, xDate, email, '', 'bibisected', base='qa') - if keyword == 'regression': - util_build_period_stat(cfg, statList, xDate, email, '', 'regression', base='qa') - if keyword == 'haveBacktrace': - util_build_period_stat(cfg, statList, xDate, email, '', 'backtrace', base='qa') - elif entry['field_name'] == 'resolution': - if entry['added'] == 'FIXED': - util_build_period_stat(cfg, statList, xDate, email, '', 'fixed', base='qa') - - - for change in row['comments']: - email = util_check_mail('*UNKNOWN*', change['creator'], statList, cfg['contributor']['combine-email']) - xDate = datetime.datetime.strptime(change['creation_time'], "%Y-%m-%dT%H:%M:%SZ") - util_build_period_stat(cfg, statList, xDate, email, 'commented', 'reviewer', base='qa') - - - -def analyze_myfunc(statList, openhubData, gerritData, gitData, bugzillaData, cfg): print("myfunc: analyze nothing", flush=True) @@ -461,7 +430,9 @@ def util_build_trend(cnt): -def analyze_trend(statList, cfg): +def analyze_trend(): + global statList + for email in statList['people']: person = statList['people'][email] @@ -470,32 +441,26 @@ def analyze_trend(statList, cfg): if not x is None: if person['isCommitter']: statList['data']['trendCommitter'][inx][x] += 1 - statList['data']['trendCommitter'][inx]['total'] += 1 elif person['isContributor']: statList['data']['trendContributor'][inx][x] += 1 - statList['data']['trendContributor'][inx]['total'] += 1 - x = util_build_trend(person['qa'][inx]['total']) - if not x is None: - statList['data']['trendQA'][inx][x] += 1 - statList['data']['trendQA'][inx]['total'] += 1 - x = util_build_trend(person['ui'][inx]['total']) + x = util_build_trend(person['ui']['total']) if not x is None: statList['data']['trendUI'][inx][x] += 1 - statList['data']['trendUI'][inx]['total'] += 1 -def analyze_final(statList, cfg): + +def analyze_final(): + global cfg, statList, openhubData, bugzillaData, gerritData, gitData + print("Analyze final") statList['addDate'] = datetime.date.today().strftime('%Y-%m-%d') - zDate = datetime.datetime(year=2001, month=1, day=1) for i in statList['people']: person = statList['people'][i] - delta = person['newestCommit'] - person['prevCommit'] person['newestCommit'] = person['newestCommit'].strftime("%Y-%m-%d") person['prevCommit'] = person['prevCommit'].strftime("%Y-%m-%d") - analyze_trend(statList, cfg) + analyze_trend() myDay = cfg['nowDate'] x = (myDay - datetime.timedelta(days=7)).strftime('%Y-%m-%d') weekList = util_load_file(cfg['homedir'] + 'archive/stats_' + x + '.json') @@ -507,11 +472,46 @@ def analyze_final(statList, cfg): x = myDay.strftime('%Y-%m-%d') os.system('cp '+ sFile + ' ' + cfg['homedir'] + 'archive/stats_' + x + '.json') if myDay.strftime('%w') == '4': - os.system('cp ' + sFile + ' ' + cfg['homedir'] + 'weeks/week_' + myDay.strftime('%Y_%W') + '.json') + del statList['people'] + util_dump_file(cfg['homedir'] + 'weeks/week_' + myDay.strftime('%Y_%W') + '.json', statList) -def runCfg(platform): +def runLoadCSV(): + global cfg, statList + + try: + fileName = cfg['homedir'] + 'gitdm-config/aliases' + cfg['aliases'] = util_load_csv(fileName, ' ') + fileName = cfg['homedir'] + 'gitdm-config/licenseCompany.csv' + cfg['companies'] = util_load_csv(fileName, ';') + fileName = cfg['homedir'] + 'gitdm-config/licensePersonal.csv' + licencePersonalData = util_load_csv(fileName, ';') + + # check consistency + for i in cfg['aliases']: + if i in licencePersonalData: + raise Exception('alias ' + i + ' in aliases is licensed') + elif cfg['aliases'][i] not in licencePersonalData: + raise Exception('target ' + cfg['aliases'][i] + ' for alias ' + i + ' in aliases is NOT licensed') + + # create base people info + for id, row in licencePersonalData.items(): + statList['people'][id] = util_create_person_gerrit(row['name'], id) + statList['people'][id]['licenseOK'] = True + x = row['license'] + if not x.startswith('http') and not x.startswith('COMPANY') and not x.startswith('AUDIT'): + statList['people'][id]['licenseText'] = row['license'] + + except Exception as e: + print('Error load file ' + fileName + ' due to ' + str(e)) + exit(-1) + + + +def runAnalyze(platform): + global cfg, statList, openhubData, bugzillaData, gerritData, gitData + if 'esc_homedir' in os.environ: homeDir = os.environ['esc_homedir'] else: @@ -519,93 +519,27 @@ def runCfg(platform): cfg = util_load_data_file(homeDir + '/config.json') cfg['homedir'] = homeDir + '/' cfg['platform'] = platform - print("Reading and writing data to " + cfg['homedir']) - - cfg['contributor'] = util_load_data_file(cfg['homedir'] + 'dump/developers_dump.json') cfg['nowDate'] = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) cfg['cutDate'] = cfg['nowDate'] cfg['1weekDate'] = cfg['nowDate'] - datetime.timedelta(days=7) cfg['1monthDate'] = cfg['nowDate'] - datetime.timedelta(days=30) cfg['3monthDate'] = cfg['nowDate'] - datetime.timedelta(days=90) cfg['1yearDate'] = cfg['nowDate'] - datetime.timedelta(days=365) - return cfg - - - -def runAnalyze(cfg, openhubData, bugzillaData, gerritData, gitData): - statList = util_create_statList() - analyze_mentoring(statList, openhubData, gerritData, gitData, bugzillaData, cfg) - analyze_ui(statList, openhubData, gerritData, gitData, bugzillaData, cfg) - analyze_qa(statList, openhubData, gerritData, gitData, bugzillaData, cfg) - analyze_myfunc(statList, openhubData, gerritData, gitData, bugzillaData, cfg) - analyze_final(statList, cfg) - - - -def runLoad(cfg): - openhubData = util_load_data_file(cfg['homedir'] + 'dump/openhub_dump.json') - bugzillaData = util_load_data_file(cfg['homedir'] + 'dump/bugzilla_dump.json') - gerritData = util_load_data_file(cfg['homedir'] + 'dump/gerrit_dump.json') - gitData = util_load_data_file(cfg['homedir'] + 'dump/git_dump.json') - - runAnalyze(cfg, openhubData, bugzillaData, gerritData, gitData) - - + print("Reading and writing data to " + cfg['homedir']) -def runBackLoad(cfg): openhubData = util_load_data_file(cfg['homedir'] + 'dump/openhub_dump.json') bugzillaData = util_load_data_file(cfg['homedir'] + 'dump/bugzilla_dump.json') gerritData = util_load_data_file(cfg['homedir'] + 'dump/gerrit_dump.json') gitData = util_load_data_file(cfg['homedir'] + 'dump/git_dump.json') + statList = util_create_statList() + runLoadCSV() - cfg['nowDate'] = datetime.datetime(year=2016, month=11, day=1) - stopDate = datetime.datetime(year=2016, month=11, day=13) - - while stopDate > cfg['nowDate']: - cfg['cutDate'] = cfg['nowDate'] - cfg['1weekDate'] = cfg['nowDate'] - datetime.timedelta(days=7) - cfg['1monthDate'] = cfg['nowDate'] - datetime.timedelta(days=30) - cfg['3monthDate'] = cfg['nowDate'] - datetime.timedelta(days=90) - cfg['1yearDate'] = cfg['nowDate'] - datetime.timedelta(days=365) - print('regenerate ' + cfg['nowDate'].strftime('%Y-%m-%d')) - runAnalyze(cfg, openhubData, bugzillaData, gerritData, gitData) - cfg['nowDate'] = cfg['nowDate'] + datetime.timedelta(days=1) - - - -def runUpgrade(cfg, fileList): -# openhubData = util_load_data_file(cfg['homedir'] + 'dump/openhub_dump.json') -# bugzillaData = util_load_data_file(cfg['homedir'] + 'dump/bugzilla_dump.json') -# gerritData = util_load_data_file(cfg['homedir'] + 'dump/gerrit_dump.json') -# gitData = util_load_data_file(cfg['homedir'] + 'dump/git_dump.json') - -# statList = util_load_data_file(cfg['homedir'] + 'weeks/week_2016_44.json') -# cfg['nowDate'] = datetime.datetime(year=2016, month=11, day=9) - for i in fileList: - statList = util_load_data_file(cfg['homedir'] + 'archive/' + i) - if 'closed' in statList['data']['ui']: - del statList['data']['ui']['closed'] - if 'ui' in statList['diff']: - if 'closed' in statList['diff']['ui']: - del statList['diff']['ui']['closed'] - statList['diff']['ui']['1year']['resolved'] = 0 - statList['diff']['ui']['3month']['resolved'] = 0 - statList['diff']['ui']['1month']['resolved'] = 0 - statList['diff']['ui']['1week']['resolved'] = 0 - statList['data']['ui']['1year']['resolved'] = 0 - statList['data']['ui']['3month']['resolved'] = 0 - statList['data']['ui']['1month']['resolved'] = 0 - statList['data']['ui']['1week']['resolved'] = 0 - util_dump_file(cfg['homedir'] + 'archive/' + i, statList) - + analyze_mentoring() + analyze_ui() + analyze_myfunc() + analyze_final() if __name__ == '__main__': - if len(sys.argv) > 1: - if sys.argv[1] == 'backload': - runBackLoad(runCfg(sys.platform)) - elif sys.argv[1] == 'upgrade': - runUpgrade(runCfg(sys.platform), sys.argv[2:]) - else: - runLoad(runCfg(sys.platform)) + runAnalyze(sys.platform) _______________________________________________ Libreoffice-commits mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
