diff --git a/msrLab b/msrLab new file mode 160000 index 0000000..59a29e5 --- /dev/null +++ b/msrLab @@ -0,0 +1 @@ +Subproject commit 59a29e583382c41e4fea703dd8c7eca4f47d4af0 diff --git a/repo_mining/Amy_CollectFiles.py b/repo_mining/Amy_CollectFiles.py new file mode 100644 index 0000000..b2df124 --- /dev/null +++ b/repo_mining/Amy_CollectFiles.py @@ -0,0 +1,90 @@ +import json +import requests +import csv + +import os + +if not os.path.exists("data"): + os.makedirs("data") + +# GitHub Authentication function +def github_auth(url, lsttoken, ct): + jsonData = None + try: + ct = ct % len(lstTokens) + headers = {'Authorization': 'Bearer {}'.format(lsttoken[ct])} + request = requests.get(url, headers=headers) + jsonData = json.loads(request.content) + ct += 1 + except Exception as e: + pass + print(e) + return jsonData, ct + +# @dictFiles, empty dictionary of files +# @lstTokens, GitHub authentication tokens +# @repo, GitHub repo +def countfiles(dictfiles, lsttokens, repo): + ipage = 1 # url page counter + ct = 0 # token counter + + try: + # loop though all the commit pages until the last returned empty page + while True: + spage = str(ipage) + commitsUrl = 'https://api.github.com/repos/' + repo + '/commits?page=' + spage + '&per_page=100' + jsonCommits, ct = github_auth(commitsUrl, lsttokens, ct) + + # break out of the while loop if there are no more commits in the pages + if len(jsonCommits) == 0: + break + # iterate through the list of commits in spage + for shaObject in jsonCommits: + sha = shaObject['sha'] + # For each commit, use the GitHub commit API to extract the files touched by the commit + shaUrl = 'https://api.github.com/repos/' + repo + '/commits/' + sha + shaDetails, ct = github_auth(shaUrl, lsttokens, ct) + filesjson = shaDetails['files'] + for filenameObj in filesjson: + filename = filenameObj['filename'] + dictfiles[filename] = dictfiles.get(filename, 0) + 1 + print(filename) + ipage += 1 + except: + print("Error receiving data") + exit(0) +# GitHub repo +repo = 'scottyab/rootbeer' +# repo = 'Skyscanner/backpack' # This repo is commit heavy. It takes long to finish executing +# repo = 'k9mail/k-9' # This repo is commit heavy. It takes long to finish executing +# repo = 'mendhak/gpslogger' + + +# put your tokens here +# Remember to empty the list when going to commit to GitHub. +# Otherwise they will all be reverted and you will have to re-create them +# I would advise to create more than one token for repos with heavy commits +lstTokens = ["ghp_shjCPSicu3TEjgbCXTiqdRmIpEQAEg1t7yP6"] + +dictfiles = dict() +countfiles(dictfiles, lstTokens, repo) +print('Total number of files: ' + str(len(dictfiles))) + +file = repo.split('/')[1] +# change this to the path of your file +fileOutput = 'data/file_' + file + '.csv' +rows = ["Filename", "Touches"] +fileCSV = open(fileOutput, 'w') +writer = csv.writer(fileCSV) +writer.writerow(rows) + +bigcount = None +bigfilename = None +for filename, count in dictfiles.items(): + rows = [filename, count] + writer.writerow(rows) + if bigcount is None or count > bigcount: + bigcount = count + bigfilename = filename +fileCSV.close() +print('The file ' + bigfilename + ' has been touched ' + str(bigcount) + ' times.') \ No newline at end of file diff --git a/repo_mining/Amy_authorsFileTouches.py b/repo_mining/Amy_authorsFileTouches.py new file mode 100644 index 0000000..90c8637 --- /dev/null +++ b/repo_mining/Amy_authorsFileTouches.py @@ -0,0 +1,94 @@ +import json +import requests +import csv + +import os + +if not os.path.exists("data"): + os.makedirs("data") + +# GitHub Authentication function +def github_auth(url, lsttoken, ct): + jsonData = None + try: + ct = ct % len(lsttoken) + headers = {'Authorization': 'Bearer {}'.format(lsttoken[ct])} + request = requests.get(url, headers=headers) + jsonData = json.loads(request.content) + ct += 1 + except Exception as e: + print(e) + return jsonData, ct + +# # @dictFiles, empty dictionary of files +# @lstTokens, GitHub authentication tokens +# @repo, GitHub repoollects authors and dates for each file touched by commits +def collect_file_touches(dictfiles, lsttokens, repo): + ipage = 1 # url page counter + ct = 0 # token counter + + try: + # loop though all the commit pages until the last returned empty page + while True: + spage = str(ipage) + commitsUrl = f'https://api.github.com/repos/{repo}/commits?page={spage}&per_page=100' + jsonCommits, ct = github_auth(commitsUrl, lsttokens, ct) + + # break out of the while loop if there are no more commits in the pages + if len(jsonCommits) == 0: + break + + # iterate through the list of commits in the page + for shaObject in jsonCommits: + sha = shaObject['sha'] + # For each commit, use the GitHub commit API to extract the files touched by the commit + shaUrl = f'https://api.github.com/repos/{repo}/commits/{sha}' + shaDetails, ct = github_auth(shaUrl, lsttokens, ct) + + commit_author = shaDetails['commit']['author']['name'] + commit_date = shaDetails['commit']['author']['date'] + filesjson = shaDetails['files'] + + for filenameObj in filesjson: + filename = filenameObj['filename'] + if filename not in dictfiles: + dictfiles[filename] = [] + dictfiles[filename].append((commit_author, commit_date)) + print(filename, commit_author, commit_date) + + ipage += 1 + except Exception as e: + print("Error receiving data:", e) + exit(0) + +# GitHub repo +repo = 'scottyab/rootbeer' +# repo = 'Skyscanner/backpack' # This repo is commit heavy. It takes long to finish executing +# repo = 'k9mail/k-9' # This repo is commit heavy. It takes long to finish executing +# repo = 'mendhak/gpslogger' + +# put your tokens here +# Remember to empty the list when going to commit to GitHub. +# Otherwise they will all be reverted and you will have to re-create them +# I would advise to create more than one token for repos with heavy commits +lstTokens = ["lol_shjCPSicu3TEjgbCXTiqdRmIpEQAEnaw7yP6"] + +dictfiles = dict() +collect_file_touches(dictfiles, lstTokens, repo) +print('Total number of files:', len(dictfiles)) + +file = repo.split('/')[1] +# change this to the path of your file +fileOutput = f'data/file_{file}_touches.csv' +rows = ["Filename", "Author", "Date"] +fileCSV = open(fileOutput, 'w', newline='') +writer = csv.writer(fileCSV) +writer.writerow(rows) + +for filename, touches in dictfiles.items(): + for author, date in touches: + rows = [filename, author, date] + writer.writerow(rows) + +fileCSV.close() +print(f'File touches data written to {fileOutput}') diff --git a/repo_mining/Amy_executive_summary.pdf b/repo_mining/Amy_executive_summary.pdf new file mode 100644 index 0000000..d9e3dbf Binary files /dev/null and b/repo_mining/Amy_executive_summary.pdf differ diff --git a/repo_mining/Amy_scatterplot.py b/repo_mining/Amy_scatterplot.py new file mode 100644 index 0000000..8d0e4a9 --- /dev/null +++ b/repo_mining/Amy_scatterplot.py @@ -0,0 +1,39 @@ +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np +from datetime import datetime + +csv_file = 'data/file_rootbeer_touches.csv' +df = pd.read_csv(csv_file) + +df['Date'] = pd.to_datetime(df['Date']) + +# find earliest commit date +earliest_date = df['Date'].min() + +# calc the number of weeks since the earliest commit date +df['Weeks'] = df['Date'].apply(lambda x: (x - earliest_date).days // 7) + + +authors = df['Author'].unique() # get list of unique authors +colors = plt.cm.rainbow(np.linspace(0, 1, len(authors))) +author_color_map = dict(zip(authors, colors)) # assign each a color + +# assign each unique file a unique number for x-coord +unique_files = df['Filename'].unique() +file_map = {file: i for i, file in enumerate(unique_files)} +df['FileIndex'] = df['Filename'].map(file_map) + + +plt.figure(figsize=(14, 8)) + +for author in authors: + author_data = df[df['Author'] == author] + plt.scatter(author_data['FileIndex'], author_data['Weeks'], color=author_color_map[author], label=author, alpha=0.6, edgecolors='w', s=100) + +plt.xlabel('Files') +plt.ylabel('Weeks') +plt.title('File Touches by Week and Author') +plt.legend(loc='upper right', bbox_to_anchor=(1.15, 1)) +plt.xticks(ticks=np.arange(0, len(unique_files), 10), labels=np.arange(0, len(unique_files), 10)) +plt.show()