Mining report

repo_mining/Amy_CollectFiles.py

@@ -0,0 +1,90 @@
+import json
+import requests
+import csv
+
+import os
+
+if not os.path.exists("data"):
+ os.makedirs("data")
+
+# GitHub Authentication function
+def github_auth(url, lsttoken, ct):
+    jsonData = None
+    try:
+        ct = ct % len(lstTokens)
+        headers = {'Authorization': 'Bearer {}'.format(lsttoken[ct])}
+        request = requests.get(url, headers=headers)
+        jsonData = json.loads(request.content)
+        ct += 1
+    except Exception as e:
+        pass
+        print(e)
+    return jsonData, ct
+
+# @dictFiles, empty dictionary of files
+# @lstTokens, GitHub authentication tokens
+# @repo, GitHub repo
+def countfiles(dictfiles, lsttokens, repo):
+    ipage = 1  # url page counter
+    ct = 0  # token counter
+
+    try:
+        # loop though all the commit pages until the last returned empty page
+        while True:
+            spage = str(ipage)
+            commitsUrl = 'https://api.github.com/repos/' + repo + '/commits?page=' + spage + '&per_page=100'
+            jsonCommits, ct = github_auth(commitsUrl, lsttokens, ct)
+
+            # break out of the while loop if there are no more commits in the pages
+            if len(jsonCommits) == 0:
+                break
+            # iterate through the list of commits in  spage
+            for shaObject in jsonCommits:
+                sha = shaObject['sha']
+                # For each commit, use the GitHub commit API to extract the files touched by the commit
+                shaUrl = 'https://api.github.com/repos/' + repo + '/commits/' + sha
+                shaDetails, ct = github_auth(shaUrl, lsttokens, ct)
+                filesjson = shaDetails['files']
+                for filenameObj in filesjson:
+                    filename = filenameObj['filename']
+                    dictfiles[filename] = dictfiles.get(filename, 0) + 1
+                    print(filename)
+            ipage += 1
+    except:
+        print("Error receiving data")
+        exit(0)
+# GitHub repo
+repo = 'scottyab/rootbeer'
+# repo = 'Skyscanner/backpack' # This repo is commit heavy. It takes long to finish executing
+# repo = 'k9mail/k-9' # This repo is commit heavy. It takes long to finish executing
+# repo = 'mendhak/gpslogger'
+
+
+# put your tokens here
+# Remember to empty the list when going to commit to GitHub.
+# Otherwise they will all be reverted and you will have to re-create them
+# I would advise to create more than one token for repos with heavy commits
+lstTokens = ["ghp_shjCPSicu3TEjgbCXTiqdRmIpEQAEg1t7yP6"]
+
+dictfiles = dict()
+countfiles(dictfiles, lstTokens, repo)
+print('Total number of files: ' + str(len(dictfiles)))
+
+file = repo.split('/')[1]
+# change this to the path of your file
+fileOutput = 'data/file_' + file + '.csv'
+rows = ["Filename", "Touches"]
+fileCSV = open(fileOutput, 'w')
+writer = csv.writer(fileCSV)
+writer.writerow(rows)
+
+bigcount = None
+bigfilename = None
+for filename, count in dictfiles.items():
+    rows = [filename, count]
+    writer.writerow(rows)
+    if bigcount is None or count > bigcount:
+        bigcount = count
+        bigfilename = filename
+fileCSV.close()
+print('The file ' + bigfilename + ' has been touched ' + str(bigcount) + ' times.')

repo_mining/Amy_authorsFileTouches.py

@@ -0,0 +1,94 @@
+import json
+import requests
+import csv
+
+import os
+
+if not os.path.exists("data"):
+    os.makedirs("data")
+
+# GitHub Authentication function
+def github_auth(url, lsttoken, ct):
+    jsonData = None
+    try:
+        ct = ct % len(lsttoken)
+        headers = {'Authorization': 'Bearer {}'.format(lsttoken[ct])}
+        request = requests.get(url, headers=headers)
+        jsonData = json.loads(request.content)
+        ct += 1
+    except Exception as e:
+        print(e)
+    return jsonData, ct
+
+# # @dictFiles, empty dictionary of files
+# @lstTokens, GitHub authentication tokens
+# @repo, GitHub repoollects authors and dates for each file touched by commits
+def collect_file_touches(dictfiles, lsttokens, repo):
+    ipage = 1  # url page counter
+    ct = 0  # token counter
+
+    try:
+        # loop though all the commit pages until the last returned empty page
+        while True:
+            spage = str(ipage)
+            commitsUrl = f'https://api.github.com/repos/{repo}/commits?page={spage}&per_page=100'
+            jsonCommits, ct = github_auth(commitsUrl, lsttokens, ct)
+
+            # break out of the while loop if there are no more commits in the pages
+            if len(jsonCommits) == 0:
+                break
+
+            # iterate through the list of commits in the page
+            for shaObject in jsonCommits:
+                sha = shaObject['sha']
+                # For each commit, use the GitHub commit API to extract the files touched by the commit
+                shaUrl = f'https://api.github.com/repos/{repo}/commits/{sha}'
+                shaDetails, ct = github_auth(shaUrl, lsttokens, ct)
+
+                commit_author = shaDetails['commit']['author']['name']
+                commit_date = shaDetails['commit']['author']['date']
+                filesjson = shaDetails['files']
+
+                for filenameObj in filesjson:
+                    filename = filenameObj['filename']
+                    if filename not in dictfiles:
+                        dictfiles[filename] = []
+                    dictfiles[filename].append((commit_author, commit_date))
+                    print(filename, commit_author, commit_date)
+
+            ipage += 1
+    except Exception as e:
+        print("Error receiving data:", e)
+        exit(0)
+
+# GitHub repo
+repo = 'scottyab/rootbeer'
+# repo = 'Skyscanner/backpack' # This repo is commit heavy. It takes long to finish executing
+# repo = 'k9mail/k-9' # This repo is commit heavy. It takes long to finish executing
+# repo = 'mendhak/gpslogger'
+
+# put your tokens here
+# Remember to empty the list when going to commit to GitHub.
+# Otherwise they will all be reverted and you will have to re-create them
+# I would advise to create more than one token for repos with heavy commits
+lstTokens = ["lol_shjCPSicu3TEjgbCXTiqdRmIpEQAEnaw7yP6"]
+
+dictfiles = dict()
+collect_file_touches(dictfiles, lstTokens, repo)
+print('Total number of files:', len(dictfiles))
+
+file = repo.split('/')[1]
+# change this to the path of your file
+fileOutput = f'data/file_{file}_touches.csv'
+rows = ["Filename", "Author", "Date"]
+fileCSV = open(fileOutput, 'w', newline='')
+writer = csv.writer(fileCSV)
+writer.writerow(rows)
+
+for filename, touches in dictfiles.items():
+    for author, date in touches:
+        rows = [filename, author, date]
+        writer.writerow(rows)
+
+fileCSV.close()
+print(f'File touches data written to {fileOutput}')

repo_mining/Amy_scatterplot.py

@@ -0,0 +1,39 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+from datetime import datetime
+
+csv_file = 'data/file_rootbeer_touches.csv'
+df = pd.read_csv(csv_file)
+
+df['Date'] = pd.to_datetime(df['Date'])
+
+# find earliest commit date
+earliest_date = df['Date'].min()
+
+# calc the number of weeks since the earliest commit date
+df['Weeks'] = df['Date'].apply(lambda x: (x - earliest_date).days // 7)
+
+
+authors = df['Author'].unique() # get list of unique authors
+colors = plt.cm.rainbow(np.linspace(0, 1, len(authors)))
+author_color_map = dict(zip(authors, colors)) # assign each a color
+
+# assign each unique file a unique number for x-coord
+unique_files = df['Filename'].unique()
+file_map = {file: i for i, file in enumerate(unique_files)}
+df['FileIndex'] = df['Filename'].map(file_map)
+
+
+plt.figure(figsize=(14, 8))
+
+for author in authors:
+    author_data = df[df['Author'] == author]
+    plt.scatter(author_data['FileIndex'], author_data['Weeks'], color=author_color_map[author], label=author, alpha=0.6, edgecolors='w', s=100)
+
+plt.xlabel('Files')
+plt.ylabel('Weeks')
+plt.title('File Touches by Week and Author')
+plt.legend(loc='upper right', bbox_to_anchor=(1.15, 1))
+plt.xticks(ticks=np.arange(0, len(unique_files), 10), labels=np.arange(0, len(unique_files), 10))
+plt.show()