Code Generalization

_explore/input_lists.json

@@ -1,4 +1,7 @@
 {
+    "memberOrgs": [
+        "llnl"
+    ],
     "orgs": [
         "cdat",
         "chaos",

_explore/scripts/MASTER.sh

@@ -46,7 +46,7 @@ runScript cleanup_inputs.py
 # Required before any other repo scripts (output used as repo list)
 runScript get_repos_info.py
 # Required before any other member scripts (output used as member list)
-runScript get_llnl_members.py
+runScript get_internal_members.py
 
 
 # --- EXTERNAL V INTERNAL ---

_explore/scripts/build_yearlist.py

@@ -29,6 +29,6 @@
     yearDict[prefix] = yearList
 
 yearData = qm.DataManager("%s/YEARS.json" % ghDataDir, False)
-yearData.fileSave()
+yearData.fileSave(newline="\n")
 
 print("Done!\n")

_explore/scripts/cleanup_inputs.py

@@ -17,6 +17,6 @@
     listWIP.sort()  # List in alphabetical order
     inputLists.data[aList] = listWIP
 
-inputLists.fileSave()
+inputLists.fileSave(newline="\n")
 
 print("Primary input lists cleaned!")

_explore/scripts/gather_repo_metadata.py

@@ -1,9 +1,9 @@
 from scraper.github import queryManager as qm
 
 ghDataDir = "../../explore/github-data"
-genDatafile = "%s/labReposInfo.json" % ghDataDir
-topicsDatafile = "%s/labRepos_Topics.json" % ghDataDir
-writeFile = "%s/labRepo_Metadata.json" % ghDataDir
+genDatafile = "%s/intReposInfo.json" % ghDataDir
+topicsDatafile = "%s/intRepos_Topics.json" % ghDataDir
+writeFile = "%s/intRepo_Metadata.json" % ghDataDir
 
 # initialize data manager and load repo info
 genDataCollector = qm.DataManager(genDatafile, True)
@@ -41,6 +41,6 @@
     infoWriter.data[repo] = repoData
 
 # write data to file
-infoWriter.fileSave()
+infoWriter.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_dependency_info.py

@@ -6,7 +6,7 @@
 queryPath = "../queries/dependency-Info.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labRepos_Dependencies.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intRepos_Dependencies.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting dependency repos ...")
@@ -57,6 +57,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_llnl_members.py → _explore/scripts/get_internal_members.py

@@ -2,11 +2,12 @@
 from os import environ as env
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labUsers.json" % ghDataDir
+datfilepath = "%s/intUsers.json" % ghDataDir
 queryPath = "../queries/org-Members.gql"
 
-# Only looking at LLNL org members
-orglist = ["llnl"]
+# Read input list of member organizations
+inputLists = qm.DataManager("../input_lists.json", True)
+orglist = inputLists.data["memberOrgs"]
 
 # Initialize data collector
 dataCollector = qm.DataManager(datfilepath, False)
@@ -43,6 +44,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_members_extrepos.py

@@ -6,18 +6,18 @@
 queryPath = "../queries/user-Repos.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
 repolist = sorted(inputLists.data["data"].keys())
 print("Repo list complete. Found %d repos." % (len(repolist)))
 
-# Read lab user data file (to use as member list)
-inputLists = qm.DataManager("%s/labUsers.json" % ghDataDir, True)
+# Read internal user data file (to use as member list)
+inputLists = qm.DataManager("%s/intUsers.json" % ghDataDir, True)
 # Populate member list
 memberlist = []
-print("Getting LLNL members ...")
+print("Getting internal members ...")
 memberlist = sorted(inputLists.data["data"].keys())
 print("Member list complete. Found %d users." % (len(memberlist)))
 
@@ -28,7 +28,7 @@
 # Initialize query manager
 queryMan = qm.GitHubQueryManager()
 
-# Iterate through lab members
+# Iterate through internal members
 print("Gathering data across multiple paginated queries...")
 for usr in memberlist:
     print("\n'%s'" % (usr))
@@ -62,6 +62,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_activitycommits.py

@@ -4,11 +4,11 @@
 from datetime import datetime
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_ActivityCommits.json" % ghDataDir
+datfilepath = "%s/intRepos_ActivityCommits.json" % ghDataDir
 query_in = "/repos/OWNNAME/REPONAME/stats/commit_activity"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -58,6 +58,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_activitylines.py

@@ -5,11 +5,11 @@
 from dateutil.relativedelta import relativedelta
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_ActivityLines.json" % ghDataDir
+datfilepath = "%s/intRepos_ActivityLines.json" % ghDataDir
 query_in = "/repos/OWNNAME/REPONAME/stats/code_frequency"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -60,6 +60,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_creationhistory.py

@@ -3,12 +3,12 @@
 import re
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_CreationHistory.json" % ghDataDir
+datfilepath = "%s/intRepos_CreationHistory.json" % ghDataDir
 query_commits_in = "/repos/OWNNAME/REPONAME/commits?until=CREATETIME&per_page=100"
 query_commits_in2 = "/repos/OWNNAME/REPONAME/commits?per_page=100"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -119,6 +119,6 @@
         print("Removed '%s'" % (repo))
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_dependencies.py

@@ -2,11 +2,11 @@
 from os import environ as env
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_Dependencies.json" % ghDataDir
+datfilepath = "%s/intRepos_Dependencies.json" % ghDataDir
 queryPath = "../queries/repo-Dependencies.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -54,6 +54,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_info.py

@@ -2,7 +2,7 @@
 from os import environ as env
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labReposInfo.json" % ghDataDir
+datfilepath = "%s/intReposInfo.json" % ghDataDir
 queryPath = "../queries/org-Repos-Info.gql"
 queryPathInd = "../queries/repo-Info.gql"
 
@@ -70,6 +70,6 @@
 print("\nCollective data gathering Part2of2 complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_languages.py

@@ -2,11 +2,11 @@
 from os import environ as env
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_Languages.json" % ghDataDir
+datfilepath = "%s/intRepos_Languages.json" % ghDataDir
 queryPath = "../queries/repo-Languages.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -47,6 +47,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_releases.py

@@ -2,11 +2,11 @@
 from os import environ as env
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_ReleaseHistory.json" % ghDataDir
+datfilepath = "%s/intRepos_ReleaseHistory.json" % ghDataDir
 queryPath = "../queries/repo-Releases.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -47,6 +47,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output files
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_starhistory.py

@@ -3,11 +3,11 @@
 from datetime import date, timedelta
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_StarHistory.json" % ghDataDir
+datfilepath = "%s/intRepos_StarHistory.json" % ghDataDir
 queryPath = "../queries/repo-Stargazers.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -78,6 +78,6 @@ def toDate(isoStr):
     dataCollector.data["data"][repo] = dateList
 
 # Write output files
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_topics.py

@@ -2,11 +2,11 @@
 from os import environ as env
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
-datfilepath = "%s/labRepos_Topics.json" % ghDataDir
+datfilepath = "%s/intRepos_Topics.json" % ghDataDir
 queryPath = "../queries/repo-Topics.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
@@ -47,6 +47,6 @@
 print("\nCollective data gathering complete!")
 
 # Write output file
-dataCollector.fileSave()
+dataCollector.fileSave(newline="\n")
 
 print("\nDone!\n")

_explore/scripts/get_repos_users.py

@@ -3,23 +3,23 @@
 
 ghDataDir = env.get("GITHUB_DATA", "../github-data")
 datfilepathExt = "%s/extUsers.json" % ghDataDir
-datfilepathInt = "%s/labUsers.json" % ghDataDir
+datfilepathInt = "%s/intUsers.json" % ghDataDir
 queryPath = "../queries/repo-Users.gql"
 
 # Read repo info data file (to use as repo list)
-inputLists = qm.DataManager("%s/labReposInfo.json" % ghDataDir, True)
+inputLists = qm.DataManager("%s/intReposInfo.json" % ghDataDir, True)
 # Populate repo list
 repolist = []
 print("Getting internal repos ...")
 repolist = sorted(inputLists.data["data"].keys())
 print("Repo list complete. Found %d repos." % (len(repolist)))
 
 # Initialize internal user data collector
-# and Read lab user data file (to use as member list)
+# and Read internal user data file (to use as member list)
 dataCollectorInt = qm.DataManager(datfilepathInt, True)
 # Populate member list
 memberlist = []
-print("Getting LLNL members ...")
+print("Getting internal members ...")
 memberlist = sorted(dataCollectorInt.data["data"].keys())
 print("Member list complete. Found %d users." % (len(memberlist)))
 
@@ -84,7 +84,7 @@
 print("\nCollective data gathering complete!")
 
 # Write output files
-dataCollectorExt.fileSave()
-dataCollectorInt.fileSave()
+dataCollectorExt.fileSave(newline="\n")
+dataCollectorInt.fileSave(newline="\n")
 
 print("\nDone!\n")

explore/popular-repos/index.html

@@ -54,7 +54,7 @@ <h1 class="page-header text-center">
         stdLgndDotRadius = 5,
         stdLgndSpacing = 20;
     // Call draw functions
-    var popularityURL = ghDataDir + '/labReposInfo.json';
+    var popularityURL = ghDataDir + '/intReposInfo.json';
     var popularityFiles = [popularityURL];
     var mostPopularRepositories = [];
     var cutOffSize = 10;

js/category-info.js

@@ -3,11 +3,11 @@ app.controller('gitHubDataController', function($scope, $http, $window, Category
         cache: true
     });
 
-    var getReposTopics = $http.get('./explore/github-data/labRepos_Topics.json', {
+    var getReposTopics = $http.get('./explore/github-data/intRepos_Topics.json', {
         cache: true
     });
 
-    var getReposInfo = $http.get('./explore/github-data/labReposInfo.json', {
+    var getReposInfo = $http.get('./explore/github-data/intReposInfo.json', {
         cache: true
     });
 

js/explore/cloud_languages.js

@@ -1,7 +1,7 @@
 /* Creates word cloud visualization for webpage */
 function draw_cloud_languages(areaID, repoNameWOwner) {
     // load data file, process data, and draw visualization
-    var url = ghDataDir + '/labRepos_Languages.json';
+    var url = ghDataDir + '/intRepos_Languages.json';
     var files = [url];
     Promise.all(files.map(url => d3.json(url))).then(values => {
         var data = reformatData(values[0]);