Fix metrics code gov (#15)

* Tweak * Revert traffic metrics data and add try catch block for error handling on null values or more pages * Automatic update * Add new repositories * Automatic update * Update index.js * Add issues similar try catch logic to pull request * Automatic update * fix null * Automatic update --------- Co-authored-by: SaikrishnaBairamoni <SaikrishnaBairamoni@users.noreply.github.com> Co-authored-by: dan-du-car <dan.du@leidos.com>
usdot-fhwa-stol · Sep 6, 2024 · 21692e9 · 21692e9
1 parent 37f30f9
commit 21692e9
Show file tree

Hide file tree

Showing 9 changed files with 300 additions and 87 deletions.
diff --git a/.github/workflows/code-gov.yml b/.github/workflows/code-gov.yml
@@ -13,15 +13,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Install code-gov
         run: |
           cd code-gov/code-gov-repo-metrics
           npm install
-          echo "GITHUB_PERSONAL_ACCESS_TOKEN=${{ secrets.METRICS_GITHUB_TOKEN }}" > .env
+          echo "METRICS_GITHUB_TOKEN=${{ secrets.METRICS_GITHUB_TOKEN }}" > .env
 
       - name: Run code-gov
         run: |
@@ -40,7 +40,7 @@ jobs:
         run: |
           rm code-gov/code-gov-repo-metrics/{config.json,.env}
 
-      - uses: stefanzweifel/git-auto-commit-action@v4
+      - uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: "Automatic update"
           commit_user_name: "Saikrishna Bairamoni"

diff --git a/.github/workflows/traffic-metrics.yml b/.github/workflows/traffic-metrics.yml
@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -28,23 +28,16 @@ jobs:
           python -m pip install -r traffic-metrics/requirements.txt
           python -m pip install pandas  # Ensure pandas is installed
 
-      - name: Run export and GitHub metrics scripts
+      - name: Run traffic-metrics
         run: |
-          for ORG in ${{ env.ORGS }}; do
+          for ORG in $ORGS; do
             python3 traffic-metrics/export_traffic.py -t ${{ secrets.METRICS_GITHUB_TOKEN }} -o "$ORG"
             python3 traffic-metrics/github_metrics.py -t ${{ secrets.METRICS_GITHUB_TOKEN }} -o "$ORG"
           done
 
-      - name: Aggregate Metrics
-        run: python3 traffic-metrics/aggregate_metrics.py
-
-      - name: List aggregated output files
-        run: ls -l ./aggregated_output 
-
       - name: Commit Files
-        uses: stefanzweifel/git-auto-commit-action@v4
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit_message: "Update aggregated metrics"
+          commit_message: "Update Traffic metrics"
           commit_user_name: "Saikrishna Bairamoni"
-          commit_user_email: "saikrishna.bairamoni@leidos.com"
-          file_pattern: "./aggregated_output/*.csv"  
+          commit_user_email: "saikrishna.bairamoni@leidos.com"
diff --git a/code-gov/code-gov-repo-metrics/index.js b/code-gov/code-gov-repo-metrics/index.js
@@ -27,7 +27,7 @@ async function queryGitHub(repoName) {
     // Create a graphQLClient
     const graphQLClient = new GraphQLClient(endpoint, {
         headers: {
-            authorization: 'Bearer ' + process.env.GITHUB_PERSONAL_ACCESS_TOKEN,
+            authorization: 'Bearer ' + process.env.METRICS_GITHUB_TOKEN,
         },
     });
 
@@ -43,6 +43,12 @@ async function queryGitHub(repoName) {
     // Request the data
     const dataJSON = await graphQLClient.request(query, variables);
 
+    // Handle case where repository data is null
+    if (!dataJSON.repository) {
+        console.error(`Failed to retrieve repository data for ${repoName}.`);
+        return null;
+    }
+
     // If the repo has more than 100 issues, get the rest of the issues
     if (dataJSON.repository.issues.pageInfo.hasNextPage) {
         var issues = await queryIssuesDeep(repoName, dataJSON.repository.issues.pageInfo.endCursor, dataJSON.repository.issues.nodes);
@@ -74,7 +80,7 @@ async function queryIssuesDeep(repoName, cursor, issues) {
     // Create a graphQLClient
     const graphQLClient = new GraphQLClient(endpoint, {
         headers: {
-            authorization: 'Bearer ' + process.env.GITHUB_PERSONAL_ACCESS_TOKEN,
+            authorization: 'Bearer ' + process.env.METRICS_GITHUB_TOKEN,
         },
     });
 
@@ -89,14 +95,25 @@ async function queryIssuesDeep(repoName, cursor, issues) {
     };
 
     // Request the additional issues
-    const dataJSON = await graphQLClient.request(query, variables);
+    try {
+        const dataJSON = await graphQLClient.request(query, variables);
 
-    // Push the new issues to the running issue list
-    dataJSON.repository.issues.nodes.forEach(issue => {issues.push(issue)});
+        // Handle case where repository or issues data is null
+        if (!dataJSON.repository || !dataJSON.repository.issues) {
+            console.error(`Failed to retrieve issues for ${repoName}.`);
+            return issues;
+        }
 
-    // Recurse if there are still more issues
-    if (dataJSON.repository.issues.pageInfo.hasNextPage) {
-        return await queryIssuesDeep(repoName, dataJSON.repository.issues.pageInfo.endCursor, issues);
+        // Push the new issues to the running issue list
+        dataJSON.repository.issues.nodes.forEach(issue => { issues.push(issue) });
+
+        // Recurse if there are still more issues
+        if (dataJSON.repository.issues.pageInfo.hasNextPage) {
+            return await queryIssuesDeep(repoName, dataJSON.repository.issues.pageInfo.endCursor, issues);
+        }
+
+    } catch (error) {
+        console.error(`Error querying issues for ${repoName} with cursor ${cursor}:`, error);
     }
 
     return issues;
@@ -118,7 +135,7 @@ async function queryPullRequestsDeep(repoName, cursor, pullRequests) {
     // Create a graphQLClient
     const graphQLClient = new GraphQLClient(endpoint, {
         headers: {
-            authorization: 'Bearer ' + process.env.GITHUB_PERSONAL_ACCESS_TOKEN,
+            authorization: 'Bearer ' + process.env.METRICS_GITHUB_TOKEN,
         },
     });
 
@@ -133,14 +150,25 @@ async function queryPullRequestsDeep(repoName, cursor, pullRequests) {
     };
 
     // Request the additional pull requests
-    const dataJSON = await graphQLClient.request(query, variables);
+    try {
+        const dataJSON = await graphQLClient.request(query, variables);
 
-    // Push the new pull requests to the running pull requests list
-    dataJSON.repository.pullRequests.nodes.forEach(pullRequest => {pullRequests.push(pullRequest)});
+        // Handle case where repository or pull requests data is null
+        if (!dataJSON.repository || !dataJSON.repository.pullRequests) {
+            console.error(`Failed to retrieve pull requests for ${repoName}.`);
+            return pullRequests;
+        }
 
-    // Recurse if there are still more pull requests
-    if (dataJSON.repository.pullRequests.pageInfo.hasNextPage) {
-        return await queryIssuesDeep(repoName, dataJSON.repository.pullRequests.pageInfo.endCursor, pullRequests);
+        // Push the new pull requests to the running pull requests list
+        dataJSON.repository.pullRequests.nodes.forEach(pullRequest => { pullRequests.push(pullRequest) });
+
+        // Recurse if there are still more pull requests
+        if (dataJSON.repository.pullRequests.pageInfo.hasNextPage) {
+            return await queryPullRequestsDeep(repoName, dataJSON.repository.pullRequests.pageInfo.endCursor, pullRequests);
+        }
+
+    } catch (error) {
+        console.error(`Error querying pull requests for ${repoName} with cursor ${cursor}:`, error);
     }
 
     return pullRequests;
@@ -156,6 +184,15 @@ async function queryPullRequestsDeep(repoName, cursor, pullRequests) {
  * @return {JSON} a JSON of metrics calculated for repo
  */
 function processRepo(repo) {
+
+    // Skip processing if repo is null
+    if (!repo) {
+        console.error("Skipping processing due to missing repository data.");
+        return null;
+    }
+
+    console.log(repo);
+    console.log("processRepo");
     // Set up
     var issueMetaData = getIssueMetaData(repo);
     var pullRequestMetaData = getPullRequestMetaData(repo);
@@ -508,9 +545,9 @@ function getPullRequestMetaData(repo) {
  */
 function aggregateRepoData(repos) {
     // Set up
-    var openIssues = utils.sumList(repos.map(repo => repo.openIssues));
-    var staleIssues = utils.sumList(repos.map(repo => repo.staleIssues));
-    var oldIssues = utils.sumList(repos.map(repo => repo.oldIssues));
+    var openIssues = utils.sumList(repos.map(repo => repo?.openIssues));
+    var staleIssues = utils.sumList(repos.map(repo => repo?.staleIssues));
+    var oldIssues = utils.sumList(repos.map(repo => repo?.oldIssues));
 
     // Make JSON of aggregate processed data
     var totalData = {

diff --git a/code-gov/code-gov-repo-metrics/usdot-fhwa-OPS.config.json b/code-gov/code-gov-repo-metrics/usdot-fhwa-OPS.config.json
@@ -1,6 +1,11 @@
 {
   "owner": "usdot-fhwa-OPS",
   "repoList": [
-    "V2X-Hub"
+    "V2X-Hub",
+    "v2xhub-angular-ui-prototype",
+    "ITS-Secure-Prototype-Frontend",
+    "ITS-Secure-Prototype-Backend",
+    "ITS-Secure-Prototype-App",
+    "sample_angular_saml_app"
   ]
 }
diff --git a/code-gov/code-gov-repo-metrics/usdot-fhwa-stol.config.json b/code-gov/code-gov-repo-metrics/usdot-fhwa-stol.config.json
@@ -1,59 +1,101 @@
 {
   "owner": "usdot-fhwa-stol",
   "repoList": [
-    "CARMAConcept",
     "WxDE",
-    "ads-traffic-regs",
-    "arena_camera_ros",
-    "autoware.ai",
-    "autoware.auto",
-    "avt_vimba_camera",
-    "c1t_razor_imu_m0_driver",
-    "c1t_rplidar_driver",
-    "c1t_vesc_driver",
-    "c1t_zed_driver",
-    "carla-simulation",
-    "carma-1-tenth",
-    "carma-analytics-fotda",
-    "carma-base",
-    "carma-cadillac-srx-2013-controller-driver",
-    "carma-carla-integration",
     "carma-cloud",
+    "carma-platform",
+    "CARMAConcept",
+    "carma-msgs",
+    "carma-utils",
+    "carma-torc-pinpoint-driver",
+    "carma-torc-xgv-controller-driver",
+    "carma-cadillac-srx-2013-controller-driver",
+    "carma-freightliner-2012-controller-driver",
     "carma-cohda-dsrc-driver",
-    "carma-config",
+    "carma-web-ui",
     "carma-delphi-esr-driver",
     "carma-delphi-srr2-driver",
-    "carma-freightliner-2012-controller-driver",
+    "avt_vimba_camera",
+    "carma-vehicle-model-framework",
+    "novatel_gps_driver",
+    "carma-velodyne-lidar-driver",
+    "autoware.ai",
+    "carma-ssc-interface-wrapper",
+    "carma-base",
+    "carma-config",
     "carma-garmin-lidar-lite-v3-driver-wrapper",
-    "carma-lightbar-driver",
-    "carma-message-filters",
     "carma-messenger",
-    "carma-mobileye-driver",
-    "carma-msgs",
-    "carma-novatel-oem7-driver-wrapper",
-    "carma-platform",
+    "carma-1-tenth",
+    "opendrive2lanelet",
+    "carma-lightbar-driver",
     "carma-rtk",
-    "carma-simulation",
-    "carma-ssc-interface-wrapper",
-    "carma-streets",
-    "carma-torc-pinpoint-driver",
-    "carma-torc-xgv-controller-driver",
-    "carma-utils",
-    "carma-validation",
-    "carma-vehicle-model-framework",
-    "carma-velodyne-lidar-driver",
-    "carma-web-ui",
+    "carla-simulation",
+    "cdasim",
+    "ads-traffic-regs",
     "cav-education",
-    "github_metrics",
+    "carma-analytics-fotda",
+    "carma-validation",
+    "carma-streets",
     "mosaic",
-    "novatel_gps_driver",
-    "opendrive2lanelet",
-    "ros1_bridge",
-    "rosbridge_suite",
     "sumo",
-    "testing-training-tools",
+    "c1t_vesc_driver",
+    "c1t_razor_imu_m0_driver",
+    "c1t_rplidar_driver",
+    "c1t_zed_driver",
+    "arena_camera_ros",
+    "carma-mobileye-driver",
+    "autoware.auto",
     "tim-bc",
+    "carma-novatel-oem7-driver-wrapper",
     "voices-cda-use-case-scenario-database",
-    "voices-poc"
+    "voices-poc",
+    "ros1_bridge",
+    "testing-training-tools",
+    "cda-telematics",
+    "carma-carla-integration",
+    "github_metrics",
+    "carma-message-filters",
+    "rosbridge_suite",
+    "c2c-ri",
+    "devops",
+    "documentation",
+    "dwm1001_ros2",
+    "c1t-tools",
+    "multiple_object_tracking",
+    "c1t2x-emulator",
+    "carma-builds",
+    ".github",
+    "carma-j2735",
+    "snmp-client",
+    "actions",
+    "carma-time-lib",
+    "cavams",
+    "carma-ns3-adapter",
+    "voices-common-interface",
+    "rosbag2",
+    "novatel_oem7_driver",
+    "ns3-federate",
+    "ns-3_c-v2x",
+    "carma-geodetic-lib",
+    "ros2_tracing",
+    "tracetools_analysis",
+    "carma-slamtec-lidar-driver",
+    "carma_ament_lint",
+    "scenario-runner",
+    "carma-vesc-ssc-wrapper",
+    "carla-sensor-lib",
+    "cave-lite",
+    "j2735decoder",
+    "vesc",
+    "Stol-scratchpad",
+    "c1t_bringup",
+    "connectedvcs-tools",
+    "navigation2",
+    "usdot-asn1c",
+    "navigation2_extensions",
+    "twist_to_ackermann",
+    "robot_localization",
+    "To-21-426-modeling-HV-interactions-with-inconspicuous-ACC-equipped-vehicles",
+    "To-21-426-multivariate-piecewise-linear-ACC-car-following-model"
   ]
 }