feat: github pr search

ENV_SAMPLE

@@ -1,2 +1,5 @@
 REPO_OWNER="Git-Commit-Show"
-GITHUB_PERSONAL_TOKEN="ghp_adsfdsf32sdfasdfcdcsdfsdf23sfasdf1"
+GITHUB_PERSONAL_TOKEN="ghp_adsfdsf32sdfasdfcdcsdfsdf23sfasdf1"
+APERTURE_SERVICE_ADDRESS="my.app.fluxninja.com:443"
+APERTURE_API_KEY="4dsfs323db7dsfsde310ca6dsdf12sfr4"
+MAX_RATE_LIMIT_WAIT_DURATION_MS=60000

archive.js

@@ -0,0 +1,61 @@
+import * as path from 'path';
+import * as fs from 'fs';
+
+/**
+ * Writes json data to a csv file
+ * @param {Array<Object>} data - The data to be written to the file.
+ * @param {string} [options.archiveFolder=process.cwd()] - The folder where the file will be saved. Defaults to the current working directory.
+ * @param {string} [options.archiveFileName='archive-YYYYMMDDHHmmss.csv'] - The name of the file to be written. Defaults to 'archive-YYYYMMDDHHmmss.csv'.
+ */
+export function save(data, options = {}) {
+    if (!data || !Array.isArray(data) || data.length<1) {
+        console.log("No content to write.");
+        return;
+    }
+    // Prepare content for csv
+    let allKeys = Array.from(new Set(data.flatMap(Object.keys)));    
+    const headers = allKeys.join(',');
+    const rows = data.map(obj => formatCSVRow(obj, allKeys)).join("\n");
+    const csvContent = headers + "\n" + rows;
+    writeToFile(csvContent, options);
+    return csvContent;
+}
+
+export function writeToFile(content, options){
+    const ARCHIVE_FOLDER = options.archiveFolder || process.cwd();
+    const ARCHIVE_FULL_PATH = path.join(ARCHIVE_FOLDER, options.archiveFileName || `archive-${getFormattedDate()}.csv`);
+    fs.writeFile(ARCHIVE_FULL_PATH, content, { flag: 'a+' }, err => {
+        if (err) {
+            console.error(err);
+            return;
+        }
+        console.log("The file was saved!");
+    });
+}
+
+function formatCSVRow(obj, keys) {
+    return keys.map(key => formatCSVCell(obj[key])).join(',');
+}
+
+function formatCSVCell(value) {
+    if (value === undefined || value === null) {
+        return '';
+    } else if (typeof value === 'object') {
+        // Stringify objects/arrays and escape double quotes
+        return `"${JSON.stringify(value).replace(/"/g, '""')}"`;
+    } else if (typeof value === 'string') {
+        // Check for commas or line breaks and escape double quotes
+        if (value.includes(',') || value.includes('\n')) {
+            return `"${value.replace(/"/g, '""')}"`;
+        } else {
+            return value;
+        }
+    } else {
+        return value.toString();
+    }
+}
+
+export function getFormattedDate() {
+    const now = new Date();
+    return now.toISOString().replace(/[\-\:T\.Z]/g, '');
+}

contributors.js → github.js

@@ -1,12 +1,10 @@
 /**
- * @file Functions to analyze and archive meaningful github contributors data
+ * @file Functions to analyze and archive meaningful github data such as contributors
  * @example To archive contributors leaderboard data in csv file, run `node contributors.js`
  */
 
-import * as path from 'path';
-import * as fs from 'fs';
-
 import { makeRequest, makeRequestWithRateLimit } from './network.js';
+import * as archive from './archive.js';
 
 // Configurations (Optional)
 // Repo owner that you want to analyze
@@ -26,6 +24,26 @@ if(GITHUB_PERSONAL_TOKEN){
     GITHUB_REQUEST_OPTIONS.headers["Authorization"] = "token "+GITHUB_PERSONAL_TOKEN;
 }
 
+
+/**
+ * Get details of a Github repo
+ * @param {string} fullRepoNameOrUrl e.g. myorghandle/myreponame
+ * @param {number} pageNo
+ * @returns Promise<Array<Object> | String>
+ * @example getRepoDetail('myorghandle/myreponame').then((repoDetail) => console.log(repoDetail)).catch((err) => console.log(err))
+ */
+export async function getRepoDetail(fullRepoNameOrUrl, pageNo = 1) {
+    if(!fullRepoNameOrUrl) throw new Error("Invalid input")
+    let fullRepoName = fullRepoNameOrUrl.match(/github\.com(?:\/repos)?\/([^\/]+\/[^\/]+)/)?.[1] || fullRepoNameOrUrl;
+    let url = `https://api.github.com/repos/${fullRepoName}`;
+    console.log(url);
+    const { res, data } = await makeRequestWithRateLimit('GET', url, Object.assign({},GITHUB_REQUEST_OPTIONS));
+    console.log("Repo detail request finished for " + fullRepoName)
+    // console.log(data)
+    let dataJson = JSON.parse(data);
+    return dataJson;
+}
+
 /**
  * Get all github repos of an owner(user/org)
  * @param {string} owner The organization or user name on GitHub
@@ -176,18 +194,12 @@ function writeContributorLeaderboardToFile(contributors, options={}) {
     if(!contributors || contributors.length<1){
         return;
     }
-    const ARCHIVE_FOLDER = options.archiveFolder || process.cwd();
-    const ARCHIVE_FULL_PATH = path.join(ARCHIVE_FOLDER, options.archiveFileName || 'archive-gh-contributors-leaderboard.csv');
+    // Prepare data
     let ghContributorLeaderboard = contributors.map((contributor) => {
         return ["@" + contributor.login, contributor.contributions, contributor.html_url, contributor.avatar_url, contributor.topContributedRepo, contributor.allContributedRepos].join();
     }).join("\n");
     ghContributorLeaderboard = "Github Username,Total Contributions,Profile,Avatar,Most Contribution To,Contributed To\n" + ghContributorLeaderboard;
-    fs.writeFile(ARCHIVE_FULL_PATH, ghContributorLeaderboard, { flag: 'a+' }, function (err) {
-        if (err) {
-            return console.log(err);
-        }
-        console.log("The file was saved!");
-    });
+    archive.writeToFile(ghContributorLeaderboard, Object.assign({ archiveFileName: 'archive-gh-contributors-leaderboard.csv' }, options));
 }
 
 /**
@@ -210,4 +222,100 @@ export async function archiveContributorsLeaderboard(owner=REPO_OWNER, options)
     writeContributorLeaderboardToFile(contributors);
 
     return ghHandles;
+}
+
+/**
+ * Search pull requests
+ * @param {string} query 
+ * @param {Object} [options] Additional options
+ * @param {Object} [options.pageNo=1] Result page number
+ */
+export async function searchPullRequests(query, options) {
+    let pageNo = (options && options.pageNo) ? options.pageNo : 1;
+    if(options && options.GITHUB_PERSONAL_TOKEN){
+        GITHUB_REQUEST_OPTIONS.headers["Authorization"] = "token "+options.GITHUB_PERSONAL_TOKEN;
+    }
+    let queryString = encodeURIComponent((query || ''))+'+is:pull-request';
+    let url = `https://api.github.com/search/issues?q=${queryString}&per_page=100&page=${pageNo}&sort=${options.sort || 'created'}`;
+    const { res, data } = await makeRequestWithRateLimit('GET', url, Object.assign({},GITHUB_REQUEST_OPTIONS));
+    console.log("PR search request finished");
+    console.log('HTTP status: ', res.statusCode);
+    // console.log(data)
+    let searchResultObject = JSON.parse(data);
+    return searchResultObject;
+}
+
+/**
+ * Get all PRs matching query
+ * @param {string} query 
+ * @param {Object} [options]
+ * @param {Object} [options.maxResults=1000] limit maximum results
+ */
+export async function recursivelySearchPullRequests(query, options){
+    let searchRequestOptions = Object.assign({ pageNo: 1, maxResults: 1000 }, options)
+    let prList = [];
+    let searchResultObject = await searchPullRequests(query, searchRequestOptions);
+    // Iterate over results if there are more results expected by the user
+    if(!searchResultObject || !searchResultObject.items || searchResultObject.items.length<1){
+        return prList;
+    }
+    prList.push(...searchResultObject.items);
+    while(prList.length < searchRequestOptions.maxResults && prList.length < searchResultObject.total_count){
+        searchRequestOptions.pageNo++;
+        try {
+            let nextPageSearchResultObject = await searchPullRequests(query, searchRequestOptions);
+            prList.push(...nextPageSearchResultObject.items);
+        } catch (err) {
+            console.log("Some issue in recursive search for pull requests");
+            break;
+        }
+    }
+    console.log("Found "+prList.length +" PRs"+" for "+query);
+    return prList;
+}
+
+
+/**
+ * Aggregates all pull requests based on a specified field
+ * @param {Object[]} pullRequests - An array of pull request objects.
+ * @param {string} aggregatorField - The field name used to aggregate the pull requests. Defaults to "repository_url".
+ * @returns {Object[]} An array of objects, each containing a unique value of the aggregator field and an array of all pull requests that share that value.
+ */
+export function aggregateAllPullRequests(pullRequests, aggregatorField = "repository_url") {
+    return pullRequests.reduce((grouped, currentItem) => {
+        // Skipping the items without aggregatorField
+        if (!currentItem[aggregatorField]) {
+            return grouped;
+        }
+        // Find or create the group for the current item
+        let group = grouped.find(g => g[aggregatorField] === currentItem[aggregatorField]);
+        if (!group) {
+            group = { [aggregatorField]: currentItem[aggregatorField], pull_requests: [] };
+            grouped.push(group);
+        }
+        // Add the current item to the group
+        group.pull_requests.push(currentItem);
+        return grouped;
+    }, []);
+}
+
+/**
+ * Archives repos that PRs
+ * @param {string} owner The organization or user name on GitHub
+ * @param {Object} options Additional options
+ */
+export async function archiveReposWithMatchingPullRequests(query, options) {
+    let pullRequests = await recursivelySearchPullRequests(query, options);
+    if (!pullRequests || pullRequests.length < 1) {
+        console.log("Failed to get PRs for query: "+query);
+        return;
+    }
+    let repos = aggregateAllPullRequests(pullRequests, 'repository_url');
+    if(!repos) throw new Error("No repo found");
+    for(let repo of repos){
+        let repoDetail = await getRepoDetail(repo['repository_url']);
+        Object.assign(repo, repoDetail);
+    }
+    archive.save(repos, { archiveFileName: `repos-pr-${query}-${options.maxResults || 1000}-${archive.getFormattedDate()}.csv` });
+    return repos;
 }

index.js

@@ -1,4 +1,4 @@
-import * as contributorsLib from './contributors.js';
+import * as contributorsLib from './github.js';
 
 /**
  * Bundling all APIs together

network.js

@@ -1,6 +1,7 @@
 import * as https from 'https';
-import { ApertureClient } from "@fluxninja/aperture-js";
+import { ApertureClient, FlowStatus } from "@fluxninja/aperture-js";
 
+const MAX_RATE_LIMIT_WAIT_DURATION_MS = process.env.MAX_RATE_LIMIT_WAIT_DURATION_MS || 60000; // 10 mins
 
 var apertureClient;
 
@@ -47,7 +48,9 @@ export async function makeRequest(method, url, requestOptions) {
             // Handle HTTP response stream
             let data = '';
             res.on('data', chunk => data += chunk);
-            res.on('end', () => resolve({ res, data }));
+            res.on('end', function(){
+              resolve({ res, data })
+          });
         });
 
         req.on('error', error => {
@@ -92,17 +95,18 @@ export async function makeRequest(method, url, requestOptions) {
  * }
  * */
 export async function makeRequestWithRateLimit(method, url, options){
+  if(!method) method = "GET";
   let flow;
   try {
     flow = await getApertureClient().startFlow("external-api", {
       labels: {
         url: url,
         priority: 1,
-        tokens: 1,
+        tokens: ["POST", "PUT", "DELETE", "PATCH"].includes(method.toUpperCase()) ? 5 : 1,
         workload: 'api.github'
       },
       grpcCallOptions: {
-        deadline: Date.now() + 300, // ms
+        deadline: Date.now() + MAX_RATE_LIMIT_WAIT_DURATION_MS, // ms
       },
     });
   } catch(err){
@@ -115,6 +119,8 @@ export async function makeRequestWithRateLimit(method, url, options){
     if (!flow || flow.shouldRun()) {
       // Add business logic to process incoming request
       console.log("Request accepted. Processing...");
+      // Wait for 500ms
+      await new Promise(resolve => setTimeout(resolve, 500));
       const {res, data} = await makeRequest(...arguments)
       return { res, data}
     } else {

package.json

@@ -2,7 +2,7 @@
   "name": "open-community-kit",
   "version": "1.2.1",
   "description": "Tools and stats for open-source communities",
-  "main": "contributors.js",
+  "main": "index.js",
   "type": "module",
   "bin": {
     "ock": "cli.js",

test/archive.test.js

@@ -0,0 +1,20 @@
+import { expect, assert } from "chai";
+import * as archive from "../archive.js";
+
+import * as pullRequestsFixture from './fixtures/pullRequests.fixture.js';
+
+describe('archive.js', function() {
+
+    /** Archive test --START-- */
+
+    describe.skip('#archive(jsonArray);', async function() {
+      it('should save jsonArray to a csv file', async function() {
+        this.timeout(100000);
+        let content = await archive.save(pullRequestsFixture.VALID_PR_SEARCH_RESULT_ITEMS);
+        assert.isNotNull(content, "Repos not returned");
+        expect(content).to.be.an('string');
+        expect(content).to.have.lengthOf.greaterThan(2000);
+      })
+    })
+
+})