From faf2f91a87deb1cc3bac939dbe4a07b2fa01bc65 Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Fri, 5 Apr 2024 11:23:09 +0530 Subject: [PATCH 1/7] initial commit --- app/feed/page.tsx | 4 ++++ scraper/src/github.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/app/feed/page.tsx b/app/feed/page.tsx index 6b5b458b..c69923a3 100644 --- a/app/feed/page.tsx +++ b/app/feed/page.tsx @@ -29,6 +29,10 @@ export default async function FeedPage({ searchParams }: Props) { if (!Object.entries(events).length) { return ; } + console.log( + "Fetched Events", + events.map((e) => console.log(e.type)), + ); return (

Feed

diff --git a/scraper/src/github.py b/scraper/src/github.py index 797a5899..3c012972 100755 --- a/scraper/src/github.py +++ b/scraper/src/github.py @@ -159,6 +159,17 @@ def parse_event(self, event, event_time): }, ) + elif event["type"] == "PushEvent": + self.append( + user, + { + "type": "commit_direct", + "time": event_time, + "title": f'{event["repo"]["name"]}#{event["payload"]["commit"]["sha"]}', + "link": event["payload"]["commit"]["html_url"], + "text": event["payload"]["commit"]["message"], + }, + ) def add_collaborations(self, event, event_time): collaborators = set() @@ -344,6 +355,8 @@ def fetch_events(self, page=1): return self.fetch_events(int(next_page)) return self.data + # def fetch_direct_commits + def fetch_open_pulls(self, user): self.log.debug(f"Fetching open pull requests for {user}") resp = requests.get( From 2933b699b055ec322b819d554833208441c02b0e Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Sun, 7 Apr 2024 10:42:31 +0530 Subject: [PATCH 2/7] Added direct_commit fetaure in scrapper and UI --- app/feed/page.tsx | 5 +-- components/contributors/GithubActivity.tsx | 28 ++++++++++++++ lib/api.ts | 5 ++- lib/types.ts | 2 + scraper/src/github.py | 44 +++++++++++++++------- 5 files changed, 65 insertions(+), 19 deletions(-) diff --git a/app/feed/page.tsx b/app/feed/page.tsx index c69923a3..d079833a 100644 --- a/app/feed/page.tsx +++ b/app/feed/page.tsx @@ -29,10 +29,7 @@ export default async function FeedPage({ searchParams }: Props) { if (!Object.entries(events).length) { return ; } - console.log( - "Fetched Events", - events.map((e) => console.log(e.type)), - ); + return (

Feed

diff --git a/components/contributors/GithubActivity.tsx b/components/contributors/GithubActivity.tsx index 3e7cca76..d2fe1830 100644 --- a/components/contributors/GithubActivity.tsx +++ b/components/contributors/GithubActivity.tsx @@ -161,6 +161,33 @@ let renderText = (activity: Activity) => {
); + case "commit_direct": + return ( +
+
+

+ {"Direct Commit to "} + {activity["branch"]} + {" in "} + + {activity["link"].split("/").slice(3, 5).join("/")} + + + + {" "} + + +

+
+
+ + + {activity["text"]} + + +
+
+ ); default: return (
@@ -480,6 +507,7 @@ export const ActivityCheckbox = (props: { pr_merged: "PR merged", pr_opened: "PR opened", pr_reviewed: "Code Review", + commit_direct: "Direct Commit", }[props.type] } diff --git a/lib/api.ts b/lib/api.ts index 2bebe85b..106e4302 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -7,7 +7,7 @@ import { existsSync } from "fs"; const root = join(process.cwd(), "data-repo/contributors"); const slackRoot = join(process.cwd(), "data-repo/data/slack"); -const githubRoot = join(process.cwd(), "data-repo/data/github"); +const githubRoot = join(process.cwd(), "data-repo/github"); const points = { comment_created: 1, @@ -19,6 +19,7 @@ const points = { pr_merged: 7, pr_collaborated: 2, issue_closed: 0, + commit_direct: 2, }; // Comments will get a single point // Picking up an issue would get a point @@ -26,7 +27,7 @@ const points = { // Finding a bug would add up to 4 points // Opening a PR would give a single point and merging it would give you the other 7 points, making 8 per PR // Updating the EOD would get 2 points per day and additional 20 for regular daily updates plus 10 for just missing one - +// Push direct commmmit have two points function formatSlug(slug: string) { return slug.replace(/\.md$/, ""); } diff --git a/lib/types.ts b/lib/types.ts index a9959c99..d47d3c71 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -66,6 +66,7 @@ export const ACTIVITY_TYPES = [ "pr_opened", "pr_merged", "pr_collaborated", + "commit_direct", ] as const; export interface Activity { @@ -74,6 +75,7 @@ export interface Activity { time: string; link: string; text: string; + branch?: string; collaborated_with?: string[]; turnaround_time?: number; } diff --git a/scraper/src/github.py b/scraper/src/github.py index 3c012972..d9c65cb2 100755 --- a/scraper/src/github.py +++ b/scraper/src/github.py @@ -26,6 +26,7 @@ "codecov-commenter", } +default_branch_repos = {} def is_blacklisted(login: str): return "[bot]" in login or login in user_blacklist @@ -70,7 +71,18 @@ def append(self, user, event): "open_prs": [], "authored_issue_and_pr": [], } - + + def get_default_banch(self, repoOwner, reponame): + if reponame in default_branch_repos: + return default_branch_repos[reponame] + + response = requests.get(f"https://api.github.com/repos/{repoOwner}/{reponame}") + response_json = response.json() + defaultBranchName = response_json["default_branch"] + default_branch_repos[reponame] = defaultBranchName + + return defaultBranchName + def parse_event(self, event, event_time): user = event["actor"]["login"] try: @@ -160,16 +172,24 @@ def parse_event(self, event, event_time): ) elif event["type"] == "PushEvent": - self.append( - user, - { - "type": "commit_direct", - "time": event_time, - "title": f'{event["repo"]["name"]}#{event["payload"]["commit"]["sha"]}', - "link": event["payload"]["commit"]["html_url"], - "text": event["payload"]["commit"]["message"], - }, - ) + repoOwner = "coronasafe" + reponame = event["repo"]["name"].split('/')[-1] + branch = event["payload"]["ref"].split('/')[-1] + commits = event["payload"]["commits"] + defaultBranchName = self.get_default_banch(repoOwner, reponame) + for commit in commits: + self.append( + user, + { + "type": "commit_direct", + "time": event_time, + "title": f'{event["repo"]["name"]}#{commit["sha"]}', + "link": commit["url"], + "text": commit["message"], + "branch": f'{branch} (default)' if branch == defaultBranchName else branch, + }, + ) + def add_collaborations(self, event, event_time): collaborators = set() @@ -355,8 +375,6 @@ def fetch_events(self, page=1): return self.fetch_events(int(next_page)) return self.data - # def fetch_direct_commits - def fetch_open_pulls(self, user): self.log.debug(f"Fetching open pull requests for {user}") resp = requests.get( From 7c0bbf1c01ba806ea0c92e0248272aa170cd1322 Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Mon, 8 Apr 2024 15:48:43 +0530 Subject: [PATCH 3/7] commit array storing --- scraper/src/github.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scraper/src/github.py b/scraper/src/github.py index d9c65cb2..bd05bfac 100755 --- a/scraper/src/github.py +++ b/scraper/src/github.py @@ -176,19 +176,25 @@ def parse_event(self, event, event_time): reponame = event["repo"]["name"].split('/')[-1] branch = event["payload"]["ref"].split('/')[-1] commits = event["payload"]["commits"] - defaultBranchName = self.get_default_banch(repoOwner, reponame) + # defaultBranchName = self.get_default_banch(repoOwner, reponame) + all_commits = [] for commit in commits: - self.append( - user, - { - "type": "commit_direct", - "time": event_time, + commit_data = { "title": f'{event["repo"]["name"]}#{commit["sha"]}', "link": commit["url"], "text": commit["message"], - "branch": f'{branch} (default)' if branch == defaultBranchName else branch, - }, - ) + } + all_commits.append(commit_data) # Append commit data to the array + print(all_commits) + self.append( + user, + { + "type": "commit_direct", + "time": event_time, + "title": f'Pushed({len(commits)}) to {event["repo"]["name"]}({branch})', + "commits": all_commits, + }, + ) def add_collaborations(self, event, event_time): collaborators = set() From 114ba056581419edaed006f0213f075ed5a7d00f Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Tue, 9 Apr 2024 15:01:53 +0530 Subject: [PATCH 4/7] Update scrapper to track direct commits to branch and show in contrbutor's page --- components/contributors/GithubActivity.tsx | 33 ++++++++++++---------- lib/api.ts | 4 +-- lib/types.ts | 9 ++++-- scraper/src/github.py | 25 ++++++++++------ 4 files changed, 43 insertions(+), 28 deletions(-) diff --git a/components/contributors/GithubActivity.tsx b/components/contributors/GithubActivity.tsx index d2fe1830..ef4d1273 100644 --- a/components/contributors/GithubActivity.tsx +++ b/components/contributors/GithubActivity.tsx @@ -8,6 +8,7 @@ import { usePathname, useRouter, useSearchParams } from "next/navigation"; import RelativeTime from "../RelativeTime"; import DateRangePicker from "../DateRangePicker"; import { format } from "date-fns"; +import Link from "next/link"; let commentTypes = (activityEvent: string[]) => { switch (activityEvent[0]) { @@ -161,33 +162,34 @@ let renderText = (activity: Activity) => {
); - case "commit_direct": + case "pushed_commits": return (

- {"Direct Commit to "} - {activity["branch"]} - {" in "} - - {activity["link"].split("/").slice(3, 5).join("/")} - - + {activity.title} - {" "} - +

- - - {activity["text"]} - - + {activity?.commits?.map((commit, index) => ( +
+ + + {index + 1}. + {" "} + + {commit.text} + + +
+ ))}
); + default: return (
@@ -508,6 +510,7 @@ export const ActivityCheckbox = (props: { pr_opened: "PR opened", pr_reviewed: "Code Review", commit_direct: "Direct Commit", + pushed_commits: "Pushed Commits", }[props.type] } diff --git a/lib/api.ts b/lib/api.ts index 106e4302..9a806528 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -19,7 +19,7 @@ const points = { pr_merged: 7, pr_collaborated: 2, issue_closed: 0, - commit_direct: 2, + pushed_commits: 0, }; // Comments will get a single point // Picking up an issue would get a point @@ -27,7 +27,7 @@ const points = { // Finding a bug would add up to 4 points // Opening a PR would give a single point and merging it would give you the other 7 points, making 8 per PR // Updating the EOD would get 2 points per day and additional 20 for regular daily updates plus 10 for just missing one -// Push direct commmmit have two points +// pushed commits directly to the repository would not be counted as of now function formatSlug(slug: string) { return slug.replace(/\.md$/, ""); } diff --git a/lib/types.ts b/lib/types.ts index d47d3c71..2b35b349 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -66,16 +66,21 @@ export const ACTIVITY_TYPES = [ "pr_opened", "pr_merged", "pr_collaborated", - "commit_direct", + "pushed_commits", ] as const; +export interface commit { + link: string; + text: string; +} + export interface Activity { type: (typeof ACTIVITY_TYPES)[number]; title: string; time: string; link: string; text: string; - branch?: string; + commits?: commit[]; collaborated_with?: string[]; turnaround_time?: number; } diff --git a/scraper/src/github.py b/scraper/src/github.py index bd05bfac..6ae095c7 100755 --- a/scraper/src/github.py +++ b/scraper/src/github.py @@ -172,30 +172,37 @@ def parse_event(self, event, event_time): ) elif event["type"] == "PushEvent": - repoOwner = "coronasafe" - reponame = event["repo"]["name"].split('/')[-1] branch = event["payload"]["ref"].split('/')[-1] commits = event["payload"]["commits"] - # defaultBranchName = self.get_default_banch(repoOwner, reponame) all_commits = [] + + # Iterate through each commit for commit in commits: + # Fetch full commit details + commit_details = requests.get(commit["url"], headers=self.headers).json() + + if len(commit_details.get("parents", [])) in [0, 1]: + # Prepare commit data commit_data = { - "title": f'{event["repo"]["name"]}#{commit["sha"]}', - "link": commit["url"], + "link": commit_details["html_url"], "text": commit["message"], } - all_commits.append(commit_data) # Append commit data to the array - print(all_commits) + # Append commit data to the array + all_commits.append(commit_data) + + # Append pushed commits event self.append( user, { - "type": "commit_direct", + "type": "pushed_commits", "time": event_time, - "title": f'Pushed({len(commits)}) to {event["repo"]["name"]}({branch})', + "title": f'Pushed({len(all_commits)}) commits to {event["repo"]["name"]}({branch})', "commits": all_commits, }, ) + + def add_collaborations(self, event, event_time): collaborators = set() From 519acbd4fa2cdd87883dff0bb2dfc30489c87bbe Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Tue, 9 Apr 2024 15:13:45 +0530 Subject: [PATCH 5/7] Delete testing changes --- lib/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api.ts b/lib/api.ts index 9a806528..2962384a 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -7,7 +7,7 @@ import { existsSync } from "fs"; const root = join(process.cwd(), "data-repo/contributors"); const slackRoot = join(process.cwd(), "data-repo/data/slack"); -const githubRoot = join(process.cwd(), "data-repo/github"); +const githubRoot = join(process.cwd(), "data-repo/data/github"); const points = { comment_created: 1, From 754242f6d5b98674f2b2bb7c70b544ff96c9fd24 Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Thu, 11 Apr 2024 11:34:12 +0530 Subject: [PATCH 6/7] Modify the way to displaying of commit --- components/contributors/GithubActivity.tsx | 15 +++++++++------ lib/types.ts | 7 +++++-- scraper/src/github.py | 7 +++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/components/contributors/GithubActivity.tsx b/components/contributors/GithubActivity.tsx index ef4d1273..6bfdb807 100644 --- a/components/contributors/GithubActivity.tsx +++ b/components/contributors/GithubActivity.tsx @@ -167,7 +167,8 @@ let renderText = (activity: Activity) => {

- {activity.title} + pushed({activity?.commits?.length}) commits to {activity.branch}{" "} + in {activity.repo}{" "} @@ -176,12 +177,14 @@ let renderText = (activity: Activity) => {

{activity?.commits?.map((commit, index) => (
- - - {index + 1}. - {" "} + + {commit.sha} - {commit.text} + {commit.text.split("\n")[0]}
diff --git a/lib/types.ts b/lib/types.ts index 2b35b349..dd58a156 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -69,9 +69,10 @@ export const ACTIVITY_TYPES = [ "pushed_commits", ] as const; -export interface commit { +export interface Commit { link: string; text: string; + sha: string; } export interface Activity { @@ -80,7 +81,9 @@ export interface Activity { time: string; link: string; text: string; - commits?: commit[]; + branch?: string; + repo?: string; + commits?: Commit[]; collaborated_with?: string[]; turnaround_time?: number; } diff --git a/scraper/src/github.py b/scraper/src/github.py index 6ae095c7..4776a8a9 100755 --- a/scraper/src/github.py +++ b/scraper/src/github.py @@ -181,11 +181,12 @@ def parse_event(self, event, event_time): # Fetch full commit details commit_details = requests.get(commit["url"], headers=self.headers).json() - if len(commit_details.get("parents", [])) in [0, 1]: + if len(commit_details.get("parents", [])) < 2: # Prepare commit data commit_data = { "link": commit_details["html_url"], "text": commit["message"], + "sha" : commit["sha"][-7:], } # Append commit data to the array all_commits.append(commit_data) @@ -196,7 +197,9 @@ def parse_event(self, event, event_time): { "type": "pushed_commits", "time": event_time, - "title": f'Pushed({len(all_commits)}) commits to {event["repo"]["name"]}({branch})', + # "title": f'Pushed({len(all_commits)}) commits to {event["repo"]["name"]}({branch})', + "repo": event["repo"]["name"], + "branch": branch, "commits": all_commits, }, ) From 3605fc8f2acbde6f7a0e733b2626731aaeec47dd Mon Sep 17 00:00:00 2001 From: dhrumit parmar Date: Fri, 19 Apr 2024 17:28:03 +0530 Subject: [PATCH 7/7] remove unused code --- scraper/src/github.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scraper/src/github.py b/scraper/src/github.py index 4776a8a9..b020a3b6 100755 --- a/scraper/src/github.py +++ b/scraper/src/github.py @@ -71,17 +71,6 @@ def append(self, user, event): "open_prs": [], "authored_issue_and_pr": [], } - - def get_default_banch(self, repoOwner, reponame): - if reponame in default_branch_repos: - return default_branch_repos[reponame] - - response = requests.get(f"https://api.github.com/repos/{repoOwner}/{reponame}") - response_json = response.json() - defaultBranchName = response_json["default_branch"] - default_branch_repos[reponame] = defaultBranchName - - return defaultBranchName def parse_event(self, event, event_time): user = event["actor"]["login"] @@ -197,15 +186,12 @@ def parse_event(self, event, event_time): { "type": "pushed_commits", "time": event_time, - # "title": f'Pushed({len(all_commits)}) commits to {event["repo"]["name"]}({branch})', "repo": event["repo"]["name"], "branch": branch, "commits": all_commits, }, ) - - def add_collaborations(self, event, event_time): collaborators = set()