Skip to content

Commit ead8bd2

Browse files
lowlighterlowlighter
authored
feat(plugins/base): split base queries to reduce GitHub API timeouts (lowlighter#584)
1 parent f75638d commit ead8bd2

File tree

14 files changed

+223
-170
lines changed

14 files changed

+223
-170
lines changed

‎source/app/mocks/api/github/graphql/base.calendar.mjs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/**Mocked data */
2+
export default function({faker, query, login = faker.internet.userName()}) {
3+
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
4+
return ({
5+
user:{
6+
calendar:{
7+
contributionCalendar:{
8+
weeks:[
9+
{
10+
contributionDays:[
11+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
12+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
13+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
14+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
15+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
16+
],
17+
},
18+
{
19+
contributionDays:[
20+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
21+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
22+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
23+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
24+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
25+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
26+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
27+
],
28+
},
29+
{
30+
contributionDays:[
31+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
32+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
33+
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
34+
],
35+
},
36+
],
37+
},
38+
},
39+
},
40+
})
41+
}

‎source/app/mocks/api/github/graphql/base.contributions.mjs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**Mocked data */
2+
export default function({faker, query, login = faker.internet.userName()}) {
3+
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
4+
return ({
5+
user:{
6+
contributionsCollection:{
7+
totalRepositoriesWithContributedCommits:faker.datatype.number(100),
8+
totalCommitContributions:faker.datatype.number(10000),
9+
restrictedContributionsCount:faker.datatype.number(10000),
10+
totalIssueContributions:faker.datatype.number(100),
11+
totalPullRequestContributions:faker.datatype.number(1000),
12+
totalPullRequestReviewContributions:faker.datatype.number(1000),
13+
},
14+
},
15+
})
16+
}

‎source/app/mocks/api/github/graphql/base.field.mjs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/**Mocked data */
2+
export default function({faker, query, login = faker.internet.userName()}) {
3+
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
4+
return ({
5+
user:{
6+
packages:{totalCount:faker.datatype.number(10)},
7+
starredRepositories:{totalCount:faker.datatype.number(1000)},
8+
watching:{totalCount:faker.datatype.number(100)},
9+
sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)},
10+
sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)},
11+
repositoriesContributedTo:{totalCount:faker.datatype.number(100)},
12+
followers:{totalCount:faker.datatype.number(1000)},
13+
following:{totalCount:faker.datatype.number(1000)},
14+
issueComments:{totalCount:faker.datatype.number(1000)},
15+
organizations:{totalCount:faker.datatype.number(10)},
16+
},
17+
})
18+
}

‎source/app/mocks/api/github/graphql/base.field.repositories.mjs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/**Mocked data */
2+
export default function({faker, query, login = faker.internet.userName()}) {
3+
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
4+
return ({
5+
user:{
6+
repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)},
7+
},
8+
})
9+
}

‎source/app/mocks/api/github/graphql/base.user.mjs‎

Lines changed: 0 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -11,58 +11,6 @@ export default function({faker, query, login = faker.internet.userName()}) {
1111
websiteUrl:faker.internet.url(),
1212
isHireable:faker.datatype.boolean(),
1313
twitterUsername:login,
14-
repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)},
15-
packages:{totalCount:faker.datatype.number(10)},
16-
starredRepositories:{totalCount:faker.datatype.number(1000)},
17-
watching:{totalCount:faker.datatype.number(100)},
18-
sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)},
19-
sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)},
20-
contributionsCollection:{
21-
totalRepositoriesWithContributedCommits:faker.datatype.number(100),
22-
totalCommitContributions:faker.datatype.number(10000),
23-
restrictedContributionsCount:faker.datatype.number(10000),
24-
totalIssueContributions:faker.datatype.number(100),
25-
totalPullRequestContributions:faker.datatype.number(1000),
26-
totalPullRequestReviewContributions:faker.datatype.number(1000),
27-
},
28-
calendar:{
29-
contributionCalendar:{
30-
weeks:[
31-
{
32-
contributionDays:[
33-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
34-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
35-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
36-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
37-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
38-
],
39-
},
40-
{
41-
contributionDays:[
42-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
43-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
44-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
45-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
46-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
47-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
48-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
49-
],
50-
},
51-
{
52-
contributionDays:[
53-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
54-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
55-
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
56-
],
57-
},
58-
],
59-
},
60-
},
61-
repositoriesContributedTo:{totalCount:faker.datatype.number(100)},
62-
followers:{totalCount:faker.datatype.number(1000)},
63-
following:{totalCount:faker.datatype.number(1000)},
64-
issueComments:{totalCount:faker.datatype.number(1000)},
65-
organizations:{totalCount:faker.datatype.number(10)},
6614
},
6715
})
6816
}

‎source/plugins/base/index.mjs‎

Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,65 @@ export default async function({login, graphql, rest, data, q, queries, imports},
2727
try {
2828
//Query data from GitHub API
2929
console.debug(`metrics/compute/${login}/base > account ${account}`)
30-
const queried = await graphql(queries.base[account]({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString(), forks, affiliations}))
30+
const queried = await graphql(queries.base[account]({login}))
3131
Object.assign(data, {user:queried[account]})
3232
postprocess?.[account]({login, data})
33+
//Query basic fields
34+
const fields = {
35+
user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"],
36+
organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"],
37+
}[account] ?? []
38+
for (const field of fields) {
39+
try {
40+
Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account])
41+
}
42+
catch {
43+
console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`)
44+
data.user[field] = {totalCount:NaN}
45+
}
46+
}
47+
//Query repositories fields
48+
for (const field of ["totalCount", "totalDiskUsage"]) {
49+
try {
50+
Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories)
51+
}
52+
catch (error) {
53+
console.log(error)
54+
console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`)
55+
data.user.repositories[field] = NaN
56+
}
57+
}
58+
//Query user account fields
59+
if (account === "user") {
60+
//Query contributions collection
61+
{
62+
const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"]
63+
for (const field of fields) {
64+
try {
65+
Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection)
66+
}
67+
catch {
68+
console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`)
69+
data.user.contributionsCollection[field] = NaN
70+
}
71+
}
72+
}
73+
//Query calendar
74+
try {
75+
Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account])
76+
}
77+
catch {
78+
console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`)
79+
data.user.calendar = {contributionCalendar:{weeks:[]}}
80+
}
81+
}
3382
//Query repositories from GitHub API
34-
data.user.repositoriesContributedTo.nodes = data.user.repositoriesContributedTo.nodes ?? []
3583
for (const type of ({user:["repositories", "repositoriesContributedTo"], organization:["repositories"]}[account] ?? [])) {
3684
//Iterate through repositories
3785
let cursor = null
3886
let pushed = 0
3987
const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null
88+
data.user[type] = data.user[type] ?? {}
4089
data.user[type].nodes = data.user[type].nodes ?? []
4190
do {
4291
console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`)
@@ -45,7 +94,7 @@ export default async function({login, graphql, rest, data, q, queries, imports},
4594
data.user[type].nodes.push(...nodes)
4695
pushed = nodes.length
4796
console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`)
48-
} while ((pushed) && (cursor) && (data.user.repositories.nodes.length + data.user.repositoriesContributedTo.nodes.length < repositories))
97+
} while ((pushed) && (cursor) && ((data.user.repositories?.nodes?.length ?? 0) + (data.user.repositoriesContributedTo?.nodes?.length ?? 0) < repositories))
4998
//Limit repositories
5099
console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`)
51100
data.user[type].nodes.splice(repositories)
@@ -93,6 +142,8 @@ const postprocess = {
93142
data.account = "user"
94143
Object.assign(data.user, {
95144
isVerified:false,
145+
repositories:{},
146+
contributionsCollection:{},
96147
})
97148
},
98149
//Organization
@@ -101,22 +152,23 @@ const postprocess = {
101152
data.account = "organization"
102153
Object.assign(data.user, {
103154
isHireable:false,
104-
starredRepositories:{totalCount:0},
105-
watching:{totalCount:0},
155+
repositories:{},
156+
starredRepositories:{totalCount:NaN},
157+
watching:{totalCount:NaN},
106158
contributionsCollection:{
107-
totalRepositoriesWithContributedCommits:0,
108-
totalCommitContributions:0,
109-
restrictedContributionsCount:0,
110-
totalIssueContributions:0,
111-
totalPullRequestContributions:0,
112-
totalPullRequestReviewContributions:0,
159+
totalRepositoriesWithContributedCommits:NaN,
160+
totalCommitContributions:NaN,
161+
restrictedContributionsCount:NaN,
162+
totalIssueContributions:NaN,
163+
totalPullRequestContributions:NaN,
164+
totalPullRequestReviewContributions:NaN,
113165
},
114166
calendar:{contributionCalendar:{weeks:[]}},
115-
repositoriesContributedTo:{totalCount:0},
116-
followers:{totalCount:0},
117-
following:{totalCount:0},
118-
issueComments:{totalCount:0},
119-
organizations:{totalCount:0},
167+
repositoriesContributedTo:{totalCount:NaN, nodes:[]},
168+
followers:{totalCount:NaN},
169+
following:{totalCount:NaN},
170+
issueComments:{totalCount:NaN},
171+
organizations:{totalCount:NaN},
120172
})
121173
},
122174
//Skip base content query and instantiate an empty user instance
@@ -127,16 +179,16 @@ const postprocess = {
127179
postprocess?.[account]({login, data})
128180
data.account = "bypass"
129181
Object.assign(data.user, {
130-
databaseId:0,
182+
databaseId:NaN,
131183
name:login,
132184
login,
133185
createdAt:new Date(),
134186
avatarUrl:`https://github.com/${login}.png`,
135187
websiteUrl:null,
136188
twitterUsername:login,
137-
repositories:{totalCount:0, totalDiskUsage:0, nodes:[]},
138-
packages:{totalCount:0},
139-
repositoriesContributedTo:{nodes:[]},
189+
repositories:{totalCount:NaN, totalDiskUsage:NaN, nodes:[]},
190+
packages:{totalCount:NaN},
191+
repositoriesContributedTo:{totalCount:NaN, nodes:[]},
140192
})
141193
},
142194
}

‎source/plugins/base/queries/calendar.graphql‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
query BaseCalendar {
2+
user(login: "$login") {
3+
calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") {
4+
contributionCalendar {
5+
weeks {
6+
contributionDays {
7+
color
8+
}
9+
}
10+
}
11+
}
12+
}
13+
}

‎source/plugins/base/queries/contributions.graphql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
query BaseContributions {
2+
user(login: "$login") {
3+
contributionsCollection {
4+
$field
5+
}
6+
}
7+
}
8+
9+

‎source/plugins/base/queries/field.graphql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
query BaseField {
2+
$account(login: "$login") {
3+
$field {
4+
totalCount
5+
}
6+
}
7+
}

‎source/plugins/base/queries/field.repositories.graphql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
query BaseFieldRepositories {
2+
$account(login: "$login") {
3+
repositories(last: 0) {
4+
$field
5+
}
6+
}
7+
}

0 commit comments

Comments
 (0)