Skip to content

Commit e824186

Browse files
divine-comediandivine-comedian
authored
Merge pull request #121 from Giveth/filter-rl-projects
Filter rl projects
2 parents 4dfbef3 + 0d5c608 commit e824186

File tree

3 files changed

+220
-71
lines changed

3 files changed

+220
-71
lines changed

‎src/features/import-projects/helpers.ts‎

Lines changed: 46 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { SourceConfig } from "./types";
77

88
export const updateOrCreateProject = async (
99
project: any,
10-
sourConfig: SourceConfig
10+
sourceConfig: SourceConfig
1111
) => {
1212
const {
1313
source,
@@ -19,7 +19,7 @@ export const updateOrCreateProject = async (
1919
imageField,
2020
rfRoundField,
2121
prelimResult,
22-
} = sourConfig;
22+
} = sourceConfig;
2323

2424
const projectId = project[idField].toLowerCase();
2525
const id = `${source}-${projectId}`;
@@ -45,65 +45,31 @@ export const updateOrCreateProject = async (
4545
const descriptionHtml = descriptionHtmlField && project[descriptionHtmlField];
4646
const rfRound = rfRoundField && project[rfRoundField];
4747

48-
// Remove project if prelimResult is "Remove"
48+
// Skip project if prelimResult is "Remove"
4949
if (prelimResult && project[prelimResult] === "Remove") {
50-
// If project is imported, delete it from the database
51-
if (existingProject?.imported) {
52-
try {
53-
await dataSource
54-
.createQueryBuilder()
55-
.delete()
56-
.from(Project)
57-
.where("id = :id", { id })
58-
.execute();
59-
60-
console.log(
61-
`[${new Date().toISOString()}] - INFO: Project Deleted. Project ID: ${id}`
62-
);
63-
} catch (error) {
64-
// make project as not imported if failed to delete because it may have some vouches
65-
console.log(
66-
`[${new Date().toISOString()}] - ERROR: Failed to delete project. Project ID: ${id}`
67-
);
68-
const updatedProject = new Project({
69-
...existingProject,
70-
imported: false,
71-
});
72-
try {
73-
await dataSource
74-
.createQueryBuilder()
75-
.update(Project)
76-
.set(updatedProject)
77-
.where("id = :id", { id })
78-
.execute();
79-
} catch (error) {
80-
console.log(
81-
`[${new Date().toISOString()}] - ERROR: Failed to make project un-imported. Project ID: ${id}`
82-
);
83-
}
84-
}
85-
}
8650
return;
8751
}
8852

53+
const descriptionSummary = getHtmlTextSummary(descriptionHtml || description);
54+
8955
if (existingProject) {
56+
// Update existing project
9057
const isUpdated =
9158
existingProject.title !== title ||
9259
existingProject.description !== description ||
9360
existingProject.url !== url ||
9461
existingProject.image !== image ||
9562
(rfRound && !existingProject.rfRounds?.some((rfr) => rfr === rfRound)) ||
96-
existingProject.descriptionHtml !== descriptionHtml ||
63+
existingProject.descriptionHtml != descriptionHtml ||
9764
(!existingProject.descriptionSummary && description);
9865

99-
const descriptionSummary = getHtmlTextSummary(
100-
descriptionHtml || description
101-
);
102-
10366
if (isUpdated) {
104-
const rfRounds = new Set(existingProject.rfRounds || []);
105-
rfRound && rfRounds.add(rfRound);
106-
const updatedProject = new Project({
67+
// Add the current round to rfRounds if not already present
68+
const rfRoundsSet = new Set(existingProject.rfRounds || []);
69+
if (rfRound) {
70+
rfRoundsSet.add(rfRound);
71+
}
72+
const updatedProject = {
10773
...existingProject,
10874
title,
10975
description,
@@ -112,25 +78,29 @@ export const updateOrCreateProject = async (
11278
descriptionHtml,
11379
descriptionSummary,
11480
lastUpdatedTimestamp: new Date(),
115-
rfRounds: Array.from(rfRounds),
81+
rfRounds: Array.from(rfRoundsSet),
11682
imported: true,
117-
});
83+
};
11884

119-
await dataSource
120-
.createQueryBuilder()
121-
.update(Project)
122-
.set(updatedProject)
123-
.where("id = :id", { id })
124-
.execute();
85+
try {
86+
await dataSource
87+
.createQueryBuilder()
88+
.update(Project)
89+
.set(updatedProject)
90+
.where("id = :id", { id })
91+
.execute();
12592

126-
console.log(
127-
`[${new Date().toISOString()}] - INFO: Project Updated. Project ID: ${id}`
128-
);
93+
console.log(
94+
`[${new Date().toISOString()}] - INFO: Project Updated. Project ID: ${id}`
95+
);
96+
} catch (error: any) {
97+
console.log(
98+
`[${new Date().toISOString()}] - ERROR: Failed to update project. Project ID: ${id}, Error: ${error.message}`
99+
);
100+
}
129101
}
130102
} else {
131-
const descriptionSummary = getHtmlTextSummary(
132-
descriptionHtml || description
133-
);
103+
// Create new project
134104
const newProject = new Project({
135105
id,
136106
title,
@@ -149,16 +119,22 @@ export const updateOrCreateProject = async (
149119
imported: true,
150120
});
151121

152-
await dataSource
153-
.createQueryBuilder()
154-
.insert()
155-
.into(Project)
156-
.values([newProject])
157-
.execute();
122+
try {
123+
await dataSource
124+
.createQueryBuilder()
125+
.insert()
126+
.into(Project)
127+
.values([newProject])
128+
.execute();
158129

159-
console.log(
160-
`[${new Date().toISOString()}] - INFO: Project Created. Project ID: ${id}`
161-
);
130+
console.log(
131+
`[${new Date().toISOString()}] - INFO: Project Created. Project ID: ${id}`
132+
);
133+
} catch (error: any) {
134+
console.log(
135+
`[${new Date().toISOString()}] - ERROR: Failed to create project. Project ID: ${id}, Error: ${error.message}`
136+
);
137+
}
162138
}
163139
};
164140

‎src/features/import-projects/retroList/helper.ts‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,172 @@
1+
import { getDataSource } from "../../../helpers/db";
2+
import { Project } from "../../../model";
3+
import { SourceConfig } from "../types";
14
import { type RlProjectInfo } from "./type";
25

36
export const generateRlUrl = (project: RlProjectInfo) => {
47
return `/project/${project.id}`;
58
};
9+
10+
export const manageProjectRemovals = async (
11+
newList: RlProjectInfo[] | null,
12+
sourceConfig: SourceConfig,
13+
round: number // Pass the current round
14+
) => {
15+
if (newList === null) {
16+
console.log(
17+
`[${new Date().toISOString()}] - ERROR: Failed to manage project removals. New list is null.`
18+
);
19+
return;
20+
}
21+
try {
22+
const dataSource = await getDataSource();
23+
if (!dataSource) {
24+
console.log(
25+
`[${new Date().toISOString()}] - ERROR: Failed to remove projects. Data source not found.`
26+
);
27+
return;
28+
}
29+
30+
const shouldKeepProjects = newList
31+
.filter((project) => project.prelimResult === "Keep")
32+
.map((project) => `${sourceConfig.source}-${project.id}`);
33+
console.log("shouldKeepProjects", shouldKeepProjects);
34+
35+
const existingProjectsIds = await dataSource
36+
.getRepository(Project)
37+
.createQueryBuilder("project")
38+
.select("project.id")
39+
.where("project.source = :source", { source: sourceConfig.source })
40+
.andWhere(":round = ANY(project.rfRounds)", { round }) // Only projects in the current round
41+
.getMany()
42+
.then((projects) => projects.map((proj) => proj.id));
43+
44+
console.log("existingProjectsIds", existingProjectsIds);
45+
46+
const projectIdsToManipulate = existingProjectsIds.filter(
47+
(id) => !shouldKeepProjects.includes(id)
48+
);
49+
50+
console.log("projectIdsToManipulate", projectIdsToManipulate);
51+
52+
if (projectIdsToManipulate.length === 0) {
53+
// No projects to Manipulate
54+
return;
55+
}
56+
57+
// Fetch projects to Remove, including their attests
58+
const projectsToManipulate = await dataSource
59+
.getRepository(Project)
60+
.createQueryBuilder("project")
61+
.leftJoinAndSelect("project.attests", "attests")
62+
.where("project.id IN (:...ids)", { ids: projectIdsToManipulate })
63+
.getMany();
64+
65+
const projectsToUpdateRfRounds: Array<{
66+
id: string;
67+
updatedRfRounds: number[];
68+
}> = [];
69+
const projectsToRemove: Project[] = [];
70+
const projectsToMakeUnImported: Project[] = [];
71+
72+
for (const project of projectsToManipulate) {
73+
const updatedRfRounds = (project.rfRounds || []).filter(
74+
(rfr) => rfr !== round
75+
);
76+
if (updatedRfRounds.length > 0) {
77+
// Collect projects to update
78+
projectsToUpdateRfRounds.push({ id: project.id, updatedRfRounds });
79+
} else {
80+
if (project.attests.length > 0) {
81+
projectsToMakeUnImported.push(project);
82+
} else {
83+
projectsToRemove.push(project);
84+
}
85+
}
86+
}
87+
88+
const updateRfRoundsPromises = projectsToUpdateRfRounds.map(
89+
async ({ id, updatedRfRounds }) => {
90+
try {
91+
await dataSource
92+
.createQueryBuilder()
93+
.update(Project)
94+
.set({ rfRounds: updatedRfRounds })
95+
.where("id = :id", { id })
96+
.execute();
97+
console.log(
98+
`[INFO]: Project updated with removed round. Project ID: ${id}`
99+
);
100+
} catch (updateError: any) {
101+
console.log(
102+
`[ERROR]: Failed to update project rfRounds. Project ID: ${id}. Error: ${updateError.message}`
103+
);
104+
throw updateError; // Re-throw to let Promise.all handle it
105+
}
106+
}
107+
);
108+
try {
109+
await Promise.all(updateRfRoundsPromises);
110+
} catch (error: any) {
111+
console.log(
112+
`[ERROR]: One or more updates failed. Error: ${error.message}`
113+
);
114+
}
115+
116+
// Bulk delete projects without attests
117+
if (projectsToRemove.length > 0) {
118+
const projectIdsToDelete = projectsToRemove.map((project) => project.id);
119+
try {
120+
await dataSource
121+
.createQueryBuilder()
122+
.delete()
123+
.from(Project)
124+
.where("id IN (:...ids)", { ids: projectIdsToDelete })
125+
.execute();
126+
127+
console.log(
128+
`[${new Date().toISOString()}] - INFO: Projects Deleted. Project IDs: ${projectIdsToDelete.join(
129+
", "
130+
)}`
131+
);
132+
} catch (error: any) {
133+
console.log(
134+
`[${new Date().toISOString()}] - ERROR: Failed to delete projects. Project IDs: ${projectIdsToDelete.join(
135+
", "
136+
)}. Error: ${error.message}`
137+
);
138+
}
139+
}
140+
141+
// Bulk update projects with attests to mark as not imported
142+
if (projectsToMakeUnImported.length > 0) {
143+
const projectIdsToMakeUnImported = projectsToMakeUnImported.map(
144+
(project) => project.id
145+
);
146+
try {
147+
await dataSource
148+
.createQueryBuilder()
149+
.update(Project)
150+
.set({ imported: false, rfRounds: [round] })
151+
.where("id IN (:...ids)", { ids: projectIdsToMakeUnImported })
152+
.execute();
153+
154+
console.log(
155+
`[${new Date().toISOString()}] - INFO: Projects marked as not imported. Project IDs: ${projectIdsToMakeUnImported.join(
156+
", "
157+
)}`
158+
);
159+
} catch (error: any) {
160+
console.log(
161+
`[${new Date().toISOString()}] - ERROR: Failed to mark projects as not imported. Project IDs: ${projectIdsToMakeUnImported.join(
162+
", "
163+
)}. Error: ${error.message}`
164+
);
165+
}
166+
}
167+
} catch (error: any) {
168+
console.log(
169+
`[${new Date().toISOString()}] - ERROR: Failed to manage project removals. Error: ${error.message}`
170+
);
171+
}
172+
};

‎src/features/import-projects/retroList/index.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
import { updateOrCreateProject } from "../helpers";
22
import { rlSourceConfig } from "./constants";
3-
import { generateRlUrl } from "./helper";
3+
import { generateRlUrl, manageProjectRemovals } from "./helper";
44
import { fetchRlProjects } from "./service";
55

66
export const fetchAndProcessRlProjects = async (round: number) => {
77
try {
88
const data = await fetchRlProjects(round);
99
if (!data) return;
10+
11+
const processedProjectIds: string[] = [];
12+
1013
for (const project of data) {
1114
const processedProject = {
1215
...project,
@@ -16,6 +19,9 @@ export const fetchAndProcessRlProjects = async (round: number) => {
1619

1720
await updateOrCreateProject(processedProject, rlSourceConfig);
1821
}
22+
23+
// After processing all new projects, handle projects not in the new dataset for the current round
24+
await manageProjectRemovals(data, rlSourceConfig, round);
1925
} catch (error: any) {
2026
console.log("error on fetchAndProcessRlProjects", error.message);
2127
}

0 commit comments

Comments
 (0)