feat: backfill cv to profile (#3301)

Author: capJavert

__tests__/common/profile/import.ts

@@ -59,6 +59,7 @@ describe('UserExperienceType work import', () => {
       verified: false,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
     const skills = await con
       .getRepository(UserExperienceSkill)
@@ -96,6 +97,7 @@ describe('UserExperienceType work import', () => {
       updatedAt: expect.any(Date),
       userId: 'user-work-2',
       verified: false,
+      flags: {},
     });
   });
 });
@@ -131,6 +133,7 @@ describe('UserExperienceType education import', () => {
       grade: null,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
   });
 
@@ -163,6 +166,7 @@ describe('UserExperienceType education import', () => {
       grade: null,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
   });
 });
@@ -200,6 +204,7 @@ describe('UserExperienceType certification import', () => {
       url: null,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
   });
 
@@ -235,6 +240,7 @@ describe('UserExperienceType certification import', () => {
       url: null,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
   });
 });
@@ -273,6 +279,7 @@ describe('UserExperienceType project import', () => {
       url: null,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
     expect(skills.map((s) => s.value).sort()).toEqual(
       ['GraphQL', 'Node.js'].sort(),
@@ -308,6 +315,7 @@ describe('UserExperienceType project import', () => {
       url: null,
       createdAt: expect.any(Date),
       updatedAt: expect.any(Date),
+      flags: {},
     });
   });
 });

bin/importProfileFromJSON.ts

@@ -3,17 +3,26 @@ import '../src/config';
 import { parseArgs } from 'node:util';
 import { z } from 'zod';
 import createOrGetConnection from '../src/db';
-import { type DataSource } from 'typeorm';
-import { readFile } from 'node:fs/promises';
+import { QueryFailedError, type DataSource } from 'typeorm';
+import { readFile, stat, readdir } from 'node:fs/promises';
 import { importUserExperienceFromJSON } from '../src/common/profile/import';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
 
 /**
  * Import profile from JSON to user by id
  *
- * npx ts-node bin/importProfileFromJSON.ts --path ~/Downloads/testuser.json -u testuser
+ * Single file usage:
+ *
+ * npx ts-node bin/importProfileFromJSON.ts --path ~/Downloads/testuser.json
+ *
+ * Directory usage:
+ *
+ * npx ts-node bin/importProfileFromJSON.ts --path ~/Downloads/profiles --limit 100 --offset 0 --import import_run_test
  */
 const main = async () => {
   let con: DataSource | null = null;
+  let failedImports = 0;
 
   try {
     const { values } = parseArgs({
@@ -22,28 +31,103 @@ const main = async () => {
           type: 'string',
           short: 'p',
         },
-        userId: {
+        limit: {
+          type: 'string',
+          short: 'l',
+        },
+        offset: {
+          type: 'string',
+          short: 'o',
+        },
+        uid: {
           type: 'string',
-          short: 'u',
         },
       },
     });
 
     const paramsSchema = z.object({
       path: z.string().nonempty(),
-      userId: z.string().nonempty(),
+      limit: z.coerce.number().int().positive().default(10),
+      offset: z.coerce.number().int().positive().default(0),
+      uid: z.string().nonempty().default(randomUUID()),
     });
 
     const params = paramsSchema.parse(values);
 
+    console.log(`Starting import with ID: ${params.uid}`);
+
     con = await createOrGetConnection();
 
-    const dataJSON = JSON.parse(await readFile(params.path, 'utf-8'));
+    const pathStat = await stat(params.path);
+
+    let filePaths = [params.path];
+
+    if (pathStat.isDirectory()) {
+      filePaths = await readdir(params.path, 'utf-8');
+    }
+
+    filePaths.sort(); // ensure consistent order for offset/limit
 
-    await importUserExperienceFromJSON({
-      con: con.manager,
-      dataJson: dataJSON,
-      userId: params.userId,
+    console.log(`Found files: ${filePaths.length}`);
+
+    console.log(
+      `Importing: ${Math.min(params.limit, filePaths.length)} (limit ${params.limit}, offset ${params.offset})`,
+    );
+
+    await con.transaction(async (entityManager) => {
+      for (const [index, fileName] of filePaths
+        .slice(params.offset, params.offset + params.limit)
+        .entries()) {
+        const filePath =
+          params.path === fileName
+            ? fileName
+            : path.join(params.path, fileName);
+
+        try {
+          if (!filePath.endsWith('.json')) {
+            throw { type: 'not_json_ext', filePath };
+          }
+
+          const userId = filePath.split('/').pop()?.split('.json')[0];
+
+          if (!userId) {
+            throw { type: 'no_user_id', filePath };
+          }
+
+          const dataJSON = JSON.parse(await readFile(filePath, 'utf-8'));
+
+          await importUserExperienceFromJSON({
+            con: entityManager,
+            dataJson: dataJSON,
+            userId,
+            importId: params.uid,
+          });
+        } catch (error) {
+          failedImports += 1;
+
+          if (error instanceof QueryFailedError) {
+            console.error({
+              type: 'db_query_failed',
+              message: error.message,
+              query: error.query,
+              filePath,
+            });
+          } else if (error instanceof z.ZodError) {
+            console.error({
+              type: 'zod_error',
+              message: error.issues[0].message,
+              path: error.issues[0].path,
+              filePath,
+            });
+          } else {
+            console.error(error);
+          }
+        }
+
+        if (index && index % 100 === 0) {
+          console.log(`Done so far: ${index}, failed: ${failedImports}`);
+        }
+      }
     });
   } catch (error) {
     console.error(error instanceof z.ZodError ? z.prettifyError(error) : error);
@@ -52,6 +136,12 @@ const main = async () => {
       con.destroy();
     }
 
+    if (failedImports > 0) {
+      console.log(`Failed imports: ${failedImports}`);
+    } else {
+      console.log('Done!');
+    }
+
     process.exit(0);
   }
 };