From ada941f94ff3259a29b9c7f76be5b4126501f7d0 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Sat, 11 May 2024 12:13:17 -0300 Subject: [PATCH 01/16] refactor: using datetime instead of string form dates with Prisma --- .../migrations/20240511143905_/migration.sql | 31 +++++++++ prisma/schema.prisma | 68 +++++++++---------- 2 files changed, 65 insertions(+), 34 deletions(-) create mode 100644 prisma/migrations/20240511143905_/migration.sql diff --git a/prisma/migrations/20240511143905_/migration.sql b/prisma/migrations/20240511143905_/migration.sql new file mode 100644 index 00000000..b3be8c83 --- /dev/null +++ b/prisma/migrations/20240511143905_/migration.sql @@ -0,0 +1,31 @@ +/* + Warnings: + Por padrão o prisma dropa e recria a coluna do banco. + Esse script usa um cast do postgres para converter as colunas para o tipo correto ao invés de dropá-las + Assumo que as timestamp estão sendo salvas em UTC + Para mudar a migration para setar as datas atuais para uma timezone específica, basta adicionar o statement "AT TIME ZONE" após a conversão + alter table "category_supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz AT TIME ZONE 'America/Sao_Paulo'; +*/ + +-- AlterTable +alter table "category_supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "category_supplies" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +-- AlterTable +alter table "sessions" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "sessions" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +-- AlterTable +alter table "shelter_managers" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "shelter_managers" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +-- AlterTable +alter table "shelter_supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "shelter_supplies" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +-- AlterTable +alter table "shelters" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "shelters" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +-- AlterTable +alter table "supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "supplies" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +-- AlterTable; +alter table "users" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; +alter table "users" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz ; + diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 792b1f65..3658fc4a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -22,8 +22,8 @@ model User { password String phone String @unique accessLevel AccessLevel @default(value: User) - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() sessions Session[] shelterManagers ShelterManagers[] @@ -32,13 +32,13 @@ model User { } model Session { - id String @id @default(uuid()) - userId String @map("user_id") + id String @id @default(uuid()) + userId String @map("user_id") ip String? - userAgent String? @map("user_agent") - active Boolean @default(value: true) - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + userAgent String? @map("user_agent") + active Boolean @default(value: true) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() user User @relation(fields: [userId], references: [id]) @@ -46,10 +46,10 @@ model Session { } model SupplyCategory { - id String @id @default(uuid()) - name String @unique - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + id String @id @default(uuid()) + name String @unique + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() supplies Supply[] @@ -57,12 +57,12 @@ model SupplyCategory { } model ShelterSupply { - shelterId String @map("shelter_id") - supplyId String @map("supply_id") - priority Int @default(value: 0) + shelterId String @map("shelter_id") + supplyId String @map("supply_id") + priority Int @default(value: 0) quantity Int? - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() shelter Shelter @relation(fields: [shelterId], references: [id]) supply Supply @relation(fields: [supplyId], references: [id]) @@ -72,11 +72,11 @@ model ShelterSupply { } model Supply { - id String @id @default(uuid()) - supplyCategoryId String @map("supply_category_id") + id String @id @default(uuid()) + supplyCategoryId String @map("supply_category_id") name String - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() supplyCategory SupplyCategory @relation(fields: [supplyCategoryId], references: [id]) shelterSupplies ShelterSupply[] @@ -85,20 +85,20 @@ model Supply { } model Shelter { - id String @id @default(uuid()) - name String @unique - pix String? @unique + id String @id @default(uuid()) + name String @unique + pix String? @unique address String - petFriendly Boolean? @map("pet_friendly") - shelteredPeople Int? @map("sheltered_people") + petFriendly Boolean? @map("pet_friendly") + shelteredPeople Int? @map("sheltered_people") capacity Int? contact String? - prioritySum Int @default(value: 0) @map("priority_sum") + prioritySum Int @default(value: 0) @map("priority_sum") latitude Float? longitude Float? - verified Boolean @default(value: false) - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + verified Boolean @default(value: false) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() shelterManagers ShelterManagers[] shelterSupplies ShelterSupply[] @@ -107,10 +107,10 @@ model Shelter { } model ShelterManagers { - shelterId String @map("shelter_id") - userId String @map("user_id") - createdAt String @map("created_at") @db.VarChar(32) - updatedAt String? @map("updated_at") @db.VarChar(32) + shelterId String @map("shelter_id") + userId String @map("user_id") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() user User @relation(fields: [userId], references: [id]) shelter Shelter @relation(fields: [shelterId], references: [id]) From e826c672cc3feb7d30e6894855ed6f54d640f776 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Sat, 11 May 2024 12:52:47 -0300 Subject: [PATCH 02/16] fix: corrigindo tipo do dado e adicionando default value --- .../migrations/20240511143905_/migration.sql | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/prisma/migrations/20240511143905_/migration.sql b/prisma/migrations/20240511143905_/migration.sql index b3be8c83..529d324f 100644 --- a/prisma/migrations/20240511143905_/migration.sql +++ b/prisma/migrations/20240511143905_/migration.sql @@ -4,28 +4,35 @@ Esse script usa um cast do postgres para converter as colunas para o tipo correto ao invés de dropá-las Assumo que as timestamp estão sendo salvas em UTC Para mudar a migration para setar as datas atuais para uma timezone específica, basta adicionar o statement "AT TIME ZONE" após a conversão - alter table "category_supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz AT TIME ZONE 'America/Sao_Paulo'; + alter table "category_supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz AT TIME ZONE 'America/Sao_Paulo'; */ -- AlterTable -alter table "category_supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "category_supplies" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +alter table "category_supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "category_supplies" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "category_supplies" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; -- AlterTable -alter table "sessions" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "sessions" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +alter table "sessions" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "sessions" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "sessions" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; -- AlterTable -alter table "shelter_managers" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "shelter_managers" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +alter table "shelter_managers" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "shelter_managers" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "shelter_managers" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; -- AlterTable -alter table "shelter_supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "shelter_supplies" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +alter table "shelter_supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "shelter_supplies" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "shelter_supplies" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; -- AlterTable -alter table "shelters" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "shelters" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +alter table "shelters" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "shelters" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "shelters" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; -- AlterTable -alter table "supplies" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "supplies" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz; +alter table "supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "supplies" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "supplies" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; -- AlterTable; -alter table "users" alter column "created_at" type TIMESTAMP using "created_at"::timestamptz; -alter table "users" alter column "updated_at" type TIMESTAMP using "updated_at"::timestamptz ; +alter table "users" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; +alter table "users" alter column "created_at" set default CURRENT_TIMESTAMP; +alter table "users" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz ; From da18a35c86e9c90457c0a86e3d0f0d913bb6d3b7 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Wed, 15 May 2024 16:19:18 -0300 Subject: [PATCH 03/16] =?UTF-8?q?feat:=20adicionando=20implementa=C3=A7?= =?UTF-8?q?=C3=A3o=20inicial?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 +- .env.local | 4 +- package-lock.json | 44 ++ package.json | 2 + src/shelter-csv-importer/index.ts | 534 ++++++++++++++++++ .../shelter-csv-importer.module.ts | 4 + src/shelter-csv-importer/types.ts | 0 7 files changed, 590 insertions(+), 2 deletions(-) create mode 100644 src/shelter-csv-importer/index.ts create mode 100644 src/shelter-csv-importer/shelter-csv-importer.module.ts create mode 100644 src/shelter-csv-importer/types.ts diff --git a/.env.example b/.env.example index aedd7c2d..f0e712f8 100644 --- a/.env.example +++ b/.env.example @@ -9,4 +9,6 @@ DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_ SECRET_KEY= HOST=::0.0.0.0 -PORT=4000 \ No newline at end of file +PORT=4000 + +GEMINI_API_KEY='' \ No newline at end of file diff --git a/.env.local b/.env.local index 9cd74618..8ee218d4 100644 --- a/.env.local +++ b/.env.local @@ -10,4 +10,6 @@ DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_ SECRET_KEY=batata HOST=::0.0.0.0 -PORT=4000 \ No newline at end of file +PORT=4000 + +GEMINI_API_KEY='' \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 455fac11..2e9f65a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@fastify/static": "^7.0.3", + "@google/generative-ai": "^0.11.1", "@nestjs/common": "^10.0.0", "@nestjs/core": "^10.0.0", "@nestjs/jwt": "^10.2.0", @@ -19,6 +20,7 @@ "@nestjs/swagger": "^7.3.1", "@prisma/client": "^5.13.0", "bcrypt": "^5.1.1", + "csv": "^6.3.9", "date-fns": "^3.6.0", "passport-jwt": "^4.0.1", "reflect-metadata": "^0.2.0", @@ -1057,6 +1059,14 @@ "glob": "^10.3.4" } }, + "node_modules/@google/generative-ai": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.11.1.tgz", + "integrity": "sha512-ZiUiJJbl55TXcvu73+Kf/bUhzcRTH/bsGBeYZ9ULqU0imXg3POcd+NVYM9j+TGq4MA73UYwHPmJHwmy+QZEzyQ==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -4138,6 +4148,35 @@ "node": ">= 8" } }, + "node_modules/csv": { + "version": "6.3.9", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.9.tgz", + "integrity": "sha512-eiN+Qu8NwSLxZYia6WzB8xlX/rAQ/8EgK5A4dIF7Bz96mzcr5dW1jlcNmjG0QWySWKfPdCerH3RQ96ZqqsE8cA==", + "dependencies": { + "csv-generate": "^4.4.1", + "csv-parse": "^5.5.6", + "csv-stringify": "^6.5.0", + "stream-transform": "^3.3.2" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.1.tgz", + "integrity": "sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg==" + }, + "node_modules/csv-parse": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz", + "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==" + }, + "node_modules/csv-stringify": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.0.tgz", + "integrity": "sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==" + }, "node_modules/date-fns": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", @@ -8841,6 +8880,11 @@ "node": ">= 0.8" } }, + "node_modules/stream-transform": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.2.tgz", + "integrity": "sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ==" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", diff --git a/package.json b/package.json index b20634cd..b3eea7c8 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ }, "dependencies": { "@fastify/static": "^7.0.3", + "@google/generative-ai": "^0.11.1", "@nestjs/common": "^10.0.0", "@nestjs/core": "^10.0.0", "@nestjs/jwt": "^10.2.0", @@ -34,6 +35,7 @@ "@nestjs/swagger": "^7.3.1", "@prisma/client": "^5.13.0", "bcrypt": "^5.1.1", + "csv": "^6.3.9", "date-fns": "^3.6.0", "passport-jwt": "^4.0.1", "reflect-metadata": "^0.2.0", diff --git a/src/shelter-csv-importer/index.ts b/src/shelter-csv-importer/index.ts new file mode 100644 index 00000000..bd420950 --- /dev/null +++ b/src/shelter-csv-importer/index.ts @@ -0,0 +1,534 @@ +import { + GoogleGenerativeAI, + HarmBlockThreshold, + HarmCategory, +} from '@google/generative-ai'; +import { Logger } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; +import { parse as createParser } from 'csv'; +import { ReadStream } from 'node:fs'; +import * as path from 'node:path'; +import { Readable, Transform } from 'node:stream'; +import { TransformStream, WritableStream } from 'node:stream/web'; + +import { PrismaService } from '../prisma/prisma.service'; +import { CreateShelterSchema } from '../shelter/types/types'; +import { responseToReadable } from '../utils/utils'; + +type ShelterKey = Exclude< + Prisma.ShelterScalarFieldEnum, + 'createdAt' | 'updatedAt' | 'prioritySum' | 'verified' | 'id' +>; +type ShelterColumHeader = Record<`${ShelterKey}Field`, string> & { + shelterSuppliesField: string; +}; +interface ParseCsvArgsBaseArgs> { + /** + * link do arquivo CSV + */ + csvUrl?: string; + /** + * stream proveniente de algum arquivo CSV + */ + fileStream?: ReadStream; + /** + * mapeamento de quais cabeçalhos do csv serão usados como colunas da tabela. + */ + headers?: Partial; +} + +export type ParseCsvArgs = + | (ParseCsvArgsBaseArgs & { csvUrl: string }) + | (ParseCsvArgsBaseArgs & { fileStream: ReadStream }); + +interface EnhancedTransformArgs { + /** + * KeyValue contento as categorias e os supplies contidos naquela categorias (detectados por IA) + */ + shelterSupliesByCategory: Record; + /** + * KeyValue com Suprimentos já encontrados na base atual que podem ser utilizadas em relações + * @example + * { "Óleo de cozinha": "eb1d5056-8b9b-455d-a179-172a747e3f20", + * "Álcool em gel": "a3e3bdf8-0be4-4bdc-a3b0-b40ba931be5f" + * } + */ + suppliesAvailable: Map; + /** + * KeyValue com Categorias já encontradas na base atual que podem ser utilizadas em relações + * @example + * { "Higiene Pessoal": "718d5be3-69c3-4216-97f1-12b690d0eb97", + * "Alimentos e Água": "a3e3bdf8-0be4-4bdc-a3b0-b40ba931be5f" + * } + */ + categoriesAvailable: Map; + counter?: AtomicCounter; +} +const CSV_DEFAULT_HEADERS: ShelterColumHeader = { + nameField: 'nome_do_local', + addressField: 'endereco', + contactField: 'whatsapp', + latitudeField: 'lat', + longitudeField: 'lng', + shelterSuppliesField: 'itens_em_falta', + capacityField: 'capacidade', + cityField: 'cidade', + neighbourhoodField: 'bairro', + petFriendlyField: 'pet_friendly', + pixField: 'pix', + shelteredPeopleField: 'pessoas_abrigadas', + streetField: 'rua', + streetNumberField: 'numero', + zipCodeField: 'cep', +}; + +type ShelterInput = Partial; +/** + * Padrão utilizado + * `nome_do_local, endereco, whatsapp, lat, lng, itens_disponiveis, itens_em_falta` + */ + +// Regex que ignora vírgulas dentro de parenteses no split +const COLON_REGEX = /(? ShelterInput + * @see ShelterInput + */ +class CsvToShelterTransformStream extends TransformStream< + unknown, + ShelterInput +> { + /** + * Espera um Ojeto contento a assinatura da entidade esperada + * @param columnNames dicionário com nomes das colunas a serem mapeadas + */ + constructor(columnNames: Partial = CSV_DEFAULT_HEADERS) { + const efffectiveColumnNames = {} as ShelterColumHeader; + Object.entries(CSV_DEFAULT_HEADERS).forEach(([key, value]) => { + efffectiveColumnNames[key] = + typeof columnNames[key] === 'string' ? columnNames[key] : value; + }); + + super({ + async transform(chunk, controller) { + if (!chunk || (chunk && typeof chunk !== 'object')) { + return; + } + let supplies: string[] = []; + + if ( + typeof chunk[efffectiveColumnNames.shelterSuppliesField] === 'string' + ) { + supplies = (chunk[efffectiveColumnNames.shelterSuppliesField]) + .split(COLON_REGEX) + .filter(Boolean) + .map((s) => s.trim()); + } + + const shelter: ShelterInput = { + verified: false, + // Removendo duplicidades + supplies: [...new Set(supplies)], + }; + + Object.keys(Prisma.ShelterScalarFieldEnum).forEach((key) => { + shelter[key] ??= chunk[efffectiveColumnNames[`${key}Field`]]; + }); + + controller.enqueue(shelter); + logger.verbose( + '🚀 ~ CsvToShelterTransformStream ~ constructor ~ shelter:', + shelter, + ); + }, + }); + } +} + +/** + * Valida o schema do Input, enriquece o modelo e adicionas as relações encontradas + */ +class ShelterEnhancedStreamTransformer extends TransformStream< + ShelterInput, + ReturnType +> { + private counter!: AtomicCounter; + + /** + * + */ + constructor({ + shelterSupliesByCategory, + suppliesAvailable, + categoriesAvailable, + counter, + }: EnhancedTransformArgs) { + super({ + transform: async (shelter: ShelterInput, controller) => { + this.counter ??= counter || new AtomicCounter(); + if (!shelter.supplies) { + try { + controller.enqueue(CreateShelterSchema.parse(shelter)); + } catch (error) { + this.counter.incrementFailure(); + logger.error(error, shelter); + } + return; + } + + const missingSupplyNames = new Set(); + + const toCreate: Prisma.ShelterSupplyCreateNestedManyWithoutShelterInput['create'] = + []; + + for (const supplyName of missingSupplyNames.values()) { + for (const [categoryName, values] of Object.entries( + shelterSupliesByCategory, + )) { + const indexFound = values.findIndex( + (item) => item.toLowerCase() === supplyName.toLowerCase(), + ); + if (indexFound !== -1) { + toCreate.push({ + supplyId: suppliesAvailable.get(supplyName.toLowerCase())!, + supply: { + connect: { + id: suppliesAvailable.get(supplyName.toLowerCase())!, + name: supplyName, + }, + connectOrCreate: { + where: { + id: suppliesAvailable.get(supplyName.toLowerCase()), + }, + create: { + name: supplyName, + supplyCategory: { + connect: { + name: categoryName, + id: categoriesAvailable.get( + categoryName.toLowerCase(), + ), + }, + }, + }, + }, + }, + }); + } + } + } + shelter.shelterSupplies = { + create: toCreate, + }; + + if (shelter.latitude) { + shelter.latitude = Number.parseFloat(`${shelter.latitude}`); + } + if (shelter.longitude) { + shelter.longitude = Number.parseFloat(`${shelter.longitude}`); + } + + Object.keys(shelter).forEach((key) => { + if ( + typeof shelter[key] === 'string' && + shelter[key].trim().length === 0 + ) { + shelter[key] = null; + } + }); + + await CreateShelterSchema.parseAsync(shelter) + .then((s) => controller.enqueue(s)) + .catch((e) => { + this.counter.incrementFailure(); + logger.error(e.message, shelter); + }); + }, + }); + } +} + +async function detectSupplyCategoryUsingAI( + input: unknown, + categoriesAvailable: string[], +): Promise> { + if (typeof input !== 'string') { + logger.warn(`Input inesperado recebido: ${input}`); + return {}; + } + const apiKey = process.env.GEMINI_API_KEY; + + if (!apiKey) { + const res = require( + path.resolve(__dirname, '..', '..', 'promp_response.json'), + ); + return res; + } + const genAI = new GoogleGenerativeAI(apiKey); + + const model = genAI.getGenerativeModel({ + model: 'gemini-1.5-pro-latest', + }); + + const generationConfig = { + temperature: 1, + topP: 0.95, + topK: 64, + maxOutputTokens: 8192, + responseMimeType: 'application/json', + }; + + const safetySettings = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, + }, + ]; + + const chatSession = model.startChat({ + generationConfig, + safetySettings, + history: [ + { + role: 'user', + parts: [ + { + text: `Você é um classificador de itens donativos para tragédias ambientais.\nCategorize os INPUT em uma das seguintes categorias: ${categoriesAvailable.join(', ')}. Retorne um JSON com a seguitne assinatura: Record. Cada produto pode ter apenas uma categoria.`, + }, + { + text: 'Lanternas, Roupas para crianças, Comidas Não Perecíveis, Pratos, Lençóis, Cadeiras (tipo de praia), Absorventes higiênicos, Kits de primeiros socorros, Fórmula infantil, Colchões, Caixas de papelão, Lixeiras, Carne Gado, Gaze, Termômetro, Álcool, Repelente, Massa, Descarpack Caixa, Mamadeira, Soro, Chupeta/bico, Lenços umedecidos, Sabonetes, Escova de dentes, Shampoo e Condicionador, Plus Size/GG, Pás, Desodorante, Voluntário - Noite, Shampoo, Veterinários, Luvas, Remédios básicos, Roupas íntimas, Medicamentos prescritos, Turnos de manhã, Turnos de noite, Fraldas, Ventilador, Roupa Íntima (adulto) Feminina E Masculina, Luvas De Limpeza, Toalhas de banho, Ventiladores, Vassouras, Sacos de lixo, Pomadas, Feijão, Tênis, Chinelos, Oléo, Roupas para frio, Leite em pó, Gelo, Azitotromicina, Papel higiênico, Sacolinhas plasticas para kits, Gás, Ração para animais, Antibiótico, Suco de caixinha, Jornal (para xixi e coco), Caixa de areia (gato), Voluntários Para Animais (não Necessariamente Veterinário/a), Alimentos Diets, Luz de Emergência, Lanterna, Lonas, Água, Travesseiros/almofada, Chinelos masculinos, Água potável, Itens de limpeza, Roupa Íntima Infantil, Roupas Masculinas G E Gg, Azeite, Roupa íntima masculina e feminina, Roupas femininas G/GG, Roupa plus size, Arroz, Roupas grandes, Esponjas De Louça, Roupas para adultos, Sapato Infantil, Turno da madrugada, Sabão em pó, Pasta de dente, Caminhão Pipa, Fralda RN, Produtos de desinfecção (cloro, álcool), Copos, Voluntário - Madrugada, Roupas Femininas, Roupa Masculina, Roupas Pluz Size, Rolo Lona Preta 40m x 6m (mais grossa), Lona, Fraldas Geriatricas, Cordas, Casinha para cachorro, Lar Temporário Para Animais, Ponto De Resgate, Caixas de transporte para pets, Álcool em gel, Coleiras, Mascara, Cesta Básica, Roupa Masculina Gg, Patê Para Cachorro, Roupa Infantil Menino, Fita Durex larga, Papagaio - Urinar, Papagaio, Cama Geriatria, Escova de cabelo, Toalhas, Cadeira De Rodas, Leitos para animais, SACOS DE AREIA, Caixa De Transporte, Areia de gato, Macarrão, Desinfetante, Café, Pente/Escova de cabelo, Condicionador, Gilete, Alimentos para consumo rápido (Leite, bolacha, suco, etc), Colher Descartável, Lanches, Tapete higiênico, Medicamentos veterinários, Cobertores, Pão, Pão De Sanduíche, Sucos, Papel toalha, TELAS / CHIQUEIROS, Banana, Cebola, Frutas, Alface, Tomate, Luva G, Jogo de cartas, baralho, dama e dominó, etc, Seringa De Insulina, Seringa 3 E 5ml Com Agulha 70x30, ROUPAS INTIMAS - CUECAS - G, Bermuda Masculina, Hipoglós, Talheres, Tapetes higiênicos, Guardanapo de papel, Médicos, Psicólogos, Papelão, Voluntário - Manhã, Roupa Infantil 8-10 Anos, Roupa Masculina Adulta, Assistentes Sociais, Leite, Fralda P, Embalagem descartável para as marmitas, Sabonete infantil, Turnos de tarde, Antipulgas - Animais Pequeno Porte, Higiene Pessoal, Produtos de higiene, Garfo e faca, Pano de chão, Pano de prato, Potes (marmitex), Pilha Aa, Guarda-sol, Pomada para assadura, Fraldas Adultas, Meias, Luvas descartáveis, Baldes, Travesseiro, Talher Descartável, Detergente, Água Sanitária, Chimia, Sabonete Líquido, Luva de Latex Descartável, Pano De Chão / Saco Alvejado, Probiótico Para Animais, Escova Para Limpar Mamadeira, Molho de tomate, Açúcar, Voluntário - Tarde, Roupas leves, Toucas, Luvas Descartáveis, Lenço umedecido, Luminárias com pedestal para área saúde 1m altura pelo menos, Ganchos para montar varal, Freezer para armazenar comida, Máquina de lavar roupa, Luvas para limpeza, Bermudas, Assento para vaso sanitário, Calçado masculino, Jornal/Papelão, Frios, Carne Frango, Farinha, Travesseiros, Fronhas, Elástico Para Cabelo, Roupas plus size feminina, Malas de Viagem, Banheiro Químico (apenas Chuveiro), Bonés, Produtos de limpeza, Pano De Limpeza, Bolachinha, Vassouras e rodos, Fralda XG e XXG, Creme de pentear, Fita adesiva (durex), Roupa íntima feminina, Ração gato, Capas de chuva, Toalha de Banho, Guarda-chuva, Farinha de trigo, Gatorade/Isotônico, Latas de lixo, Massinha De Modelar, Roupas plus size masculino, Saco De Lixo De Vários Tamanhos, Baralhos, Erva Mate, Touca Descartável Sal, Polenta, Calçados, Itens de higienLanternas, Roupas para crianças, Comidas Não Perecíveis, Pratos, Lençóis, Cadeiras (tipo de praia), Absorventes higiênicos, Kits de primeiros socorros, Fórmula infantil, Colchões, Caixas de papelão, Lixeiras, Carne Gado, Gaze, Termômetro, Álcool, Repelente, Massa, Descarpack Caixa, Mamadeira, Soro, Chupeta/bico, Lenços umedecidos, Sabonetes, Escova de dentes, Shampoo e Condicionador, Plus Size/GG, Pás, Desodorante, Voluntário - Noite, Shampoo, Veterinários, Luvas, Remédios básicos, Roupas íntimas, Medicamentos prescritos, Turnos de manhã, Turnos de noite, Fraldas, Ventilador, Roupa Íntima (adulto) Feminina E Masculina, Luvas De Limpeza, Toalhas de banho, Ventiladores, Vassouras, Sacos de lixo, Pomadas, Feijão, Tênis, Chinelos, Oléo, Roupas para frio, Leite em pó, Gelo, Azitotromicina, Papel higiênico, Sacolinhas plasticas para kits, Gás, Ração para animais, Antibiótico, Suco de caixinha, Jornal (para xixi e coco), Caixa de areia (gato), Voluntários Para Animais (não Necessariamente Veterinário/a), Alimentos Diets, Luz de Emergência, Lanterna, Lonas, Água, Travesseiros/almofada, Chinelos masculinos, Água potável, Itens de limpeza, Roupa Íntima Infantil, Roupas Masculinas G E Gg, Azeite, Roupa íntima masculina e feminina, Roupas femininas G/GG, Roupa plus size, Arroz, Roupas grandes, Esponjas De Louça, Roupas para adultos, Sapato Infantil, Turno da madrugada, Sabão em pó, Pasta de dente, Caminhão Pipa, Fralda RN, Produtos de desinfecção (cloro, álcool), Copos, Voluntário - Madrugada, Roupas Femininas, Roupa Masculina, Roupas Pluz Size, Rolo Lona Preta 40m x 6m (mais grossa), Lona, Fraldas Geriatricas, Cordas, Casinha para cachorro, Lar Temporário Para Animais, Ponto De Resgate, Caixas de transporte para pets, Álcool em gel, Coleiras, Mascara, Cesta Básica, Roupa Masculina Gg, Patê Para Cachorro, Roupa Infantil Menino, Fita Durex larga, Papagaio - Urinar, Papagaio, Cama Geriatria, Escova de cabelo, Toalhas, Cadeira De Rodas, Leitos para animais, SACOS DE AREIA, Caixa De Transporte, Areia de gato, Macarrão, Desinfetante, Café, Pente/Escova de cabelo, Condicionador, Gilete, Alimentos para consumo rápido (Leite, bolacha, suco, etc), Colher Descartável, Lanches, Tapete higiênico, Medicamentos veterinários, Cobertores, Pão, Pão De Sanduíche, Sucos, Papel toalha, TELAS / CHIQUEIROS, Banana, Cebola, Frutas, Alface, Tomate, Luva G, Jogo de cartas, baralho, dama e dominó, etc, Seringa De Insulina, Seringa 3 E 5ml Com Agulha 70x30, ROUPAS INTIMAS - CUECAS - G, Bermuda Masculina, Hipoglós, Talheres, Tapetes higiênicos, Guardanapo de papel, Médicos, Psicólogos, Papelão, Voluntário - Manhã, Roupa Infantil 8-10 Anos, Roupa Masculina Adulta, Assistentes Sociais, Leite, Fralda P, Embalagem descartável para as marmitas, Sabonete infantil, Turnos de tarde, Antipulgas - Animais Pequeno Porte, Higiene Pessoal, Produtos de higiene, Garfo e faca, Pano de chão, Pano de prato, Potes (marmitex), Pilha Aa, Guarda-sol, Pomada para assadura, Fraldas Adultas, Meias, Luvas descartáveis, Baldes, Travesseiro, Talher Descartável, Detergente, Água Sanitária, Chimia, Sabonete Líquido, Luva de Latex Descartável, Pano De Chão / Saco Alvejado, Probiótico Para Animais, Escova Para Limpar Mamadeira, Molho de tomate, Açúcar, Voluntário - Tarde, Roupas leves, Toucas, Luvas Descartáveis, Lenço umedecido, Luminárias com pedestal para área saúde 1m altura pelo menos, Ganchos para montar varal, Freezer para armazenar comida, Máquina de lavar roupa, Luvas para limpeza, Bermudas, Assento para vaso sanitário, Calçado masculino, Jornal/Papelão, Frios, Carne Frango, Farinha, Travesseiros, Fronhas, Elástico Para Cabelo, Roupas plus size feminina, Malas de Viagem, Banheiro Químico (apenas Chuveiro), Bonés, Produtos de limpeza, Pano De Limpeza, Bolachinha, Vassouras e rodos, Fralda XG e XXG, Creme de pentear, Fita adesiva (durex), Roupa íntima feminina, Ração gato, Capas de chuva, Toalha de Banho, Guarda-chuva, Farinha de trigo, Gatorade/Isotônico, Latas de lixo, Massinha De Modelar, Roupas plus size masculino, Saco De Lixo De Vários Tamanhos, Baralhos, Erva Mate, Touca Descartável, Sal, Polenta, Calçados, Itens de higiena pessoal, Achocolatado pronto, Roupas de Camas, Sacolas Para Montar Kits, Sacolas ou sacos plásticos, Técnico De Enfermagem, Enfermeirosa pessoal, Achocolatado pronto, Roupas de Camas, Sacolas Para Montar Kits, Sacolas ou sacos plásticos, Técnico De Enfermagem, Enfermeiros', + }, + ], + }, + { + role: 'model', + parts: [ + { + text: '{"medicamentos": ["Kits de primeiros socorros", "Gaze", "Termômetro", "Álcool", "Repelente", "Soro", "Remédios básicos", "Medicamentos prescritos", "Pomadas", "Azitotromicina", "Antibiótico", "Álcool em gel", "Pomada para assadura", "Desinfetante", "Medicamentos veterinários", "Seringa De Insulina", "Seringa 3 E 5ml Com Agulha 70x30", "Hipoglós", "Antipulgas - Animais Pequeno Porte", "Probiótico Para Animais"], "cuidados com animais": ["Ração para animais", "Caixa de areia (gato)", "Voluntários Para Animais (não Necessariamente Veterinário/a)", "Coleiras", "Patê Para Cachorro", "Casinha para cachorro", "Lar Temporário Para Animais", "Caixas de transporte para pets", "Leitos para animais", "SACOS DE AREIA", "Caixa De Transporte", "Areia de gato", "TELAS / CHIQUEIROS", "Tapetes higiênicos", "Ração gato"], "especialistas e profissionais": ["Veterinários", "Voluntário - Noite", "Turnos de manhã", "Turnos de noite", "Turno da madrugada", "Voluntário - Madrugada", "Voluntário - Manhã", "Médicos", "Psicólogos", "Assistentes Sociais", "Turnos de tarde", "Voluntário - Tarde", "Técnico De Enfermagem", "Enfermeiros"], "acomodações e descanso": ["Lençóis", "Cadeiras (tipo de praia)", "Colchões", "Lonas", "Travesseiros/almofada", "Rolo Lona Preta 40m x 6m (mais grossa)", "Lona", "Cordas", "Cama Geriatria", "Cobertores", "Guarda-sol", "Travesseiro", "Luminárias com pedestal para área saúde 1m altura pelo menos", "Ganchos para montar varal", "Bermudas", "Capas de chuva", "Guarda-chuva", "Roupas de Camas"], "equipamentos de emergência": ["Lanternas", "Luz de Emergência", "Lanterna", "Ventilador", "Ventiladores", "Caminhão Pipa", "Fita Durex larga", "Cadeira De Rodas", "Pilha Aa", "Baldes"], "voluntariado": ["Voluntário - Noite", "Turnos de manhã", "Turnos de noite", "Turno da madrugada", "Voluntário - Madrugada", "Voluntário - Manhã", "Turnos de tarde", "Voluntário - Tarde"], "itens descartáveis": ["Pratos", "Descarpack Caixa", "Mamadeira", "Chupeta/bico", "Fraldas", "Sacos de lixo", "Sacolinhas plasticas para kits", "Jornal (para xixi e coco)", "Copos", "Papagaio - Urinar", "Colher Descartável", "Papel toalha", "Guardanapo de papel", "Embalagem descartável para as marmitas", "Garfo e faca", "Potes (marmitex)", "Talher Descartável", "Luva de Latex Descartável", "Escova Para Limpar Mamadeira", "Toucas", "Luvas Descartáveis", "Lenço umedecido", "Assento para vaso sanitário", "Jornal/Papelão", "Touca Descartável", "Sacolas Para Montar Kits", "Sacolas ou sacos plásticos"], "higiene pessoal": ["Absorventes higiênicos", "Lenços umedecidos", "Sabonetes", "Escova de dentes", "Shampoo e Condicionador", "Pás", "Desodorante", "Shampoo", "Roupas íntimas", "Roupa Íntima (adulto) Feminina E Masculina", "Toalhas de banho", "Papel higiênico", "Roupa Íntima Infantil", "Sabão em pó", "Pasta de dente", "Produtos de desinfecção (cloro, álcool)", "Escova de cabelo", "Toalhas", "Pente/Escova de cabelo", "Condicionador", "Gilete", "Tapete higiênico", "Sabonete infantil", "Higiene Pessoal", "Produtos de higiene", "Pano de chão", "Pano de prato", "Detergente", "Água Sanitária", "Chimia", "Sabonete Líquido", "Pano De Chão / Saco Alvejado", "Esponjas De Louça", "Lenço umedecido", "Luvas para limpeza", "Produtos de limpeza", "Pano De Limpeza", "Creme de pentear", "Toalha de Banho", "Itens de higiena pessoal"], "alimentos e água": ["Roupas para crianças", "Comidas Não Perecíveis", "Fórmula infantil", "Carne Gado", "Massa", "Feijão", "Oléo", "Leite em pó", "Gelo", "Suco de caixinha", "Alimentos Diets", "Água", "Chinelos masculinos", "Água potável", "Azeite", "Arroz", "Macarrão", "Café", "Alimentos para consumo rápido (Leite, bolacha, suco, etc)", "Lanches", "Pão", "Pão De Sanduíche", "Sucos", "Banana", "Cebola", "Frutas", "Alface", "Tomate", "Leite", "Molho de tomate", "Açúcar", "Frios", "Carne Frango", "Farinha", "Bolachinha", "Farinha de trigo", "Gatorade/Isotônico", "Massinha De Modelar", "Sal", "Polenta", "Achocolatado pronto"], "material de limpeza": ["Caixas de papelão", "Lixeiras", "Luvas De Limpeza", "Vassouras", "Sacos de lixo", "Itens de limpeza", "Produtos de desinfecção (cloro, álcool)", "Desinfetante", "Vassouras e rodos", "Latas de lixo", "Saco De Lixo De Vários Tamanhos"], "vestuário": ["Roupas para crianças", "Plus Size/GG", "Luvas", "Roupas para frio", "Roupas Masculinas G E Gg", "Roupa íntima masculina e feminina", "Roupas femininas G/GG", "Roupa plus size", "Roupas grandes", "Roupas para adultos", "Sapato Infantil", "Roupas Femininas", "Roupa Masculina", "Roupas Pluz Size", "Fraldas Geriatricas", "Mascara", "Roupa Masculina Gg", "Roupa Infantil Menino", "Roupas INTIMAS - CUECAS - G", "Bermuda Masculina", "Roupa Infantil 8-10 Anos", "Roupa Masculina Adulta", "Fralda P", "Fraldas Adultas", "Meias", "Roupas leves", "Bermudas", "Calçado masculino", "Roupas plus size feminina", "Bonés", "Roupa íntima feminina", "Fralda XG e XXG", "Roupas plus size masculino", "Calçados", "Roupa Íntima Infantil"], "veículos de resgate e transporte": ["Caminhão Pipa"], "eletrodomésticos e eletrônicos": ["Ventilador", "Ventiladores", "Luz de Emergência", "Lanterna", "Freezer para armazenar comida", "Máquina de lavar roupa", "Luminárias com pedestal para área saúde 1m altura pelo menos"], "mobílias": ["Cadeiras (tipo de praia)", "Colchões", "Cama Geriatria"], "jogos e passatempo": ["Jogo de cartas, baralho, dama e dominó, etc", "Baralhos"], "cosméticos": ["Shampoo e Condicionador", "Pás", "Desodorante", "Shampoo", "Creme de pentear"]}\n', + }, + ], + }, + ], + }); + let promptOutput: Record = {}; + try { + const result = await chatSession.sendMessage(input); + const response = result.response.text(); + promptOutput = JSON.parse(response); + } catch (error) { + logger.error(error); + } + + return promptOutput; +} + +class AtomicCounter { + private _successCount = 0; + private _totalCount = 0; + private _failureCount = 0; + + public get totalCount() { + return this._totalCount; + } + public get successCount() { + return this._successCount; + } + public get failureCount() { + return this._failureCount; + } + + incrementSuccess() { + this._successCount += 1; + } + + increment() { + this._totalCount += 1; + } + + incrementFailure() { + this._failureCount += 1; + } +} + +/** + * ```js + * // Pode ser uma stream de arquivo + * const fileSourceStream = createReadStream(__dirname + '/planilha_porto_alegre - comida.csv'); + * // Ou uma url de um arquivo CSV + * const csvUrl = 'https://docs.google.com/spreadsheets/d/18hY52i65lIdLE2UsugjnKdnE_ubrBCI6nCR0XQurSBk/gviz/tq?tqx=out:csv&sheet=planilha_porto_alegre'; + * // passar os headers + * parseCsv({fileStream: fileSourceStream, csvUrl,headers:{ nameField:'nome' ,addressField:'endereco',latitudeField:'lat'}}) + * ``` + * + + */ +export async function shelterToCsv({ + headers, + csvUrl, + fileStream, +}: ParseCsvArgs) { + const validInput = (csvUrl && URL.canParse(csvUrl)) || fileStream != null; + const atomicCounter = new AtomicCounter(); + if (!validInput) { + logger.warn('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); + throw new Error('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); + } + + const shelters: ShelterInput[] = []; + + let csvSourceStream: Readable; + if (csvUrl) { + csvSourceStream = responseToReadable(await fetch(csvUrl)); + } else { + csvSourceStream = fileStream!; + } + + const prismaService = new PrismaService(); + + const [categories, supplies] = await prismaService.$transaction([ + prismaService.supplyCategory.findMany({}), + prismaService.supply.findMany({ distinct: ['name'] }), + ]); + + const missingShelterSupplies = new Set(); + + const suppliesAvailable = supplies.reduce((acc, item) => { + acc.set(item.name.trim().toLowerCase(), item.id); + return acc; + }, new Map()); + + const categoriesAvailable = categories.reduce((acc, item) => { + acc.set(item.name.trim().toLowerCase(), item.id); + return acc; + }, new Map()); + + let shelterSupliesByCategory: Record = {}; + await Readable.toWeb(csvSourceStream) + .pipeThrough(Transform.toWeb(csvParser)) + .pipeThrough(new CsvToShelterTransformStream(headers)) + .pipeThrough( + new TransformStream({ + transform(shelter, controller) { + atomicCounter.increment(); + if (shelter?.supplies?.length) { + for (const supply of shelter.supplies) { + if (suppliesAvailable.has(supply)) { + continue; + } + if (!missingShelterSupplies.has(supply)) { + missingShelterSupplies.add(supply); + } + } + } + shelters.push(shelter); + controller.enqueue(shelter); + }, + }), + ) + .pipeTo( + new WritableStream({ + async close() { + const missingSheltersString = Array.from(missingShelterSupplies).join( + ', ', + ); + shelterSupliesByCategory = await detectSupplyCategoryUsingAI( + missingSheltersString, + Array.from(categoriesAvailable.keys()), + ); + }, + }), + ); + + await Readable.toWeb(Readable.from(shelters)) + .pipeThrough( + new ShelterEnhancedStreamTransformer({ + categoriesAvailable, + shelterSupliesByCategory, + suppliesAvailable, + counter: atomicCounter, + }), + ) + + .pipeTo( + new WritableStream({ + async write( + shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, + ) { + await prismaService.shelter + .create({ data: shelter, select: { name: true, id: true } }) + .then((d) => { + atomicCounter.incrementSuccess(); + logger.debug(d); + }) + .catch((e: Error) => { + atomicCounter.incrementFailure(); + if (e instanceof PrismaClientKnownRequestError) { + logger.error(translatePrismaError(e)); + } else { + logger.error(e); + } + }); + }, + close() { + logger.log( + `${atomicCounter.successCount} de ${atomicCounter.totalCount} processados. ${atomicCounter.failureCount} com erro.`, + ); + }, + }), + ); + + return { + successCount: atomicCounter.successCount, + totalCount: atomicCounter.totalCount, + failureCount: atomicCounter.failureCount, + }; +} diff --git a/src/shelter-csv-importer/shelter-csv-importer.module.ts b/src/shelter-csv-importer/shelter-csv-importer.module.ts new file mode 100644 index 00000000..d7758b1d --- /dev/null +++ b/src/shelter-csv-importer/shelter-csv-importer.module.ts @@ -0,0 +1,4 @@ +import { Module } from '@nestjs/common'; + +@Module({}) +export class ShelterCsvImporterModule {} diff --git a/src/shelter-csv-importer/types.ts b/src/shelter-csv-importer/types.ts new file mode 100644 index 00000000..e69de29b From 88f72566c39f92cc3220e546c65d367bff341151 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Thu, 16 May 2024 09:53:52 -0300 Subject: [PATCH 04/16] test: adding tests for shelter-csv-importer --- .vscode/settings.json | 4 + package-lock.json | 62 +++ package.json | 1 + ...dex.ts => shelter-csv-importer.helpers.ts} | 428 ++---------------- .../shelter-csv-importer.module.ts | 7 +- .../shelter-csv-importer.service.spec.ts | 110 +++++ .../shelter-csv-importer.service.ts | 155 +++++++ .../shelter-csv.transformer.ts | 166 +++++++ src/shelter-csv-importer/types.ts | 86 ++++ .../gemini_prompt_response_example.json | 291 ++++++++++++ test/examples/sheet_example.csv | 7 + 11 files changed, 924 insertions(+), 393 deletions(-) create mode 100644 .vscode/settings.json rename src/shelter-csv-importer/{index.ts => shelter-csv-importer.helpers.ts} (59%) create mode 100644 src/shelter-csv-importer/shelter-csv-importer.service.spec.ts create mode 100644 src/shelter-csv-importer/shelter-csv-importer.service.ts create mode 100644 src/shelter-csv-importer/shelter-csv.transformer.ts create mode 100644 test/examples/gemini_prompt_response_example.json create mode 100644 test/examples/sheet_example.csv diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..4b8293b6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "ai-test.endpoint": "a", + "ai-test.apiKey": "a" +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 2e9f65a3..639bce99 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "zod": "^3.23.6" }, "devDependencies": { + "@anatine/zod-mock": "^3.13.4", "@nestjs/cli": "^10.0.0", "@nestjs/schematics": "^10.0.0", "@nestjs/testing": "^10.0.0", @@ -67,6 +68,19 @@ "node": ">=6.0.0" } }, + "node_modules/@anatine/zod-mock": { + "version": "3.13.4", + "resolved": "https://registry.npmjs.org/@anatine/zod-mock/-/zod-mock-3.13.4.tgz", + "integrity": "sha512-yO/KeuyYsEDCTcQ+7CiRuY3dnafMHIZUMok6Ci7aERRCTQ+/XmsiPk/RnMx5wlLmWBTmX9kw+PavbMsjM+sAJA==", + "dev": true, + "dependencies": { + "randexp": "^0.5.3" + }, + "peerDependencies": { + "@faker-js/faker": "^7.0.0 || ^8.0.0", + "zod": "^3.21.4" + } + }, "node_modules/@angular-devkit/core": { "version": "17.1.2", "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-17.1.2.tgz", @@ -950,6 +964,23 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@faker-js/faker": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.4.1.tgz", + "integrity": "sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/fakerjs" + } + ], + "peer": true, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0", + "npm": ">=6.14.13" + } + }, "node_modules/@fastify/accept-negotiator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-1.1.0.tgz", @@ -4368,6 +4399,15 @@ "node": ">=6.0.0" } }, + "node_modules/drange": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/drange/-/drange-1.1.1.tgz", + "integrity": "sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -8121,6 +8161,28 @@ "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, + "node_modules/randexp": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/randexp/-/randexp-0.5.3.tgz", + "integrity": "sha512-U+5l2KrcMNOUPYvazA3h5ekF80FHTUG+87SEAmHZmolh1M+i/WyTCxVzmi+tidIa1tM4BSe8g2Y/D3loWDjj+w==", + "dev": true, + "dependencies": { + "drange": "^1.0.2", + "ret": "^0.2.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/randexp/node_modules/ret": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz", + "integrity": "sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", diff --git a/package.json b/package.json index b3eea7c8..eff4479b 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "zod": "^3.23.6" }, "devDependencies": { + "@anatine/zod-mock": "^3.13.4", "@nestjs/cli": "^10.0.0", "@nestjs/schematics": "^10.0.0", "@nestjs/testing": "^10.0.0", diff --git a/src/shelter-csv-importer/index.ts b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts similarity index 59% rename from src/shelter-csv-importer/index.ts rename to src/shelter-csv-importer/shelter-csv-importer.helpers.ts index bd420950..1aebd417 100644 --- a/src/shelter-csv-importer/index.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts @@ -4,105 +4,22 @@ import { HarmCategory, } from '@google/generative-ai'; import { Logger } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; import { parse as createParser } from 'csv'; -import { ReadStream } from 'node:fs'; -import * as path from 'node:path'; -import { Readable, Transform } from 'node:stream'; -import { TransformStream, WritableStream } from 'node:stream/web'; +import { Readable } from 'node:stream'; -import { PrismaService } from '../prisma/prisma.service'; -import { CreateShelterSchema } from '../shelter/types/types'; -import { responseToReadable } from '../utils/utils'; +const logger = new Logger('ShelterCsvImporterHelpers'); -type ShelterKey = Exclude< - Prisma.ShelterScalarFieldEnum, - 'createdAt' | 'updatedAt' | 'prioritySum' | 'verified' | 'id' ->; -type ShelterColumHeader = Record<`${ShelterKey}Field`, string> & { - shelterSuppliesField: string; -}; -interface ParseCsvArgsBaseArgs> { - /** - * link do arquivo CSV - */ - csvUrl?: string; - /** - * stream proveniente de algum arquivo CSV - */ - fileStream?: ReadStream; - /** - * mapeamento de quais cabeçalhos do csv serão usados como colunas da tabela. - */ - headers?: Partial; -} - -export type ParseCsvArgs = - | (ParseCsvArgsBaseArgs & { csvUrl: string }) - | (ParseCsvArgsBaseArgs & { fileStream: ReadStream }); - -interface EnhancedTransformArgs { - /** - * KeyValue contento as categorias e os supplies contidos naquela categorias (detectados por IA) - */ - shelterSupliesByCategory: Record; - /** - * KeyValue com Suprimentos já encontrados na base atual que podem ser utilizadas em relações - * @example - * { "Óleo de cozinha": "eb1d5056-8b9b-455d-a179-172a747e3f20", - * "Álcool em gel": "a3e3bdf8-0be4-4bdc-a3b0-b40ba931be5f" - * } - */ - suppliesAvailable: Map; - /** - * KeyValue com Categorias já encontradas na base atual que podem ser utilizadas em relações - * @example - * { "Higiene Pessoal": "718d5be3-69c3-4216-97f1-12b690d0eb97", - * "Alimentos e Água": "a3e3bdf8-0be4-4bdc-a3b0-b40ba931be5f" - * } - */ - categoriesAvailable: Map; - counter?: AtomicCounter; -} -const CSV_DEFAULT_HEADERS: ShelterColumHeader = { - nameField: 'nome_do_local', - addressField: 'endereco', - contactField: 'whatsapp', - latitudeField: 'lat', - longitudeField: 'lng', - shelterSuppliesField: 'itens_em_falta', - capacityField: 'capacidade', - cityField: 'cidade', - neighbourhoodField: 'bairro', - petFriendlyField: 'pet_friendly', - pixField: 'pix', - shelteredPeopleField: 'pessoas_abrigadas', - streetField: 'rua', - streetNumberField: 'numero', - zipCodeField: 'cep', -}; - -type ShelterInput = Partial; -/** - * Padrão utilizado - * `nome_do_local, endereco, whatsapp, lat, lng, itens_disponiveis, itens_em_falta` - */ - -// Regex que ignora vírgulas dentro de parenteses no split -const COLON_REGEX = /(? + createParser({ columns: true, relaxColumnCount: true }, function (err, data) { + if (err) { + logger.error(err); + return null; + } + return data; + }); -function translatePrismaError(err: PrismaClientKnownRequestError) { +export function translatePrismaError(err: PrismaClientKnownRequestError) { switch (err.code) { case 'P2002': case 'P2003': @@ -127,165 +44,7 @@ function translatePrismaError(err: PrismaClientKnownRequestError) { } } -/** - * JSON -> ShelterInput - * @see ShelterInput - */ -class CsvToShelterTransformStream extends TransformStream< - unknown, - ShelterInput -> { - /** - * Espera um Ojeto contento a assinatura da entidade esperada - * @param columnNames dicionário com nomes das colunas a serem mapeadas - */ - constructor(columnNames: Partial = CSV_DEFAULT_HEADERS) { - const efffectiveColumnNames = {} as ShelterColumHeader; - Object.entries(CSV_DEFAULT_HEADERS).forEach(([key, value]) => { - efffectiveColumnNames[key] = - typeof columnNames[key] === 'string' ? columnNames[key] : value; - }); - - super({ - async transform(chunk, controller) { - if (!chunk || (chunk && typeof chunk !== 'object')) { - return; - } - let supplies: string[] = []; - - if ( - typeof chunk[efffectiveColumnNames.shelterSuppliesField] === 'string' - ) { - supplies = (chunk[efffectiveColumnNames.shelterSuppliesField]) - .split(COLON_REGEX) - .filter(Boolean) - .map((s) => s.trim()); - } - - const shelter: ShelterInput = { - verified: false, - // Removendo duplicidades - supplies: [...new Set(supplies)], - }; - - Object.keys(Prisma.ShelterScalarFieldEnum).forEach((key) => { - shelter[key] ??= chunk[efffectiveColumnNames[`${key}Field`]]; - }); - - controller.enqueue(shelter); - logger.verbose( - '🚀 ~ CsvToShelterTransformStream ~ constructor ~ shelter:', - shelter, - ); - }, - }); - } -} - -/** - * Valida o schema do Input, enriquece o modelo e adicionas as relações encontradas - */ -class ShelterEnhancedStreamTransformer extends TransformStream< - ShelterInput, - ReturnType -> { - private counter!: AtomicCounter; - - /** - * - */ - constructor({ - shelterSupliesByCategory, - suppliesAvailable, - categoriesAvailable, - counter, - }: EnhancedTransformArgs) { - super({ - transform: async (shelter: ShelterInput, controller) => { - this.counter ??= counter || new AtomicCounter(); - if (!shelter.supplies) { - try { - controller.enqueue(CreateShelterSchema.parse(shelter)); - } catch (error) { - this.counter.incrementFailure(); - logger.error(error, shelter); - } - return; - } - - const missingSupplyNames = new Set(); - - const toCreate: Prisma.ShelterSupplyCreateNestedManyWithoutShelterInput['create'] = - []; - - for (const supplyName of missingSupplyNames.values()) { - for (const [categoryName, values] of Object.entries( - shelterSupliesByCategory, - )) { - const indexFound = values.findIndex( - (item) => item.toLowerCase() === supplyName.toLowerCase(), - ); - if (indexFound !== -1) { - toCreate.push({ - supplyId: suppliesAvailable.get(supplyName.toLowerCase())!, - supply: { - connect: { - id: suppliesAvailable.get(supplyName.toLowerCase())!, - name: supplyName, - }, - connectOrCreate: { - where: { - id: suppliesAvailable.get(supplyName.toLowerCase()), - }, - create: { - name: supplyName, - supplyCategory: { - connect: { - name: categoryName, - id: categoriesAvailable.get( - categoryName.toLowerCase(), - ), - }, - }, - }, - }, - }, - }); - } - } - } - shelter.shelterSupplies = { - create: toCreate, - }; - - if (shelter.latitude) { - shelter.latitude = Number.parseFloat(`${shelter.latitude}`); - } - if (shelter.longitude) { - shelter.longitude = Number.parseFloat(`${shelter.longitude}`); - } - - Object.keys(shelter).forEach((key) => { - if ( - typeof shelter[key] === 'string' && - shelter[key].trim().length === 0 - ) { - shelter[key] = null; - } - }); - - await CreateShelterSchema.parseAsync(shelter) - .then((s) => controller.enqueue(s)) - .catch((e) => { - this.counter.incrementFailure(); - logger.error(e.message, shelter); - }); - }, - }); - } -} - -async function detectSupplyCategoryUsingAI( +export async function detectSupplyCategoryUsingAI( input: unknown, categoriesAvailable: string[], ): Promise> { @@ -296,10 +55,7 @@ async function detectSupplyCategoryUsingAI( const apiKey = process.env.GEMINI_API_KEY; if (!apiKey) { - const res = require( - path.resolve(__dirname, '..', '..', 'promp_response.json'), - ); - return res; + throw new Error('Required ENV variable: GEMINI_API_KEY'); } const genAI = new GoogleGenerativeAI(apiKey); @@ -359,6 +115,7 @@ async function detectSupplyCategoryUsingAI( }, ], }); + let promptOutput: Record = {}; try { const result = await chatSession.sendMessage(input); @@ -371,7 +128,28 @@ async function detectSupplyCategoryUsingAI( return promptOutput; } -class AtomicCounter { +export function responseToReadable(response: Response) { + const reader = response.body?.getReader(); + if (!reader) { + return new Readable(); + } + + return new Readable({ + async read() { + const result = await reader.read(); + if (!result.done) { + this.push(Buffer.from(result.value)); + } else { + this.push(null); + return; + } + }, + }); +} +/** + * Classe utilitária apenas com propósito de facilitar logging de processamento em stream + */ +export class AtomicCounter { private _successCount = 0; private _totalCount = 0; private _failureCount = 0; @@ -398,137 +176,3 @@ class AtomicCounter { this._failureCount += 1; } } - -/** - * ```js - * // Pode ser uma stream de arquivo - * const fileSourceStream = createReadStream(__dirname + '/planilha_porto_alegre - comida.csv'); - * // Ou uma url de um arquivo CSV - * const csvUrl = 'https://docs.google.com/spreadsheets/d/18hY52i65lIdLE2UsugjnKdnE_ubrBCI6nCR0XQurSBk/gviz/tq?tqx=out:csv&sheet=planilha_porto_alegre'; - * // passar os headers - * parseCsv({fileStream: fileSourceStream, csvUrl,headers:{ nameField:'nome' ,addressField:'endereco',latitudeField:'lat'}}) - * ``` - * - - */ -export async function shelterToCsv({ - headers, - csvUrl, - fileStream, -}: ParseCsvArgs) { - const validInput = (csvUrl && URL.canParse(csvUrl)) || fileStream != null; - const atomicCounter = new AtomicCounter(); - if (!validInput) { - logger.warn('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); - throw new Error('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); - } - - const shelters: ShelterInput[] = []; - - let csvSourceStream: Readable; - if (csvUrl) { - csvSourceStream = responseToReadable(await fetch(csvUrl)); - } else { - csvSourceStream = fileStream!; - } - - const prismaService = new PrismaService(); - - const [categories, supplies] = await prismaService.$transaction([ - prismaService.supplyCategory.findMany({}), - prismaService.supply.findMany({ distinct: ['name'] }), - ]); - - const missingShelterSupplies = new Set(); - - const suppliesAvailable = supplies.reduce((acc, item) => { - acc.set(item.name.trim().toLowerCase(), item.id); - return acc; - }, new Map()); - - const categoriesAvailable = categories.reduce((acc, item) => { - acc.set(item.name.trim().toLowerCase(), item.id); - return acc; - }, new Map()); - - let shelterSupliesByCategory: Record = {}; - await Readable.toWeb(csvSourceStream) - .pipeThrough(Transform.toWeb(csvParser)) - .pipeThrough(new CsvToShelterTransformStream(headers)) - .pipeThrough( - new TransformStream({ - transform(shelter, controller) { - atomicCounter.increment(); - if (shelter?.supplies?.length) { - for (const supply of shelter.supplies) { - if (suppliesAvailable.has(supply)) { - continue; - } - if (!missingShelterSupplies.has(supply)) { - missingShelterSupplies.add(supply); - } - } - } - shelters.push(shelter); - controller.enqueue(shelter); - }, - }), - ) - .pipeTo( - new WritableStream({ - async close() { - const missingSheltersString = Array.from(missingShelterSupplies).join( - ', ', - ); - shelterSupliesByCategory = await detectSupplyCategoryUsingAI( - missingSheltersString, - Array.from(categoriesAvailable.keys()), - ); - }, - }), - ); - - await Readable.toWeb(Readable.from(shelters)) - .pipeThrough( - new ShelterEnhancedStreamTransformer({ - categoriesAvailable, - shelterSupliesByCategory, - suppliesAvailable, - counter: atomicCounter, - }), - ) - - .pipeTo( - new WritableStream({ - async write( - shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, - ) { - await prismaService.shelter - .create({ data: shelter, select: { name: true, id: true } }) - .then((d) => { - atomicCounter.incrementSuccess(); - logger.debug(d); - }) - .catch((e: Error) => { - atomicCounter.incrementFailure(); - if (e instanceof PrismaClientKnownRequestError) { - logger.error(translatePrismaError(e)); - } else { - logger.error(e); - } - }); - }, - close() { - logger.log( - `${atomicCounter.successCount} de ${atomicCounter.totalCount} processados. ${atomicCounter.failureCount} com erro.`, - ); - }, - }), - ); - - return { - successCount: atomicCounter.successCount, - totalCount: atomicCounter.totalCount, - failureCount: atomicCounter.failureCount, - }; -} diff --git a/src/shelter-csv-importer/shelter-csv-importer.module.ts b/src/shelter-csv-importer/shelter-csv-importer.module.ts index d7758b1d..28b03139 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.module.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.module.ts @@ -1,4 +1,9 @@ import { Module } from '@nestjs/common'; +import { PrismaModule } from '../prisma/prisma.module'; +import { ShelterCsvImporterService } from './shelter-csv-importer.service'; -@Module({}) +@Module({ + imports: [PrismaModule], + providers: [ShelterCsvImporterService], +}) export class ShelterCsvImporterModule {} diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts b/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts new file mode 100644 index 00000000..d88d51e7 --- /dev/null +++ b/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts @@ -0,0 +1,110 @@ +import { generateMock } from '@anatine/zod-mock'; +import { Test, TestingModule } from '@nestjs/testing'; +import { ShelterSchema } from 'src/shelter/types/types'; +import { SupplyCategorySchema } from 'src/supply-categories/types'; +import { SupplySchema } from 'src/supply/types'; +import { Readable } from 'node:stream'; +import { PrismaService } from '../prisma/prisma.service'; +import * as helpers from './shelter-csv-importer.helpers'; +import { ShelterCsvImporterService } from './shelter-csv-importer.service'; + +describe('ShelterCsvImporterService', () => { + let service: ShelterCsvImporterService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ShelterCsvImporterService], + }) + .useMocker((token) => { + if (token !== PrismaService) return; + + return { + shelter: { + create: jest.fn().mockResolvedValue(generateMock(ShelterSchema)), + }, + supply: { + findMany: jest.fn().mockResolvedValue([generateMock(SupplySchema)]), + }, + supplyCategory: { + findMany: jest + .fn() + .mockResolvedValue([generateMock(SupplyCategorySchema)]), + }, + $transaction: jest + .fn(PrismaService.prototype.$transaction) + .mockResolvedValue([ + [generateMock(SupplyCategorySchema)], + [generateMock(SupplySchema)], + ]), + }; + }) + .compile(); + + service = module.get(ShelterCsvImporterService); + }); + + test('test_shelterToCsv_withoutRequiredInputs', async () => { + + await expect(service.shelterToCsv({ headers: {} } as any)).rejects.toThrow( + 'Um dos campos `csvUrl` ou `fileStream` é obrigatório', + ); + }); + + test('test_shelterToCsv_withValidFileStream', async () => { + const mockFileStream = new Readable(); + mockFileStream.push('name,address,lat,lng,number,street, capacity\n'); + mockFileStream.push('name,address,1.0214,1.54525, 1234, Street,10.123'); + mockFileStream.push(null); + + jest.spyOn(global, 'fetch').mockResolvedValueOnce({ + body: mockFileStream, + } as any); + + const result = await service.shelterToCsv({ + fileStream: mockFileStream, + headers: { + nameField: 'name', + addressField: 'address', + latitudeField: 'lat', + longitudeField: 'lng', + streetField: 'street', + streetNumberField: 'number', + capacityField: 'capacity', + }, + }); + + expect(result.failureCount).toBe(0); + expect(result.successCount).toBeGreaterThan(0); + expect(result.totalCount).toBeGreaterThan(0); + }); + + test('test_shelterToCsv_missingSuppliesCategorization', async () => { + const mockFileStream = new Readable(); + mockFileStream.push('name,address,lat,lng,supplies, UnknownSupply\n'); + mockFileStream.push('name,address,1.0214,1.54525, UnknownSupply'); + mockFileStream.push(null); + + jest.spyOn(global, 'fetch').mockResolvedValueOnce({ + body: mockFileStream, + } as any); + + jest.spyOn(helpers, 'detectSupplyCategoryUsingAI').mockResolvedValue({ + UnknownSupply: ['NewCategory'], + }); + + const result = await service.shelterToCsv({ + fileStream: mockFileStream, + headers: { + nameField: 'name', + addressField: 'address', + latitudeField: 'lat', + shelterSuppliesField: 'supplies', + longitudeField: 'lng', + }, + }); + + expect(result.failureCount).toBe(0); + expect(result.totalCount).toBeGreaterThan(0); + expect(result.successCount).toBeGreaterThan(0); + }); +}); diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.ts b/src/shelter-csv-importer/shelter-csv-importer.service.ts new file mode 100644 index 00000000..9ae4f0d3 --- /dev/null +++ b/src/shelter-csv-importer/shelter-csv-importer.service.ts @@ -0,0 +1,155 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; +import { Readable, Transform } from 'node:stream'; +import { TransformStream } from 'node:stream/web'; + +import { PrismaService } from '../prisma/prisma.service'; +import { CreateShelterSchema } from '../shelter/types/types'; +import { + AtomicCounter, + createCsvParser, + detectSupplyCategoryUsingAI, + responseToReadable, + translatePrismaError, +} from './shelter-csv-importer.helpers'; +import { + CsvToShelterTransformStream, + ShelterEnhancedStreamTransformer, +} from './shelter-csv.transformer'; +import { ParseCsvArgs, ShelterColumHeader, ShelterInput } from './types'; + +@Injectable() +export class ShelterCsvImporterService { + private readonly logger = new Logger(ShelterCsvImporterService.name); + constructor(private readonly prismaService: PrismaService) {} + + /** + * ```js + * // Pode ser uma stream de arquivo + * const fileSourceStream = createReadStream(__dirname + '/planilha_porto_alegre - comida.csv'); + * // Ou uma url de um arquivo CSV + * const csvUrl = 'https://docs.google.com/spreadsheets/d/18hY52i65lIdLE2UsugjnKdnE_ubrBCI6nCR0XQurSBk/gviz/tq?tqx=out:csv&sheet=planilha_porto_alegre'; + * // passar os headers + * parseCsv({fileStream: fileSourceStream, csvUrl,headers:{ nameField:'nome' ,addressField:'endereco',latitudeField:'lat'}}) + * ``` + * + */ + async shelterToCsv({ + headers, + csvUrl, + fileStream, + }: ParseCsvArgs) { + const validInput = (csvUrl && URL.canParse(csvUrl)) || fileStream != null; + + if (!validInput) { + this.logger.warn('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); + throw new Error('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); + } + const atomicCounter = new AtomicCounter(); + const shelters: ShelterInput[] = []; + + let csvSourceStream: Readable; + if (csvUrl) { + csvSourceStream = responseToReadable(await fetch(csvUrl)); + } else { + csvSourceStream = fileStream!; + } + + const [categories, supplies] = await this.prismaService.$transaction([ + this.prismaService.supplyCategory.findMany({}), + this.prismaService.supply.findMany({ distinct: ['name'] }), + ]); + + const missingShelterSupplies = new Set(); + + const suppliesAvailable = supplies.reduce((acc, item) => { + acc.set(item.name.trim().toLowerCase(), item.id); + return acc; + }, new Map()); + + const categoriesAvailable = categories.reduce((acc, item) => { + acc.set(item.name.trim().toLowerCase(), item.id); + return acc; + }, new Map()); + + let shelterSupliesByCategory: Record = {}; + await Readable.toWeb(csvSourceStream) + .pipeThrough(Transform.toWeb(createCsvParser())) + .pipeThrough(new CsvToShelterTransformStream(headers)) + .pipeThrough( + new TransformStream({ + transform(shelter, controller) { + atomicCounter.increment(); + if (shelter?.supplies?.length) { + for (const supply of shelter.supplies) { + if (suppliesAvailable.has(supply)) { + continue; + } + if (!missingShelterSupplies.has(supply)) { + missingShelterSupplies.add(supply); + } + } + } + shelters.push(shelter); + controller.enqueue(shelter); + }, + }), + ) + .pipeTo( + new WritableStream({ + async close() { + const missingSheltersString = Array.from( + missingShelterSupplies, + ).join(', '); + shelterSupliesByCategory = await detectSupplyCategoryUsingAI( + missingSheltersString, + Array.from(categoriesAvailable.keys()), + ); + }, + }), + ); + + await Readable.toWeb(Readable.from(shelters)) + .pipeThrough( + new ShelterEnhancedStreamTransformer({ + categoriesAvailable, + shelterSupliesByCategory, + suppliesAvailable, + counter: atomicCounter, + }), + ) + .pipeTo( + new WritableStream({ + write: async ( + shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, + ) => { + await this.prismaService.shelter + .create({ data: shelter, select: { name: true, id: true } }) + .then((d) => { + atomicCounter.incrementSuccess(); + this.logger.debug?.(d); + }) + .catch((e: Error) => { + atomicCounter.incrementFailure(); + if (e instanceof PrismaClientKnownRequestError) { + this.logger.error(translatePrismaError(e)); + } else { + this.logger.error(e); + } + }); + }, + close: () => { + this.logger.log( + `${atomicCounter.successCount} de ${atomicCounter.totalCount} processados. ${atomicCounter.failureCount} com erro.`, + ); + }, + }), + ); + + return { + successCount: atomicCounter.successCount, + totalCount: atomicCounter.totalCount, + failureCount: atomicCounter.failureCount, + }; + } +} diff --git a/src/shelter-csv-importer/shelter-csv.transformer.ts b/src/shelter-csv-importer/shelter-csv.transformer.ts new file mode 100644 index 00000000..61af4c56 --- /dev/null +++ b/src/shelter-csv-importer/shelter-csv.transformer.ts @@ -0,0 +1,166 @@ +import { Prisma } from '@prisma/client'; +import { TransformStream } from 'node:stream/web'; +import { CreateShelterSchema } from '../shelter/types/types'; +import { AtomicCounter } from './shelter-csv-importer.helpers'; +import { COLON_REGEX, CSV_DEFAULT_HEADERS } from './types'; +import { + EnhancedTransformArgs, + ShelterColumHeader, + ShelterInput, +} from './types'; +import { Logger } from '@nestjs/common'; + +/** + * JSON -> ShelterInput + * @see ShelterInput + */ +export class CsvToShelterTransformStream extends TransformStream< + unknown, + ShelterInput +> { + private readonly logger = new Logger(CsvToShelterTransformStream.name); + /** + * Espera um Ojeto contento a assinatura da entidade esperada + * @param columnNames dicionário com nomes das colunas a serem mapeadas + */ + constructor(columnNames: Partial = CSV_DEFAULT_HEADERS) { + const efffectiveColumnNames = {} as ShelterColumHeader; + Object.entries(CSV_DEFAULT_HEADERS).forEach(([key, value]) => { + efffectiveColumnNames[key] = + typeof columnNames[key] === 'string' ? columnNames[key] : value; + }); + + super({ + transform: async (chunk, controller) => { + if (!chunk || (chunk && typeof chunk !== 'object')) { + this.logger.warn('Invalid chunk received', chunk); + return; + } + let supplies: string[] = []; + + if ( + typeof chunk[efffectiveColumnNames.shelterSuppliesField] === 'string' + ) { + supplies = (chunk[efffectiveColumnNames.shelterSuppliesField]) + .split(COLON_REGEX) + .filter(Boolean) + .map((s) => s.trim()); + } + + const shelter: ShelterInput = { + verified: false, + // Removendo duplicidades + supplies: [...new Set(supplies)], + }; + + Object.keys(Prisma.ShelterScalarFieldEnum).forEach((key) => { + shelter[key] ??= chunk[efffectiveColumnNames[`${key}Field`]]; + }); + + controller.enqueue(shelter); + } + }); + } +} +/** + * Valida o schema do Input, enriquece o modelo e adicionas as relações encontradas + */ +export class ShelterEnhancedStreamTransformer extends TransformStream< + ShelterInput, + ReturnType +> { + private counter!: AtomicCounter; + private readonly logger = new Logger(CsvToShelterTransformStream.name); + /** + * + */ + constructor({ + shelterSupliesByCategory, + suppliesAvailable, + categoriesAvailable, + counter, + }: EnhancedTransformArgs) { + super({ + transform: async (shelter: ShelterInput, controller) => { + this.counter ??= counter || new AtomicCounter(); + if (!shelter.supplies) { + try { + controller.enqueue(CreateShelterSchema.parse(shelter)); + } catch (error) { + this.counter.incrementFailure(); + this.logger.error(error, shelter); + } + return; + } + + const missingSupplyNames = new Set(); + + const toCreate: Prisma.ShelterSupplyCreateNestedManyWithoutShelterInput['create'] = + []; + + for (const supplyName of missingSupplyNames.values()) { + for (const [categoryName, values] of Object.entries( + shelterSupliesByCategory, + )) { + const indexFound = values.findIndex( + (item) => item.toLowerCase() === supplyName.toLowerCase(), + ); + if (indexFound !== -1) { + toCreate.push({ + supplyId: suppliesAvailable.get(supplyName.toLowerCase())!, + supply: { + connect: { + id: suppliesAvailable.get(supplyName.toLowerCase())!, + name: supplyName, + }, + connectOrCreate: { + where: { + id: suppliesAvailable.get(supplyName.toLowerCase()), + }, + create: { + name: supplyName, + supplyCategory: { + connect: { + name: categoryName, + id: categoriesAvailable.get( + categoryName.toLowerCase(), + ), + }, + }, + }, + }, + }, + }); + } + } + } + shelter.shelterSupplies = { + create: toCreate, + }; + + if (shelter.latitude) { + shelter.latitude = Number.parseFloat(`${shelter.latitude}`); + } + if (shelter.longitude) { + shelter.longitude = Number.parseFloat(`${shelter.longitude}`); + } + + Object.keys(shelter).forEach((key) => { + if ( + typeof shelter[key] === 'string' && + shelter[key].trim().length === 0 + ) { + shelter[key] = null; + } + }); + + await CreateShelterSchema.parseAsync(shelter) + .then((s) => controller.enqueue(s)) + .catch((e) => { + this.counter.incrementFailure(); + this.logger.error(e.message, shelter); + }); + }, + }); + } +} diff --git a/src/shelter-csv-importer/types.ts b/src/shelter-csv-importer/types.ts index e69de29b..c0b76477 100644 --- a/src/shelter-csv-importer/types.ts +++ b/src/shelter-csv-importer/types.ts @@ -0,0 +1,86 @@ +import { AtomicCounter } from './shelter-csv-importer.helpers'; + +import { Prisma } from '@prisma/client'; +import { ReadStream } from 'fs'; +import { Readable } from 'node:stream'; + +type ShelterKey = Exclude< + Prisma.ShelterScalarFieldEnum, + 'createdAt' | 'updatedAt' | 'prioritySum' | 'verified' | 'id' +>; + +export type ShelterColumHeader = Record<`${ShelterKey}Field`, string> & { + shelterSuppliesField: string; +}; + +interface ParseCsvArgsBaseArgs> { + /** + * link do arquivo CSV + */ + csvUrl?: string; + /** + * stream proveniente de algum arquivo CSV + */ + fileStream?: Readable; + /** + * mapeamento de quais cabeçalhos do csv serão usados como colunas da tabela. + */ + headers?: Partial; +} + +export type ParseCsvArgs = + | (ParseCsvArgsBaseArgs & { csvUrl: string }) + | (ParseCsvArgsBaseArgs & { fileStream: Readable }); +export interface EnhancedTransformArgs { + /** + * KeyValue contento as categorias e os supplies contidos naquela categorias (detectados por IA) + */ + shelterSupliesByCategory: Record; + /** + * KeyValue com Suprimentos já encontrados na base atual que podem ser utilizadas em relações + * @example + * { "Óleo de cozinha": "eb1d5056-8b9b-455d-a179-172a747e3f20", + * "Álcool em gel": "a3e3bdf8-0be4-4bdc-a3b0-b40ba931be5f" + * } + */ + suppliesAvailable: Map; + /** + * KeyValue com Categorias já encontradas na base atual que podem ser utilizadas em relações + * @example + * { "Higiene Pessoal": "718d5be3-69c3-4216-97f1-12b690d0eb97", + * "Alimentos e Água": "a3e3bdf8-0be4-4bdc-a3b0-b40ba931be5f" + * } + */ + categoriesAvailable: Map; + counter?: AtomicCounter; +} +export type ShelterInput = Partial< + Prisma.ShelterCreateInput & { supplies: string[] } +>; + +/** + * Exemplo de Padrão utilizado: + * `nome_do_local, endereco, whatsapp, lat, lng, itens_disponiveis, itens_em_falta` + */ +export const CSV_DEFAULT_HEADERS: ShelterColumHeader = { + nameField: 'nome_do_local', + addressField: 'endereco', + contactField: 'whatsapp', + latitudeField: 'lat', + longitudeField: 'lng', + shelterSuppliesField: 'itens_em_falta', + capacityField: 'capacidade', + cityField: 'cidade', + neighbourhoodField: 'bairro', + petFriendlyField: 'pet_friendly', + pixField: 'pix', + shelteredPeopleField: 'pessoas_abrigadas', + streetField: 'rua', + streetNumberField: 'numero', + zipCodeField: 'cep', +}; + +/** + * Regex que ignora vírgulas dentro de parenteses no split + */ +export const COLON_REGEX = /(? Date: Thu, 16 May 2024 10:17:13 -0300 Subject: [PATCH 05/16] test: fixing example imports in tests --- jest.config.ts | 1 + .../shelter-csv-importer.service.spec.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jest.config.ts b/jest.config.ts index 6bc3f4c4..86a65ee4 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -13,6 +13,7 @@ const config: Config = { '^src/(.*)$': '/$1', '^@/(.*)$': '/$1', '^test/(.*)$': '/../$1', + '^examples/(.*)$': '/../test/examples/$1', }, testEnvironment: 'node', }; diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts b/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts index d88d51e7..f8fdd28b 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts @@ -44,7 +44,6 @@ describe('ShelterCsvImporterService', () => { }); test('test_shelterToCsv_withoutRequiredInputs', async () => { - await expect(service.shelterToCsv({ headers: {} } as any)).rejects.toThrow( 'Um dos campos `csvUrl` ou `fileStream` é obrigatório', ); @@ -60,6 +59,12 @@ describe('ShelterCsvImporterService', () => { body: mockFileStream, } as any); + jest + .spyOn(helpers, 'detectSupplyCategoryUsingAI') + .mockResolvedValueOnce( + require('examples/gemini_prompt_response_example.json'), + ); + const result = await service.shelterToCsv({ fileStream: mockFileStream, headers: { @@ -83,7 +88,7 @@ describe('ShelterCsvImporterService', () => { mockFileStream.push('name,address,lat,lng,supplies, UnknownSupply\n'); mockFileStream.push('name,address,1.0214,1.54525, UnknownSupply'); mockFileStream.push(null); - + jest.spyOn(global, 'fetch').mockResolvedValueOnce({ body: mockFileStream, } as any); From 1af381271e73a48920e1ba5428a9944ce820ff24 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Thu, 16 May 2024 14:42:31 -0300 Subject: [PATCH 06/16] chore: declaring shelter-csv-service into module --- src/app.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app.module.ts b/src/app.module.ts index c08ea99f..d4e65698 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -12,6 +12,7 @@ import { SupplyCategoriesModule } from './supply-categories/supply-categories.mo import { ShelterManagersModule } from './shelter-managers/shelter-managers.module'; import { ShelterSupplyModule } from './shelter-supply/shelter-supply.module'; import { PartnersModule } from './partners/partners.module'; +import { ShelterCsvImporterModule } from './shelter-csv-importer/shelter-csv-importer.module'; @Module({ imports: [ @@ -19,6 +20,7 @@ import { PartnersModule } from './partners/partners.module'; UsersModule, SessionsModule, ShelterModule, + ShelterCsvImporterModule, SupplyModule, SupplyCategoriesModule, ShelterManagersModule, From 3cd457ded3a91337d91ec7415e3b477db6592fd0 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Thu, 16 May 2024 14:43:18 -0300 Subject: [PATCH 07/16] refactor: code sppliting and adding some extra functionalities to shelter-csv-importer --- .../shelter-csv-importer.helpers.ts | 12 +- .../shelter-csv-importer.service.spec.ts | 8 +- .../shelter-csv-importer.service.ts | 247 ++++++++++++++---- .../shelter-csv.transformer.ts | 22 +- src/shelter-csv-importer/types.ts | 4 + 5 files changed, 216 insertions(+), 77 deletions(-) diff --git a/src/shelter-csv-importer/shelter-csv-importer.helpers.ts b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts index 1aebd417..b98d50b2 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.helpers.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts @@ -164,15 +164,15 @@ export class AtomicCounter { return this._failureCount; } - incrementSuccess() { - this._successCount += 1; + incrementSuccess(amount?: number) { + this._successCount += amount ?? 1; } - increment() { - this._totalCount += 1; + increment(amount?: number) { + this._totalCount += amount ?? 1; } - incrementFailure() { - this._failureCount += 1; + incrementFailure(amount?: number) { + this._failureCount += amount ?? 1; } } diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts b/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts index f8fdd28b..e2a3d870 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.spec.ts @@ -1,9 +1,9 @@ import { generateMock } from '@anatine/zod-mock'; import { Test, TestingModule } from '@nestjs/testing'; +import { Readable } from 'node:stream'; import { ShelterSchema } from 'src/shelter/types/types'; import { SupplyCategorySchema } from 'src/supply-categories/types'; import { SupplySchema } from 'src/supply/types'; -import { Readable } from 'node:stream'; import { PrismaService } from '../prisma/prisma.service'; import * as helpers from './shelter-csv-importer.helpers'; import { ShelterCsvImporterService } from './shelter-csv-importer.service'; @@ -44,7 +44,7 @@ describe('ShelterCsvImporterService', () => { }); test('test_shelterToCsv_withoutRequiredInputs', async () => { - await expect(service.shelterToCsv({ headers: {} } as any)).rejects.toThrow( + await expect(service.execute({ headers: {} } as any)).rejects.toThrow( 'Um dos campos `csvUrl` ou `fileStream` é obrigatório', ); }); @@ -65,7 +65,7 @@ describe('ShelterCsvImporterService', () => { require('examples/gemini_prompt_response_example.json'), ); - const result = await service.shelterToCsv({ + const result = await service.execute({ fileStream: mockFileStream, headers: { nameField: 'name', @@ -97,7 +97,7 @@ describe('ShelterCsvImporterService', () => { UnknownSupply: ['NewCategory'], }); - const result = await service.shelterToCsv({ + const result = await service.execute({ fileStream: mockFileStream, headers: { nameField: 'name', diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.ts b/src/shelter-csv-importer/shelter-csv-importer.service.ts index 9ae4f0d3..a6c37492 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.ts @@ -3,6 +3,7 @@ import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; import { Readable, Transform } from 'node:stream'; import { TransformStream } from 'node:stream/web'; +import { Prisma } from '@prisma/client'; import { PrismaService } from '../prisma/prisma.service'; import { CreateShelterSchema } from '../shelter/types/types'; import { @@ -16,7 +17,12 @@ import { CsvToShelterTransformStream, ShelterEnhancedStreamTransformer, } from './shelter-csv.transformer'; -import { ParseCsvArgs, ShelterColumHeader, ShelterInput } from './types'; +import { + CSV_DEFAULT_HEADERS, + ParseCsvArgs, + ShelterColumHeader, + ShelterInput, +} from './types'; @Injectable() export class ShelterCsvImporterService { @@ -34,32 +40,171 @@ export class ShelterCsvImporterService { * ``` * */ - async shelterToCsv({ - headers, + async execute({ + headers = CSV_DEFAULT_HEADERS, csvUrl, fileStream, - }: ParseCsvArgs) { + dryRun = false, + useIAToPredictSupplyCategories = true, + useBatchTransaction = false, + onEntity, + }: ParseCsvArgs & { + /** + * Se deverá usar alguma LLM para tentar categorizar as categorias dos suprimentos + * @implNote por enquanto apenas `Gemini` foi implementada. + */ + useIAToPredictSupplyCategories?: boolean; + /** + * + * callback executado após cada entidade ser criada ou ser validada (caso `dryRun` seja true) + * + * ** NÃO será executada caso `useBatchTransaction` seja true + */ + onEntity?: ( + shelter: ReturnType< + (typeof CreateShelterSchema)['parse'] & { id?: string } + >, + ) => void; + /** + * Se true, guardará todas as criações em memória e executará elas em um batch `$transaction` + * [Prisma $transaction docs](https://www.prisma.io/docs/concepts/components/prisma-client/transactions). + */ + useBatchTransaction?: boolean; + }) { + this.validateInput(csvUrl, fileStream); + + const efffectiveColumnNames = this.getEffectiveColumnNames(headers); + + const atomicCounter = new AtomicCounter(); + + const output: Record[] = []; + + let csvSourceStream = csvUrl + ? responseToReadable(await fetch(csvUrl)) + : fileStream!; + + const { + entitiesToCreate, + categoriesAvailable, + shelterSupliesByCategory, + suppliesAvailable, + } = await this.preProcessPipeline( + csvSourceStream, + efffectiveColumnNames, + atomicCounter, + useIAToPredictSupplyCategories, + ); + + // const transactionArgs: ReturnType[] = [] + const transactionArgs: Prisma.ShelterCreateArgs[] = []; + + await Readable.toWeb(Readable.from(entitiesToCreate)) + .pipeThrough( + new ShelterEnhancedStreamTransformer({ + categoriesAvailable, + shelterSupliesByCategory, + suppliesAvailable, + counter: atomicCounter, + }), + ) + .pipeThrough( + new TransformStream({ + // transform(chunk,controller){ + // } + }), + ) + .pipeTo( + new WritableStream({ + write: async ( + shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, + ) => { + if (dryRun) { + onEntity?.(shelter); + atomicCounter.incrementSuccess(); + return; + } + + if (useBatchTransaction) { + transactionArgs.push(this.getShelterCreateArgs(shelter)); + return; + } + + await this.prismaService.shelter + .create(this.getShelterCreateArgs(shelter)) + .then((d) => { + atomicCounter.incrementSuccess(); + onEntity?.(d); + output.push(d); + }) + .catch((e: Error) => { + atomicCounter.incrementFailure(); + if (e instanceof PrismaClientKnownRequestError) { + this.logger.error(translatePrismaError(e)); + } else { + this.logger.error(e); + } + }); + }, + close: async () => { + if (useBatchTransaction && !dryRun) { + try { + const transactionResult = await this.prismaService.$transaction( + transactionArgs.map(this.prismaService.shelter.create), + ); + output.push(...transactionResult); + atomicCounter.incrementSuccess(transactionResult.length); + atomicCounter.incrementFailure( + transactionResult.length - transactionArgs.length, + ); + } catch (err) { + this.logger.error('Erro ao executar transaction', err); + } + } + this.logger.log( + `${atomicCounter.successCount} de ${atomicCounter.totalCount} entidades processadas com sucesso. ${atomicCounter.failureCount} com erro.`, + ); + }, + }), + ); + + return { + successCount: atomicCounter.successCount, + totalCount: atomicCounter.totalCount, + failureCount: atomicCounter.failureCount, + data: output, + }; + } + + private getEffectiveColumnNames(headers: Partial) { + const efffectiveColumnNames = {} as ShelterColumHeader; + Object.entries(CSV_DEFAULT_HEADERS).forEach(([key, value]) => { + efffectiveColumnNames[key] = + typeof headers[key] === 'string' ? headers[key] : value; + }); + return efffectiveColumnNames; + } + + private validateInput(csvUrl?: string, fileStream?: Readable) { const validInput = (csvUrl && URL.canParse(csvUrl)) || fileStream != null; if (!validInput) { this.logger.warn('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); throw new Error('Um dos campos `csvUrl` ou `fileStream` é obrigatório'); } - const atomicCounter = new AtomicCounter(); - const shelters: ShelterInput[] = []; - - let csvSourceStream: Readable; - if (csvUrl) { - csvSourceStream = responseToReadable(await fetch(csvUrl)); - } else { - csvSourceStream = fileStream!; - } + } + private async preProcessPipeline( + csvSourceStream: Readable, + headers: ShelterColumHeader, + atomicCounter: AtomicCounter, + useIAToPredictSupplyCategories: boolean, + ) { const [categories, supplies] = await this.prismaService.$transaction([ this.prismaService.supplyCategory.findMany({}), this.prismaService.supply.findMany({ distinct: ['name'] }), ]); + const entitiesToCreate: ShelterInput[] = []; const missingShelterSupplies = new Set(); const suppliesAvailable = supplies.reduce((acc, item) => { @@ -90,7 +235,7 @@ export class ShelterCsvImporterService { } } } - shelters.push(shelter); + entitiesToCreate.push(shelter); controller.enqueue(shelter); }, }), @@ -98,6 +243,9 @@ export class ShelterCsvImporterService { .pipeTo( new WritableStream({ async close() { + if (!useIAToPredictSupplyCategories) { + return; + } const missingSheltersString = Array.from( missingShelterSupplies, ).join(', '); @@ -108,48 +256,39 @@ export class ShelterCsvImporterService { }, }), ); + return { + entitiesToCreate, + categoriesAvailable, + shelterSupliesByCategory, + suppliesAvailable, + }; + } - await Readable.toWeb(Readable.from(shelters)) - .pipeThrough( - new ShelterEnhancedStreamTransformer({ - categoriesAvailable, - shelterSupliesByCategory, - suppliesAvailable, - counter: atomicCounter, - }), - ) - .pipeTo( - new WritableStream({ - write: async ( - shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, - ) => { - await this.prismaService.shelter - .create({ data: shelter, select: { name: true, id: true } }) - .then((d) => { - atomicCounter.incrementSuccess(); - this.logger.debug?.(d); - }) - .catch((e: Error) => { - atomicCounter.incrementFailure(); - if (e instanceof PrismaClientKnownRequestError) { - this.logger.error(translatePrismaError(e)); - } else { - this.logger.error(e); - } - }); - }, - close: () => { - this.logger.log( - `${atomicCounter.successCount} de ${atomicCounter.totalCount} processados. ${atomicCounter.failureCount} com erro.`, - ); - }, - }), - ); - + private getShelterCreateArgs( + shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, + ): Prisma.ShelterCreateArgs { return { - successCount: atomicCounter.successCount, - totalCount: atomicCounter.totalCount, - failureCount: atomicCounter.failureCount, + data: Object.assign(shelter, { + createdAt: new Date().toISOString(), + }), + include: { + shelterSupplies: { + select: { + supply: { + select: { + id: true, + name: true, + supplyCategory: { + select: { + name: true, + id: true, + }, + }, + }, + }, + }, + }, + }, }; } } diff --git a/src/shelter-csv-importer/shelter-csv.transformer.ts b/src/shelter-csv-importer/shelter-csv.transformer.ts index 61af4c56..5a0c404f 100644 --- a/src/shelter-csv-importer/shelter-csv.transformer.ts +++ b/src/shelter-csv-importer/shelter-csv.transformer.ts @@ -1,14 +1,13 @@ +import { Logger } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { TransformStream } from 'node:stream/web'; import { CreateShelterSchema } from '../shelter/types/types'; import { AtomicCounter } from './shelter-csv-importer.helpers'; -import { COLON_REGEX, CSV_DEFAULT_HEADERS } from './types'; import { - EnhancedTransformArgs, + COLON_REGEX, EnhancedTransformArgs, ShelterColumHeader, - ShelterInput, + ShelterInput } from './types'; -import { Logger } from '@nestjs/common'; /** * JSON -> ShelterInput @@ -23,12 +22,7 @@ export class CsvToShelterTransformStream extends TransformStream< * Espera um Ojeto contento a assinatura da entidade esperada * @param columnNames dicionário com nomes das colunas a serem mapeadas */ - constructor(columnNames: Partial = CSV_DEFAULT_HEADERS) { - const efffectiveColumnNames = {} as ShelterColumHeader; - Object.entries(CSV_DEFAULT_HEADERS).forEach(([key, value]) => { - efffectiveColumnNames[key] = - typeof columnNames[key] === 'string' ? columnNames[key] : value; - }); + constructor(columnNames: ShelterColumHeader) { super({ transform: async (chunk, controller) => { @@ -39,9 +33,9 @@ export class CsvToShelterTransformStream extends TransformStream< let supplies: string[] = []; if ( - typeof chunk[efffectiveColumnNames.shelterSuppliesField] === 'string' + typeof chunk[columnNames.shelterSuppliesField] === 'string' ) { - supplies = (chunk[efffectiveColumnNames.shelterSuppliesField]) + supplies = (chunk[columnNames.shelterSuppliesField]) .split(COLON_REGEX) .filter(Boolean) .map((s) => s.trim()); @@ -54,7 +48,7 @@ export class CsvToShelterTransformStream extends TransformStream< }; Object.keys(Prisma.ShelterScalarFieldEnum).forEach((key) => { - shelter[key] ??= chunk[efffectiveColumnNames[`${key}Field`]]; + shelter[key] ??= chunk[columnNames[`${key}Field`]]; }); controller.enqueue(shelter); @@ -108,6 +102,7 @@ export class ShelterEnhancedStreamTransformer extends TransformStream< if (indexFound !== -1) { toCreate.push({ supplyId: suppliesAvailable.get(supplyName.toLowerCase())!, + createdAt: new Date().toISOString(), supply: { connect: { id: suppliesAvailable.get(supplyName.toLowerCase())!, @@ -119,6 +114,7 @@ export class ShelterEnhancedStreamTransformer extends TransformStream< }, create: { name: supplyName, + createdAt: new Date().toISOString(), supplyCategory: { connect: { name: categoryName, diff --git a/src/shelter-csv-importer/types.ts b/src/shelter-csv-importer/types.ts index c0b76477..1855cd1f 100644 --- a/src/shelter-csv-importer/types.ts +++ b/src/shelter-csv-importer/types.ts @@ -26,6 +26,10 @@ interface ParseCsvArgsBaseArgs> { * mapeamento de quais cabeçalhos do csv serão usados como colunas da tabela. */ headers?: Partial; + /** + * se true, não salvará nada no banco + */ + dryRun?: boolean } export type ParseCsvArgs = From 15cd16e75a11892a79cc2fd9fa67baa6c0a21009 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Thu, 16 May 2024 14:52:58 -0300 Subject: [PATCH 08/16] chore: adding some types --- .../shelter-csv-importer.service.ts | 39 +++---------------- src/shelter-csv-importer/types.ts | 28 ++++++++++++- 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.ts b/src/shelter-csv-importer/shelter-csv-importer.service.ts index a6c37492..d962681c 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.ts @@ -5,7 +5,6 @@ import { TransformStream } from 'node:stream/web'; import { Prisma } from '@prisma/client'; import { PrismaService } from '../prisma/prisma.service'; -import { CreateShelterSchema } from '../shelter/types/types'; import { AtomicCounter, createCsvParser, @@ -17,12 +16,8 @@ import { CsvToShelterTransformStream, ShelterEnhancedStreamTransformer, } from './shelter-csv.transformer'; -import { - CSV_DEFAULT_HEADERS, - ParseCsvArgs, - ShelterColumHeader, - ShelterInput, -} from './types'; +import { CSV_DEFAULT_HEADERS, ShelterColumHeader, ShelterInput } from './types'; +import { ShelterCsvImporterExecutionArgs, ShelterValidInput } from './types'; @Injectable() export class ShelterCsvImporterService { @@ -48,29 +43,7 @@ export class ShelterCsvImporterService { useIAToPredictSupplyCategories = true, useBatchTransaction = false, onEntity, - }: ParseCsvArgs & { - /** - * Se deverá usar alguma LLM para tentar categorizar as categorias dos suprimentos - * @implNote por enquanto apenas `Gemini` foi implementada. - */ - useIAToPredictSupplyCategories?: boolean; - /** - * - * callback executado após cada entidade ser criada ou ser validada (caso `dryRun` seja true) - * - * ** NÃO será executada caso `useBatchTransaction` seja true - */ - onEntity?: ( - shelter: ReturnType< - (typeof CreateShelterSchema)['parse'] & { id?: string } - >, - ) => void; - /** - * Se true, guardará todas as criações em memória e executará elas em um batch `$transaction` - * [Prisma $transaction docs](https://www.prisma.io/docs/concepts/components/prisma-client/transactions). - */ - useBatchTransaction?: boolean; - }) { + }: ShelterCsvImporterExecutionArgs) { this.validateInput(csvUrl, fileStream); const efffectiveColumnNames = this.getEffectiveColumnNames(headers); @@ -115,9 +88,7 @@ export class ShelterCsvImporterService { ) .pipeTo( new WritableStream({ - write: async ( - shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, - ) => { + write: async (shelter: ShelterValidInput) => { if (dryRun) { onEntity?.(shelter); atomicCounter.incrementSuccess(); @@ -265,7 +236,7 @@ export class ShelterCsvImporterService { } private getShelterCreateArgs( - shelter: ReturnType<(typeof CreateShelterSchema)['parse']>, + shelter: ShelterValidInput, ): Prisma.ShelterCreateArgs { return { data: Object.assign(shelter, { diff --git a/src/shelter-csv-importer/types.ts b/src/shelter-csv-importer/types.ts index 1855cd1f..bf10787e 100644 --- a/src/shelter-csv-importer/types.ts +++ b/src/shelter-csv-importer/types.ts @@ -1,8 +1,8 @@ import { AtomicCounter } from './shelter-csv-importer.helpers'; import { Prisma } from '@prisma/client'; -import { ReadStream } from 'fs'; import { Readable } from 'node:stream'; +import { CreateShelterSchema } from '../shelter/types/types'; type ShelterKey = Exclude< Prisma.ShelterScalarFieldEnum, @@ -29,7 +29,7 @@ interface ParseCsvArgsBaseArgs> { /** * se true, não salvará nada no banco */ - dryRun?: boolean + dryRun?: boolean; } export type ParseCsvArgs = @@ -88,3 +88,27 @@ export const CSV_DEFAULT_HEADERS: ShelterColumHeader = { * Regex que ignora vírgulas dentro de parenteses no split */ export const COLON_REGEX = /(? { + id?: string; +} +export type ShelterCsvImporterExecutionArgs = + ParseCsvArgs & { + /** + * Se deverá usar alguma LLM para tentar categorizar as categorias dos suprimentos + * @implNote por enquanto apenas `Gemini` foi implementada. + */ + useIAToPredictSupplyCategories?: boolean; + /** + * + * callback executado após cada entidade ser criada ou ser validada (caso `dryRun` seja true) + * + * ** NÃO será executada caso `useBatchTransaction` seja true + */ + onEntity?: (shelter: ShelterValidInput) => void; + /** + * Se true, guardará todas as criações em memória e executará elas em um batch `$transaction` + * [Prisma $transaction docs](https://www.prisma.io/docs/concepts/components/prisma-client/transactions). + */ + useBatchTransaction?: boolean; + }; From c0a08ca1b99e40f1398ceb198f304f32aa5486b8 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Thu, 16 May 2024 22:00:48 -0300 Subject: [PATCH 09/16] feat: implementando controller para importar csv --- package-lock.json | 115 ++++++- package.json | 3 + routes.txt | 300 ++++++++++++++++++ src/interceptors/file-upload.interceptor.ts | 54 ++++ src/main.ts | 7 +- src/shelter-csv-importer/dto/file.dto.ts | 9 + .../shelter-csv-importer.helpers.ts | 36 ++- .../shelter-csv-importer.module.ts | 1 + src/shelter/shelter.controller.ts | 47 ++- src/shelter/shelter.module.ts | 3 +- 10 files changed, 558 insertions(+), 17 deletions(-) create mode 100644 routes.txt create mode 100644 src/interceptors/file-upload.interceptor.ts create mode 100644 src/shelter-csv-importer/dto/file.dto.ts diff --git a/package-lock.json b/package-lock.json index 639bce99..d1fb3450 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,8 @@ "bcrypt": "^5.1.1", "csv": "^6.3.9", "date-fns": "^3.6.0", + "fastify-multer": "^2.0.3", + "multer": "^1.4.5-lts.1", "passport-jwt": "^4.0.1", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1", @@ -34,6 +36,7 @@ "@nestjs/testing": "^10.0.0", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/multer": "^1.4.11", "@types/node": "^20.3.1", "@types/query-string": "^6.3.0", "@types/supertest": "^6.0.0", @@ -999,6 +1002,17 @@ "fast-uri": "^2.0.0" } }, + "node_modules/@fastify/busboy": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-1.2.1.tgz", + "integrity": "sha512-7PQA7EH43S0CxcOa9OeAnaeA0oQ+e/DHNPZwSQM9CQHW76jle5+OvLdibRp/Aafs9KXbLhxyjOTkRjWUbQEd3Q==", + "dependencies": { + "text-decoding": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/@fastify/cors": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-9.0.1.tgz", @@ -2110,6 +2124,23 @@ "@nestjs/core": "^10.0.0" } }, + "node_modules/@nestjs/platform-express/node_modules/multer": { + "version": "1.4.4-lts.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", + "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/@nestjs/platform-fastify": { "version": "10.3.8", "resolved": "https://registry.npmjs.org/@nestjs/platform-fastify/-/platform-fastify-10.3.8.tgz", @@ -2597,6 +2628,15 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true }, + "node_modules/@types/multer": { + "version": "1.4.11", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.11.tgz", + "integrity": "sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w==", + "dev": true, + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/node": { "version": "20.12.8", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.8.tgz", @@ -5097,6 +5137,70 @@ "toad-cache": "^3.3.0" } }, + "node_modules/fastify-multer": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/fastify-multer/-/fastify-multer-2.0.3.tgz", + "integrity": "sha512-QnFqrRgxmUwWHTgX9uyQSu0C/hmVCfcxopqjApZ4uaZD5W9MJ+nHUlW4+9q7Yd3BRxDIuHvgiM5mjrh6XG8cAA==", + "dependencies": { + "@fastify/busboy": "^1.0.0", + "append-field": "^1.0.0", + "concat-stream": "^2.0.0", + "fastify-plugin": "^2.0.1", + "mkdirp": "^1.0.4", + "on-finished": "^2.3.0", + "type-is": "~1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/fastify-multer/node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/fastify-multer/node_modules/fastify-plugin": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-2.3.4.tgz", + "integrity": "sha512-I+Oaj6p9oiRozbam30sh39BiuiqBda7yK2nmSPVwDCfIBlKnT8YB3MY+pRQc2Fcd07bf6KPGklHJaQ2Qu81TYQ==", + "dependencies": { + "semver": "^7.3.2" + } + }, + "node_modules/fastify-multer/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fastify-multer/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fastify-plugin": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.1.tgz", @@ -7342,9 +7446,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/multer": { - "version": "1.4.4-lts.1", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", - "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==", + "version": "1.4.5-lts.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz", + "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==", "dependencies": { "append-field": "^1.0.0", "busboy": "^1.0.0", @@ -9368,6 +9472,11 @@ "node": "*" } }, + "node_modules/text-decoding": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-decoding/-/text-decoding-1.0.0.tgz", + "integrity": "sha512-/0TJD42KDnVwKmDK6jj3xP7E2MG7SHAOG4tyTgyUCRPdHwvkquYNLEQltmdMa3owq3TkddCVcTsoctJI8VQNKA==" + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", diff --git a/package.json b/package.json index eff4479b..4553fd63 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,8 @@ "bcrypt": "^5.1.1", "csv": "^6.3.9", "date-fns": "^3.6.0", + "fastify-multer": "^2.0.3", + "multer": "^1.4.5-lts.1", "passport-jwt": "^4.0.1", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1", @@ -49,6 +51,7 @@ "@nestjs/testing": "^10.0.0", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/multer": "^1.4.11", "@types/node": "^20.3.1", "@types/query-string": "^6.3.0", "@types/supertest": "^6.0.0", diff --git a/routes.txt b/routes.txt new file mode 100644 index 00000000..b7a84e4c --- /dev/null +++ b/routes.txt @@ -0,0 +1,300 @@ +{ + "Comment": "aaa", + "Changes": [ + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "guedder.com.", + "Type": "A", + "TTL": 60, + "ResourceRecords": [ + { + "Value": "35.86.23.115" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "guedder.com.", + "Type": "MX", + "TTL": 3600, + "ResourceRecords": [ + { + "Value": "1\tASPMX.L.GOOGLE.COM" + }, + { + "Value": "5\tALT1.ASPMX.L.GOOGLE.COM" + }, + { + "Value": "5\tALT2.ASPMX.L.GOOGLE.COM" + }, + { + "Value": "10\tALT3.ASPMX.L.GOOGLE.COM" + }, + { + "Value": "10\tALT4.ASPMX.L.GOOGLE.COM" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "guedder.com.", + "Type": "TXT", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "\"v=spf1 include:_spf.google.com ~all\"" + }, + { + "Value": "\"google-site-verification=6IGUHktwmqB1RHvTIZJlKsFkmqF1iqBwaiWvtB3PdbQ\"" + }, + { + "Value": "\"facebook-domain-verification=7b5s07221rinif6iso3tvajh34nxtz\"" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "_cc98d301c046cc32d416b9eb5632c6f1.guedder.com.", + "Type": "CNAME", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "_5f0fadf5c4885a67410f7f601135d9bc.dhzvlrndnj.acm-validations.aws." + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "google._domainkey.guedder.com.", + "Type": "TXT", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "\"v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAj35YXTEgL69Tj85RBFy6oNIu2q1gyXzWrfiW9DSWdBlxe6eqhq56bQVv7mRyoVVxW9IWPX4+Ut8InIEJFWam95SXlPx2O+l/\" \"CJ7TjcRtNe+ykIl+d0zpgkqZyXEPbiZYIsaU+O0nb/I0gtiGE4bYqb96S6OHN+6G4gTIpm5CPkC68DS2OsteOmzPUO7QKKWau4lG/\" \"oIROlcDPdNWucanF513VEdnRJsKHXk097JHq+YUhLyCjwDBpPJBq2H/0L6Usf9GZ9/jw9EI55NtnGOiuMPxBXncYeRVXXoyhlLkY5r7Bz7G8/\" \"XagO24b5Z5CGIJF3v5vDmOExGLwNZziYKC7QIDAQAB\"" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "lwdgihpjz4zecxghfjlwi5kk3wwepgb7._domainkey.guedder.com.", + "Type": "CNAME", + "TTL": 1800, + "ResourceRecords": [ + { + "Value": "lwdgihpjz4zecxghfjlwi5kk3wwepgb7.dkim.amazonses.com" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "ntit4w7do46e5gt4kndaznfvft2dhz4c._domainkey.guedder.com.", + "Type": "CNAME", + "TTL": 1800, + "ResourceRecords": [ + { + "Value": "ntit4w7do46e5gt4kndaznfvft2dhz4c.dkim.amazonses.com" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "woinkreggvmnukmv6m73vf762ghtdf4x._domainkey.guedder.com.", + "Type": "CNAME", + "TTL": 1800, + "ResourceRecords": [ + { + "Value": "woinkreggvmnukmv6m73vf762ghtdf4x.dkim.amazonses.com" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "api-loadbalance.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z1H1FL5HABSF5", + "DNSName": "guedder-load-balancer-2-737976362.us-west-2.elb.amazonaws.com.", + "EvaluateTargetHealth": true + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "api.guedder.com.", + "Type": "CNAME", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "api-loadbalance.guedder.com" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "app.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z2FDTNDATAQYW2", + "DNSName": "d1p99rujyyi7pk.cloudfront.net.", + "EvaluateTargetHealth": false + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "beta.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z2FDTNDATAQYW2", + "DNSName": "d6gohgvl1bk0m.cloudfront.net.", + "EvaluateTargetHealth": false + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "crux.guedder.com.", + "Type": "A", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "35.85.65.26" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "d4sign.guedder.com.", + "Type": "TXT", + "TTL": 3600, + "ResourceRecords": [ + { + "Value": "\"d4sign-domain-verification=e1455ea6-ad7e-436f-80f0-36bcb1634172\"" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "dev-api.guedder.com.", + "Type": "A", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "34.221.192.181" + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "_6eddddbbfb6bf75cbbe96037e6451c9e.dev-api.guedder.com.", + "Type": "CNAME", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "_df81fbb08064f9efa2917c1d25de75f6.dqxlbvzbzt.acm-validations.aws." + } + ] + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "institucional.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z2FDTNDATAQYW2", + "DNSName": "d2jp1q4t2zduz8.cloudfront.net.", + "EvaluateTargetHealth": false + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "portal.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z2FDTNDATAQYW2", + "DNSName": "d142ydjmx8obeg.cloudfront.net.", + "EvaluateTargetHealth": false + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "staging-portal.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z2FDTNDATAQYW2", + "DNSName": "d1tydsuuoy2nar.cloudfront.net.", + "EvaluateTargetHealth": false + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "staging.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z2FDTNDATAQYW2", + "DNSName": "d31p8c3qwfzi7x.cloudfront.net.", + "EvaluateTargetHealth": false + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "webapp.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z06176872A9AQ4DNUVAT0", + "DNSName": "guedder.com.", + "EvaluateTargetHealth": true + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "www.guedder.com.", + "Type": "A", + "AliasTarget": { + "HostedZoneId": "Z06176872A9AQ4DNUVAT0", + "DNSName": "guedder.com.", + "EvaluateTargetHealth": true + } + } + } + ] +} \ No newline at end of file diff --git a/src/interceptors/file-upload.interceptor.ts b/src/interceptors/file-upload.interceptor.ts new file mode 100644 index 00000000..950758f4 --- /dev/null +++ b/src/interceptors/file-upload.interceptor.ts @@ -0,0 +1,54 @@ +import { + CallHandler, + ExecutionContext, + Inject, + mixin, + NestInterceptor, + Optional, + Type, +} from '@nestjs/common'; +import { Observable } from 'rxjs'; +import FastifyMulter from 'fastify-multer'; +import { Options, Multer } from 'multer'; + + +export function FastifyFileInterceptor( + fieldName: string, + localOptions: Options, +): Type { + class MixinInterceptor implements NestInterceptor { + protected multer: Multer; + + constructor( + @Optional() + @Inject('MULTER_MODULE_OPTIONS') + options: Multer, + ) { + this.multer = (FastifyMulter as any)({ ...options, ...localOptions }); + } + + async intercept( + context: ExecutionContext, + next: CallHandler, + ): Promise> { + const ctx = context.switchToHttp(); + + await new Promise((resolve, reject) => + this.multer.single(fieldName)( + ctx.getRequest(), + ctx.getResponse(), + (error: any) => { + if (error) { + return reject(error); + } + resolve(); + }, + ), + ); + + return next.handle(); + } + } + const Interceptor = mixin(MixinInterceptor); + return Interceptor as Type; +} diff --git a/src/main.ts b/src/main.ts index 649e75d4..825120ff 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,17 +3,20 @@ import { FastifyAdapter, NestFastifyApplication, } from '@nestjs/platform-fastify'; -import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger'; - +import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; +import multer from 'fastify-multer'; import { AppModule } from './app.module'; async function bootstrap() { const fastifyAdapter = new FastifyAdapter(); + const app = await NestFactory.create( AppModule, fastifyAdapter, ); + app.register(multer().contentParser); + const config = new DocumentBuilder() .setTitle('SOS - Rio Grande do Sul') .setDescription('...') diff --git a/src/shelter-csv-importer/dto/file.dto.ts b/src/shelter-csv-importer/dto/file.dto.ts new file mode 100644 index 00000000..9ee7fa47 --- /dev/null +++ b/src/shelter-csv-importer/dto/file.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; + +/** + * Classe utilizada apenas para fins de documentação do swagger + */ +export class FileDtoStub { + @ApiProperty({ type: 'string', format: 'binary' }) + file?: string; +} diff --git a/src/shelter-csv-importer/shelter-csv-importer.helpers.ts b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts index b98d50b2..1cfe52e7 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.helpers.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts @@ -3,22 +3,40 @@ import { HarmBlockThreshold, HarmCategory, } from '@google/generative-ai'; -import { Logger } from '@nestjs/common'; +import { BadRequestException, Logger } from '@nestjs/common'; import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; import { parse as createParser } from 'csv'; +import { FileFilterCallback } from 'fastify-multer/lib/interfaces'; import { Readable } from 'node:stream'; const logger = new Logger('ShelterCsvImporterHelpers'); -export const createCsvParser = () => - createParser({ columns: true, relaxColumnCount: true }, function (err, data) { - if (err) { - logger.error(err); - return null; - } - return data; - }); +export function createCsvParser() { + return createParser( + { columns: true, relaxColumnCount: true }, + function (err, data) { + if (err) { + logger.error(err); + return null; + } + return data; + }, + ); +} +export function csvImporterFilter( + _req: Express.Request, + file: Express.Multer.File, + callback: FileFilterCallback, +) { + if (!file.originalname.match(/\.(csv)$/)) { + return callback( + new BadRequestException('Apenas arquivos .csv são aceitos no momento!'), + false, + ); + } + callback(null, true); +} export function translatePrismaError(err: PrismaClientKnownRequestError) { switch (err.code) { case 'P2002': diff --git a/src/shelter-csv-importer/shelter-csv-importer.module.ts b/src/shelter-csv-importer/shelter-csv-importer.module.ts index 28b03139..5bd05368 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.module.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.module.ts @@ -5,5 +5,6 @@ import { ShelterCsvImporterService } from './shelter-csv-importer.service'; @Module({ imports: [PrismaModule], providers: [ShelterCsvImporterService], + exports: [ShelterCsvImporterService], }) export class ShelterCsvImporterModule {} diff --git a/src/shelter/shelter.controller.ts b/src/shelter/shelter.controller.ts index 24603857..bdeddd0a 100644 --- a/src/shelter/shelter.controller.ts +++ b/src/shelter/shelter.controller.ts @@ -8,22 +8,36 @@ import { Post, Put, Query, + Req, + UploadedFile, UseGuards, + UseInterceptors, } from '@nestjs/common'; -import { ApiTags } from '@nestjs/swagger'; +import { ApiConsumes, ApiTags } from '@nestjs/swagger'; import { ShelterService } from './shelter.service'; import { ServerResponse } from '../utils'; import { StaffGuard } from '@/guards/staff.guard'; import { ApplyUser } from '@/guards/apply-user.guard'; import { UserDecorator } from '@/decorators/UserDecorator/user.decorator'; +import { AdminGuard } from '@/guards/admin.guard'; +import { FastifyFileInterceptor } from '@/interceptors/file-upload.interceptor'; + +import { createReadStream, rmSync } from 'fs'; +import { diskStorage } from 'multer'; +import { FileDtoStub } from 'src/shelter-csv-importer/dto/file.dto'; +import { ShelterCsvImporterService } from 'src/shelter-csv-importer/shelter-csv-importer.service'; +import { csvImporterFilter } from 'src/shelter-csv-importer/shelter-csv-importer.helpers'; @ApiTags('Abrigos') @Controller('shelters') export class ShelterController { private logger = new Logger(ShelterController.name); - constructor(private readonly shelterService: ShelterService) {} + constructor( + private readonly shelterService: ShelterService, + private readonly shelterCsvImporter: ShelterCsvImporterService, + ) {} @Get('') async index(@Query() query) { @@ -95,4 +109,33 @@ export class ShelterController { throw new HttpException(err?.code ?? err?.name ?? `${err}`, 400); } } + + @UseGuards(AdminGuard) + @Post('import-csv') + @ApiConsumes('multipart/form-data', 'text/csv') + @UseInterceptors( + FastifyFileInterceptor('file', { + storage: diskStorage({ + filename: (_req, file, cb) => cb(null, file.originalname), + }), + fileFilter: csvImporterFilter, + }), + ) + async single( + @Req() _req: Request, + @UploadedFile() file: Express.Multer.File, + // empty body. Usado apenas para facilitar testes / upload usando swagger + @Body() _body: FileDtoStub, + ) { + const fileStream = createReadStream(file.path); + const res = await this.shelterCsvImporter.execute({ + fileStream, + // dryRun: true, + onEntity: console.log, + useIAToPredictSupplyCategories: false, + }); + rmSync(file.path); + + return res; + } } diff --git a/src/shelter/shelter.module.ts b/src/shelter/shelter.module.ts index 7859237a..f03f1a8a 100644 --- a/src/shelter/shelter.module.ts +++ b/src/shelter/shelter.module.ts @@ -3,9 +3,10 @@ import { Module } from '@nestjs/common'; import { ShelterService } from './shelter.service'; import { ShelterController } from './shelter.controller'; import { PrismaModule } from '../prisma/prisma.module'; +import { ShelterCsvImporterModule } from 'src/shelter-csv-importer/shelter-csv-importer.module'; @Module({ - imports: [PrismaModule], + imports: [PrismaModule, ShelterCsvImporterModule], providers: [ShelterService], controllers: [ShelterController], }) From 2d8cb25af77d06db459d5173c2ff2cb234856035 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Thu, 16 May 2024 22:39:00 -0300 Subject: [PATCH 10/16] =?UTF-8?q?feat:=20adicionando=20importa=C3=A7=C3=A3?= =?UTF-8?q?o=20via=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/shelter-csv-importer/dto/file.dto.ts | 7 ++--- .../shelter-csv-importer.helpers.ts | 26 ++++++++++++++-- .../shelter-csv-importer.service.ts | 9 ++++-- src/shelter-csv-importer/types.ts | 4 +-- src/shelter/shelter.controller.ts | 30 ++++++++++++------- 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/shelter-csv-importer/dto/file.dto.ts b/src/shelter-csv-importer/dto/file.dto.ts index 9ee7fa47..5cfc3df2 100644 --- a/src/shelter-csv-importer/dto/file.dto.ts +++ b/src/shelter-csv-importer/dto/file.dto.ts @@ -1,9 +1,8 @@ import { ApiProperty } from '@nestjs/swagger'; -/** - * Classe utilizada apenas para fins de documentação do swagger - */ export class FileDtoStub { - @ApiProperty({ type: 'string', format: 'binary' }) + @ApiProperty({ type: 'string', format: 'binary', required: false }) file?: string; + @ApiProperty({ required: false }) + csvUrl?: string; } diff --git a/src/shelter-csv-importer/shelter-csv-importer.helpers.ts b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts index 1cfe52e7..435fbd00 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.helpers.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.helpers.ts @@ -146,10 +146,30 @@ export async function detectSupplyCategoryUsingAI( return promptOutput; } -export function responseToReadable(response: Response) { +export const emptyReadable = () => + new Readable({ + read() { + this.push(null); + }, + }); + +export function csvResponseToReadable(response: Response) { + const contentType = response.headers.get('content-type'); + console.log( + '🚀 ~ ShelterCsvImporterService ~ awaitfetch ~ contentType:', + contentType, + ); const reader = response.body?.getReader(); - if (!reader) { - return new Readable(); + + if ( + !reader || + !contentType || + !contentType.toLowerCase().includes('text/csv') + ) { + logger.warn( + `reader não encontrado ou content-type não permitido. "${contentType}"`, + ); + return emptyReadable(); } return new Readable({ diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.ts b/src/shelter-csv-importer/shelter-csv-importer.service.ts index d962681c..94efa0dd 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.ts @@ -1,6 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; -import { Readable, Transform } from 'node:stream'; +import { Duplex, Readable, Transform, Writable } from 'node:stream'; import { TransformStream } from 'node:stream/web'; import { Prisma } from '@prisma/client'; @@ -9,7 +9,7 @@ import { AtomicCounter, createCsvParser, detectSupplyCategoryUsingAI, - responseToReadable, + csvResponseToReadable, translatePrismaError, } from './shelter-csv-importer.helpers'; import { @@ -53,7 +53,7 @@ export class ShelterCsvImporterService { const output: Record[] = []; let csvSourceStream = csvUrl - ? responseToReadable(await fetch(csvUrl)) + ? csvResponseToReadable(await fetch(csvUrl)) : fileStream!; const { @@ -129,6 +129,9 @@ export class ShelterCsvImporterService { ); } catch (err) { this.logger.error('Erro ao executar transaction', err); + atomicCounter.incrementFailure( + transactionArgs.length - atomicCounter.successCount, + ); } } this.logger.log( diff --git a/src/shelter-csv-importer/types.ts b/src/shelter-csv-importer/types.ts index bf10787e..6d6df232 100644 --- a/src/shelter-csv-importer/types.ts +++ b/src/shelter-csv-importer/types.ts @@ -33,8 +33,8 @@ interface ParseCsvArgsBaseArgs> { } export type ParseCsvArgs = - | (ParseCsvArgsBaseArgs & { csvUrl: string }) - | (ParseCsvArgsBaseArgs & { fileStream: Readable }); + | (ParseCsvArgsBaseArgs & { csvUrl: string; fileStream?: Readable }) + | (ParseCsvArgsBaseArgs & { fileStream: Readable; csvUrl?: string }); export interface EnhancedTransformArgs { /** * KeyValue contento as categorias e os supplies contidos naquela categorias (detectados por IA) diff --git a/src/shelter/shelter.controller.ts b/src/shelter/shelter.controller.ts index bdeddd0a..d03d1124 100644 --- a/src/shelter/shelter.controller.ts +++ b/src/shelter/shelter.controller.ts @@ -15,19 +15,20 @@ import { } from '@nestjs/common'; import { ApiConsumes, ApiTags } from '@nestjs/swagger'; -import { ShelterService } from './shelter.service'; -import { ServerResponse } from '../utils'; -import { StaffGuard } from '@/guards/staff.guard'; -import { ApplyUser } from '@/guards/apply-user.guard'; import { UserDecorator } from '@/decorators/UserDecorator/user.decorator'; -import { AdminGuard } from '@/guards/admin.guard'; +import { ApplyUser } from '@/guards/apply-user.guard'; +import { StaffGuard } from '@/guards/staff.guard'; import { FastifyFileInterceptor } from '@/interceptors/file-upload.interceptor'; +import { ServerResponse } from '../utils'; +import { ShelterService } from './shelter.service'; +import { AdminGuard } from '@/guards/admin.guard'; import { createReadStream, rmSync } from 'fs'; import { diskStorage } from 'multer'; import { FileDtoStub } from 'src/shelter-csv-importer/dto/file.dto'; -import { ShelterCsvImporterService } from 'src/shelter-csv-importer/shelter-csv-importer.service'; import { csvImporterFilter } from 'src/shelter-csv-importer/shelter-csv-importer.helpers'; +import { ShelterCsvImporterService } from 'src/shelter-csv-importer/shelter-csv-importer.service'; +import { Readable } from 'stream'; @ApiTags('Abrigos') @Controller('shelters') @@ -121,20 +122,27 @@ export class ShelterController { fileFilter: csvImporterFilter, }), ) - async single( + async importFromCsv( @Req() _req: Request, @UploadedFile() file: Express.Multer.File, - // empty body. Usado apenas para facilitar testes / upload usando swagger - @Body() _body: FileDtoStub, + @Body() body: FileDtoStub, ) { - const fileStream = createReadStream(file.path); + let fileStream!: Readable; + if (file?.path) { + fileStream = createReadStream(file.path); + } + const res = await this.shelterCsvImporter.execute({ fileStream, + csvUrl: body?.csvUrl, // dryRun: true, onEntity: console.log, + useBatchTransaction: true, useIAToPredictSupplyCategories: false, }); - rmSync(file.path); + if (file?.path) { + rmSync(file.path); + } return res; } From a36de3891558473ef37d87d7983a2760d101c4e3 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Sat, 18 May 2024 10:31:39 -0300 Subject: [PATCH 11/16] Revert "refactor: using datetime instead of string form dates with Prisma" This reverts commit ada941f94ff3259a29b9c7f76be5b4126501f7d0. --- .../migrations/20240511143905_/migration.sql | 38 ----------- prisma/schema.prisma | 68 +++++++++---------- 2 files changed, 34 insertions(+), 72 deletions(-) delete mode 100644 prisma/migrations/20240511143905_/migration.sql diff --git a/prisma/migrations/20240511143905_/migration.sql b/prisma/migrations/20240511143905_/migration.sql deleted file mode 100644 index 529d324f..00000000 --- a/prisma/migrations/20240511143905_/migration.sql +++ /dev/null @@ -1,38 +0,0 @@ -/* - Warnings: - Por padrão o prisma dropa e recria a coluna do banco. - Esse script usa um cast do postgres para converter as colunas para o tipo correto ao invés de dropá-las - Assumo que as timestamp estão sendo salvas em UTC - Para mudar a migration para setar as datas atuais para uma timezone específica, basta adicionar o statement "AT TIME ZONE" após a conversão - alter table "category_supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz AT TIME ZONE 'America/Sao_Paulo'; -*/ - --- AlterTable -alter table "category_supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "category_supplies" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "category_supplies" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; --- AlterTable -alter table "sessions" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "sessions" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "sessions" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; --- AlterTable -alter table "shelter_managers" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "shelter_managers" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "shelter_managers" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; --- AlterTable -alter table "shelter_supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "shelter_supplies" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "shelter_supplies" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; --- AlterTable -alter table "shelters" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "shelters" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "shelters" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; --- AlterTable -alter table "supplies" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "supplies" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "supplies" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz; --- AlterTable; -alter table "users" alter column "created_at" type TIMESTAMPTZ using "created_at"::timestamptz; -alter table "users" alter column "created_at" set default CURRENT_TIMESTAMP; -alter table "users" alter column "updated_at" type TIMESTAMPTZ using "updated_at"::timestamptz ; - diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 955416c9..7c726855 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -22,8 +22,8 @@ model User { password String phone String @unique accessLevel AccessLevel @default(value: User) - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) sessions Session[] shelterManagers ShelterManagers[] @@ -32,13 +32,13 @@ model User { } model Session { - id String @id @default(uuid()) - userId String @map("user_id") + id String @id @default(uuid()) + userId String @map("user_id") ip String? - userAgent String? @map("user_agent") - active Boolean @default(value: true) - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + userAgent String? @map("user_agent") + active Boolean @default(value: true) + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) user User @relation(fields: [userId], references: [id]) @@ -46,10 +46,10 @@ model Session { } model SupplyCategory { - id String @id @default(uuid()) - name String @unique - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + id String @id @default(uuid()) + name String @unique + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) supplies Supply[] @@ -57,12 +57,12 @@ model SupplyCategory { } model ShelterSupply { - shelterId String @map("shelter_id") - supplyId String @map("supply_id") - priority Int @default(value: 0) + shelterId String @map("shelter_id") + supplyId String @map("supply_id") + priority Int @default(value: 0) quantity Int? - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) shelter Shelter @relation(fields: [shelterId], references: [id]) supply Supply @relation(fields: [supplyId], references: [id]) @@ -72,11 +72,11 @@ model ShelterSupply { } model Supply { - id String @id @default(uuid()) - supplyCategoryId String @map("supply_category_id") + id String @id @default(uuid()) + supplyCategoryId String @map("supply_category_id") name String - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) supplyCategory SupplyCategory @relation(fields: [supplyCategoryId], references: [id]) shelterSupplies ShelterSupply[] @@ -85,25 +85,25 @@ model Supply { } model Shelter { - id String @id @default(uuid()) - name String @unique - pix String? @unique + id String @id @default(uuid()) + name String @unique + pix String? @unique address String street String? neighbourhood String? city String? streetNumber String? @map("street_number") zipCode String? @map("zip_code") - petFriendly Boolean? @map("pet_friendly") - shelteredPeople Int? @map("sheltered_people") + petFriendly Boolean? @map("pet_friendly") + shelteredPeople Int? @map("sheltered_people") capacity Int? contact String? - prioritySum Int @default(value: 0) @map("priority_sum") + prioritySum Int @default(value: 0) @map("priority_sum") latitude Float? longitude Float? - verified Boolean @default(value: false) - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + verified Boolean @default(value: false) + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) shelterManagers ShelterManagers[] shelterSupplies ShelterSupply[] @@ -112,10 +112,10 @@ model Shelter { } model ShelterManagers { - shelterId String @map("shelter_id") - userId String @map("user_id") - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz() - updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz() + shelterId String @map("shelter_id") + userId String @map("user_id") + createdAt String @map("created_at") @db.VarChar(32) + updatedAt String? @map("updated_at") @db.VarChar(32) user User @relation(fields: [userId], references: [id]) shelter Shelter @relation(fields: [shelterId], references: [id]) From 3e9614f7aaeb96d0c31d012342037797ac8e34ac Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Sat, 18 May 2024 10:31:55 -0300 Subject: [PATCH 12/16] Removendo arquivo trash adicionado ao git This reverts commit c0a08ca1b99e40f1398ceb198f304f32aa5486b8. --- routes.txt | 300 ----------------------------------------------------- 1 file changed, 300 deletions(-) delete mode 100644 routes.txt diff --git a/routes.txt b/routes.txt deleted file mode 100644 index b7a84e4c..00000000 --- a/routes.txt +++ /dev/null @@ -1,300 +0,0 @@ -{ - "Comment": "aaa", - "Changes": [ - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "guedder.com.", - "Type": "A", - "TTL": 60, - "ResourceRecords": [ - { - "Value": "35.86.23.115" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "guedder.com.", - "Type": "MX", - "TTL": 3600, - "ResourceRecords": [ - { - "Value": "1\tASPMX.L.GOOGLE.COM" - }, - { - "Value": "5\tALT1.ASPMX.L.GOOGLE.COM" - }, - { - "Value": "5\tALT2.ASPMX.L.GOOGLE.COM" - }, - { - "Value": "10\tALT3.ASPMX.L.GOOGLE.COM" - }, - { - "Value": "10\tALT4.ASPMX.L.GOOGLE.COM" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "guedder.com.", - "Type": "TXT", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "\"v=spf1 include:_spf.google.com ~all\"" - }, - { - "Value": "\"google-site-verification=6IGUHktwmqB1RHvTIZJlKsFkmqF1iqBwaiWvtB3PdbQ\"" - }, - { - "Value": "\"facebook-domain-verification=7b5s07221rinif6iso3tvajh34nxtz\"" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "_cc98d301c046cc32d416b9eb5632c6f1.guedder.com.", - "Type": "CNAME", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "_5f0fadf5c4885a67410f7f601135d9bc.dhzvlrndnj.acm-validations.aws." - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "google._domainkey.guedder.com.", - "Type": "TXT", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "\"v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAj35YXTEgL69Tj85RBFy6oNIu2q1gyXzWrfiW9DSWdBlxe6eqhq56bQVv7mRyoVVxW9IWPX4+Ut8InIEJFWam95SXlPx2O+l/\" \"CJ7TjcRtNe+ykIl+d0zpgkqZyXEPbiZYIsaU+O0nb/I0gtiGE4bYqb96S6OHN+6G4gTIpm5CPkC68DS2OsteOmzPUO7QKKWau4lG/\" \"oIROlcDPdNWucanF513VEdnRJsKHXk097JHq+YUhLyCjwDBpPJBq2H/0L6Usf9GZ9/jw9EI55NtnGOiuMPxBXncYeRVXXoyhlLkY5r7Bz7G8/\" \"XagO24b5Z5CGIJF3v5vDmOExGLwNZziYKC7QIDAQAB\"" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "lwdgihpjz4zecxghfjlwi5kk3wwepgb7._domainkey.guedder.com.", - "Type": "CNAME", - "TTL": 1800, - "ResourceRecords": [ - { - "Value": "lwdgihpjz4zecxghfjlwi5kk3wwepgb7.dkim.amazonses.com" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "ntit4w7do46e5gt4kndaznfvft2dhz4c._domainkey.guedder.com.", - "Type": "CNAME", - "TTL": 1800, - "ResourceRecords": [ - { - "Value": "ntit4w7do46e5gt4kndaznfvft2dhz4c.dkim.amazonses.com" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "woinkreggvmnukmv6m73vf762ghtdf4x._domainkey.guedder.com.", - "Type": "CNAME", - "TTL": 1800, - "ResourceRecords": [ - { - "Value": "woinkreggvmnukmv6m73vf762ghtdf4x.dkim.amazonses.com" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "api-loadbalance.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z1H1FL5HABSF5", - "DNSName": "guedder-load-balancer-2-737976362.us-west-2.elb.amazonaws.com.", - "EvaluateTargetHealth": true - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "api.guedder.com.", - "Type": "CNAME", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "api-loadbalance.guedder.com" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "app.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z2FDTNDATAQYW2", - "DNSName": "d1p99rujyyi7pk.cloudfront.net.", - "EvaluateTargetHealth": false - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "beta.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z2FDTNDATAQYW2", - "DNSName": "d6gohgvl1bk0m.cloudfront.net.", - "EvaluateTargetHealth": false - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "crux.guedder.com.", - "Type": "A", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "35.85.65.26" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "d4sign.guedder.com.", - "Type": "TXT", - "TTL": 3600, - "ResourceRecords": [ - { - "Value": "\"d4sign-domain-verification=e1455ea6-ad7e-436f-80f0-36bcb1634172\"" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "dev-api.guedder.com.", - "Type": "A", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "34.221.192.181" - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "_6eddddbbfb6bf75cbbe96037e6451c9e.dev-api.guedder.com.", - "Type": "CNAME", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "_df81fbb08064f9efa2917c1d25de75f6.dqxlbvzbzt.acm-validations.aws." - } - ] - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "institucional.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z2FDTNDATAQYW2", - "DNSName": "d2jp1q4t2zduz8.cloudfront.net.", - "EvaluateTargetHealth": false - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "portal.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z2FDTNDATAQYW2", - "DNSName": "d142ydjmx8obeg.cloudfront.net.", - "EvaluateTargetHealth": false - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "staging-portal.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z2FDTNDATAQYW2", - "DNSName": "d1tydsuuoy2nar.cloudfront.net.", - "EvaluateTargetHealth": false - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "staging.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z2FDTNDATAQYW2", - "DNSName": "d31p8c3qwfzi7x.cloudfront.net.", - "EvaluateTargetHealth": false - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "webapp.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z06176872A9AQ4DNUVAT0", - "DNSName": "guedder.com.", - "EvaluateTargetHealth": true - } - } - }, - { - "Action": "CREATE", - "ResourceRecordSet": { - "Name": "www.guedder.com.", - "Type": "A", - "AliasTarget": { - "HostedZoneId": "Z06176872A9AQ4DNUVAT0", - "DNSName": "guedder.com.", - "EvaluateTargetHealth": true - } - } - } - ] -} \ No newline at end of file From a09e6ecff9f803bc9e2acc36542a148f66122774 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Sat, 18 May 2024 10:36:44 -0300 Subject: [PATCH 13/16] chore: removing vscode config file --- .vscode/settings.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 4b8293b6..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "ai-test.endpoint": "a", - "ai-test.apiKey": "a" -} \ No newline at end of file From fd37932fb914507820b6ffda776fdf05b31e3989 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Tue, 21 May 2024 11:00:17 -0300 Subject: [PATCH 14/16] chore: fix tests --- src/shelter-csv-importer/shelter-csv-importer.service.ts | 2 +- src/shelter/shelter.controller.spec.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.ts b/src/shelter-csv-importer/shelter-csv-importer.service.ts index 94efa0dd..bcf87671 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.ts @@ -40,7 +40,7 @@ export class ShelterCsvImporterService { csvUrl, fileStream, dryRun = false, - useIAToPredictSupplyCategories = true, + useIAToPredictSupplyCategories = false, useBatchTransaction = false, onEntity, }: ShelterCsvImporterExecutionArgs) { diff --git a/src/shelter/shelter.controller.spec.ts b/src/shelter/shelter.controller.spec.ts index eab66b66..937eab15 100644 --- a/src/shelter/shelter.controller.spec.ts +++ b/src/shelter/shelter.controller.spec.ts @@ -2,6 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { PrismaService } from 'src/prisma/prisma.service'; import { ShelterController } from './shelter.controller'; import { ShelterService } from './shelter.service'; +import { ShelterCsvImporterService } from 'src/shelter-csv-importer/shelter-csv-importer.service'; describe('ShelterController', () => { let controller: ShelterController; @@ -9,7 +10,7 @@ describe('ShelterController', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [ShelterController], - providers: [ShelterService], + providers: [ShelterService, ShelterCsvImporterService], }) .useMocker((token) => { if (token === PrismaService) { From 9fb4e2ba17f6c987295097f6fe900dfa0a05583f Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Tue, 21 May 2024 12:03:52 -0300 Subject: [PATCH 15/16] chore: defining usage of IA to detect supply categories by environment variable --- .env.example | 4 +++- .env.local | 4 +++- src/shelter-csv-importer/shelter-csv-importer.service.ts | 4 +++- src/shelter/shelter.controller.ts | 3 --- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index f0e712f8..9e013376 100644 --- a/.env.example +++ b/.env.example @@ -11,4 +11,6 @@ SECRET_KEY= HOST=::0.0.0.0 PORT=4000 -GEMINI_API_KEY='' \ No newline at end of file +CSV_IMPORTER_USE_IA_TO_PREDICT_SUPPLY_CATEGORIES=false +# Opcional (Utilizado para detectar tipos de categorias ao importar abrigos) +GEMINI_API_KEY= diff --git a/.env.local b/.env.local index 8ee218d4..a5b5d958 100644 --- a/.env.local +++ b/.env.local @@ -12,4 +12,6 @@ SECRET_KEY=batata HOST=::0.0.0.0 PORT=4000 -GEMINI_API_KEY='' \ No newline at end of file +CSV_IMPORTER_USE_IA_TO_PREDICT_SUPPLY_CATEGORIES=false +# Opcional (Utilizado para detectar tipos de categorias ao importar abrigos) +GEMINI_API_KEY= \ No newline at end of file diff --git a/src/shelter-csv-importer/shelter-csv-importer.service.ts b/src/shelter-csv-importer/shelter-csv-importer.service.ts index bcf87671..b8c59be0 100644 --- a/src/shelter-csv-importer/shelter-csv-importer.service.ts +++ b/src/shelter-csv-importer/shelter-csv-importer.service.ts @@ -40,7 +40,9 @@ export class ShelterCsvImporterService { csvUrl, fileStream, dryRun = false, - useIAToPredictSupplyCategories = false, + useIAToPredictSupplyCategories = Boolean( + process.env.CSV_IMPORTER_USE_IA_TO_PREDICT_SUPPLY_CATEGORIES, + ), useBatchTransaction = false, onEntity, }: ShelterCsvImporterExecutionArgs) { diff --git a/src/shelter/shelter.controller.ts b/src/shelter/shelter.controller.ts index d03d1124..f0a4535d 100644 --- a/src/shelter/shelter.controller.ts +++ b/src/shelter/shelter.controller.ts @@ -135,10 +135,7 @@ export class ShelterController { const res = await this.shelterCsvImporter.execute({ fileStream, csvUrl: body?.csvUrl, - // dryRun: true, - onEntity: console.log, useBatchTransaction: true, - useIAToPredictSupplyCategories: false, }); if (file?.path) { rmSync(file.path); From 9d3e66e971aab014073f7cea2d80b9af9c5b6350 Mon Sep 17 00:00:00 2001 From: Danilo Romano Date: Tue, 21 May 2024 12:04:01 -0300 Subject: [PATCH 16/16] =?UTF-8?q?chore:=20declarando=20tipagem=20de=20vari?= =?UTF-8?q?=C3=A1veis=20de=20ambiente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- env.d.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 env.d.ts diff --git a/env.d.ts b/env.d.ts new file mode 100644 index 00000000..35eb4b5c --- /dev/null +++ b/env.d.ts @@ -0,0 +1,20 @@ +declare global { + declare namespace NodeJS { + export interface ProcessEnv { + TZ?: string; + DB_HOST: string; + DB_PORT: string; + DB_DATABASE_NAME: string; + DB_USER: string; + DB_PASSWORD: string; + DATABASE_URL: string; + SECRET_KEY: string; + HOST: string; + PORT: string; + CSV_IMPORTER_USE_IA_TO_PREDICT_SUPPLY_CATEGORIES: boolean; + GEMINI_API_KEY?: string; + } + } +} + +export {};