feat(Agents): Merges xapi-agents service. (#469)

Author: ryasmi

package-lock.json

@@ -38,22 +38,36 @@
     "npm": ">3.0.0"
   },
   "dependencies": {
-    "@learninglocker/xapi-agents": "4.4.3",
+    "@azure/storage-blob": "^10.3.0",
+    "@google-cloud/storage": "^1.5.2",
+    "@learninglocker/xapi-validation": "^2.1.10",
     "accept-language-parser": "^1.5.0",
+    "atob": "^2.0.3",
+    "aws-sdk": "^2.205.0",
+    "bluebird": "3.5.0",
     "boolean": "^0.2.0",
+    "btoa": "^1.1.2",
     "dotenv": "^5.0.0",
     "express": "^4.14.1",
+    "fs-extra": "^5.0.0",
+    "http-status-codes": "^1.3.0",
     "install": "^0.13.0",
     "ioredis": "^4.14.0",
-    "jscommons": "^2.3.3",
+    "jscommons": "^2.3.0",
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.4",
+    "mime-types": "2.1.17",
     "mongodb": "^3.0.2",
+    "node-fetch": "^2.0.0",
     "object-hash": "^1.3.1",
     "query-string": "^6.8.2",
     "redis": "^2.8.0",
+    "rulr": "^4.0.0",
+    "sha1": "^1.1.1",
     "source-map-support": "^0.5.0",
-    "string-to-stream": "^2.0.0"
+    "stream-to-string": "^1.2.0",
+    "string-to-stream": "^2.0.0",
+    "uuid": "^3.0.1"
   },
   "devDependencies": {
     "@ht2-labs/semantic-release": "1.1.90",
@@ -72,6 +86,7 @@
     "@types/mocha": "5.2.7",
     "@types/mongodb": "3.1.10",
     "@types/node": "9.6.51",
+    "@types/node-fetch": "2.5.0",
     "@types/object-hash": "1.3.0",
     "@types/redis": "2.8.6",
     "@types/source-map-support": "0.5.0",

package.json

@@ -1,7 +1,6 @@
 import { Router } from 'express';
 import mixinCors from 'jscommons/dist/expressPresenter/mixins/cors';
 import mixinHelmet from 'jscommons/dist/expressPresenter/mixins/helmet';
-import mixinMorgan from 'jscommons/dist/expressPresenter/mixins/morgan';
 import Config from './Config';
 import deleteProfile from './deleteProfile';
 import getProfiles from './getProfiles';
@@ -13,7 +12,6 @@ export default (config: Config): Router => {
 
   router.use(mixinCors());
   router.use(mixinHelmet());
-  router.use(mixinMorgan(config.morganDirectory));
 
   router.delete('', deleteProfile(config));
   router.get('', getProfiles(config));

src/apps/activities/expressPresenter/index.ts

@@ -5,7 +5,7 @@ import {
   StorageURL,
 } from '@azure/storage-blob';
 import Storage from '@google-cloud/storage';
-import { S3 } from 'aws-sdk';
+import S3 from 'aws-sdk/clients/s3';
 import azureStorageRepo from '../../azureStorageRepo';
 import googleStorageRepo from '../../googleStorageRepo';
 import localStorageRepo from '../../localStorageRepo';
@@ -18,7 +18,7 @@ export default (factoryConfig: FactoryConfig): Repo => {
     case 's3':
       return s3StorageRepo({
         bucketName: factoryConfig.s3.bucketName,
-        client: new S3(factoryConfig.s3.awsConfig),
+        client: new S3(factoryConfig.s3.awsConfig) as any,
         subFolder: factoryConfig.s3.subFolder,
       });
     case 'google':

src/apps/activities/repo/storage/factory.ts

@@ -68,7 +68,7 @@ const getStorageRepo = (): StorageRepo => {
     case 's3':
       return s3StorageRepo({
         bucketName: config.s3StorageRepo.bucketName,
-        client: new S3(config.s3StorageRepo.awsConfig),
+        client: new S3(config.s3StorageRepo.awsConfig) as any,
         subFolder: config.storageSubFolders.activities,
       });
     case 'google':

src/apps/activities/repoFactory/index.ts

@@ -1,10 +1,10 @@
 /* tslint:disable:max-file-line-count */
-import presenterFactory from '@learninglocker/xapi-agents/dist/expressPresenter';
-import serviceFactory from '@learninglocker/xapi-agents/dist/service';
-import enTranslator from '@learninglocker/xapi-agents/dist/translatorFactory/en';
 import { Router } from 'express';
 import AppConfig from './AppConfig';
+import presenterFactory from './expressPresenter';
 import repoFactory from './repo/factory';
+import serviceFactory from './service';
+import enTranslator from './translatorFactory/en';
 
 export default (appConfig: AppConfig): Router => {
   const translator = enTranslator;

src/apps/agents/app.ts

@@ -0,0 +1,6 @@
+import { ContainerURL } from '@azure/storage-blob';
+
+export default interface Config {
+  readonly containerUrl: ContainerURL;
+  readonly subFolder: string;
+}

src/apps/agents/azureStorageRepo/Config.ts

@@ -0,0 +1,27 @@
+import {
+  Aborter,
+  BlobURL,
+  Models,
+} from '@azure/storage-blob';
+import Config from './Config';
+
+export default (config: Config) => {
+  return async (): Promise<void> => {
+    // tslint:disable-next-line:no-let
+    let marker;
+    do {
+      const listBlobsResponse: Models.ContainerListBlobFlatSegmentResponse =
+        await config.containerUrl.listBlobFlatSegment(Aborter.none, marker);
+      marker = listBlobsResponse.nextMarker;
+      const deletePromises = listBlobsResponse.segment.blobItems.map(
+        async (blobItem: Models.BlobItem) => {
+          const blobUrl = BlobURL.fromContainerURL(config.containerUrl, blobItem.name);
+          if (blobItem.name.startsWith(config.subFolder)) {
+            await blobUrl.delete(Aborter.none);
+          }
+        },
+      );
+      await Promise.all(deletePromises);
+    } while (marker !== '');
+  };
+};

src/apps/agents/azureStorageRepo/clearRepo.ts

@@ -0,0 +1,18 @@
+import {
+  Aborter,
+  BlobURL,
+} from '@azure/storage-blob';
+import DeleteProfileContentOptions from '../repoFactory/options/DeleteProfileContentOptions';
+import getStorageDir from '../utils/getStorageDir';
+import Config from './Config';
+
+ export default (config: Config) => {
+  return async (opts: DeleteProfileContentOptions): Promise<void> => {
+    const profileDir = getStorageDir({ subfolder: config.subFolder, lrs_id: opts.lrs_id });
+    const filePath = `${profileDir}/${opts.key}`;
+
+     const blobUrl = BlobURL.fromContainerURL(config.containerUrl, filePath);
+
+     await blobUrl.delete(Aborter.none);
+  };
+};

src/apps/agents/azureStorageRepo/deleteProfileContent.ts

@@ -0,0 +1,20 @@
+import { Aborter, BlobURL } from '@azure/storage-blob';
+import GetProfileContentOptions from '../repoFactory/options/GetProfileContentOptions';
+import GetProfileContentResult from '../repoFactory/results/GetProfileContentResult';
+import getStorageDir from '../utils/getStorageDir';
+import Config from './Config';
+
+ export default (config: Config) => {
+  return async (opts: GetProfileContentOptions): Promise<GetProfileContentResult> => {
+    const profileDir = getStorageDir({ subfolder: config.subFolder, lrs_id: opts.lrs_id });
+    const filePath = `${profileDir}/${opts.key}`;
+
+     const blobUrl = BlobURL.fromContainerURL(config.containerUrl, filePath);
+    const content = (await blobUrl.download(Aborter.none, 0)).readableStreamBody;
+    if (content === undefined) {
+      throw new Error('Blob not found');
+    }
+
+    return { content };
+  };
+};