Add in-app settings editor (#137)

* Testing before deleting stuff

* Delete more stuff

* Move alasql out of dependencies

* Delete more stuff

* Mark more as unused in desktop

* Fix quotes

* Drop more

* More dropped stuff

* Drop parquetjs

* More removals

* Try yarn removing instead

* Fix typo

* Exclude more

* Dont ignore runner

* Change which are removed

* Cant drop those

* Revert community edition

* Comment out upx

* Drop upx

* Fix for settings

* Add settings button

* Some fixes for desktop

* Fixes for formatter

* Only show message on desktop

* Fix go checks

* Closer!

* Fixes

* Add notes about building language

* More fixes

* Fixes for everything

* Format after build

* Sketch for clojure

* Make jsonnet available in path

* Reset after release build

* Drop clojure

* Fixes

* Drop clojure tests

* In memory init

* Fix for empty sql panel

* Fix for format

* Fix for project name with dot in it

* Try ignoring browser eval tests

* Try killing and waiting before continuing

* Fix format

* Drop coverage reqs

* Add language override settings

Author: eatonphil

.github/workflows/pull_requests.yml

@@ -35,7 +35,7 @@ jobs:
     - run: yarn format
     - run: yarn tsc
     - run: cd runner && gofmt -w .
-    - run: ./runner/scripts/generate_program_type_info.sh
+    - run: yarn build-language-definitions
     - run: ./scripts/fail_on_diff.sh
     - run: yarn test-licenses
     # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests
@@ -104,6 +104,7 @@ jobs:
 
     - run: ./scripts/ci/prepare_linux.sh
     - run: yarn release-desktop 0.0.0-e2etest
+    - run: git reset --hard # release blows everything up
       # Set up a virtual framebuffer so Chrome will start
       # https://www.electronjs.org/docs/tutorial/testing-on-headless-ci
       # https://github.com/juliangruber/browser-run/issues/147
@@ -136,6 +137,7 @@ jobs:
     - run: yarn test --runInBand --detectOpenHandles --forceExit --verbose shared ui desktop
     - run: cd runner && go test -cover
     - run: yarn release-desktop 0.0.0-e2etest
+    - run: git reset --hard # release blows everything up
     - run: yarn e2e-test
 
   e2e-windows:
@@ -166,6 +168,7 @@ jobs:
     - run: yarn test --runInBand --detectOpenHandles --forceExit --verbose shared ui desktop
     - run: cd runner && go test -cover
     - run: yarn release-desktop 0.0.0-e2etest
+    - run: git reset --hard # release blows everything up
     - run: yarn e2e-test
 
   dsq-tests-ubuntu:

DEV_FAQ.md

@@ -42,4 +42,17 @@ For example:
 
 ```bash
 yarn test desktop/panel/file.test.js --dsrunner=memory
+```
+
+## Adding/editing language libraries
+
+Language libraries (i.e. the implementation of
+DM_getPanel/DM_setPanel) are in shared/libraries/*.jsonnet. They are
+in jsonnet so that you can have nicely formatted code. They get
+compiled to JSON and committed. They also get embedded into Go code.
+
+To re-run this process, use on Mac or Linux:
+
+```
+$ yarn build-language-definitions
 ```

desktop/panel/columns.test.js

@@ -14,6 +14,10 @@ const { fetchResultsHandler } = require('./columns');
 const { RUNNERS } = require('./testutil');
 
 for (const runner of RUNNERS) {
+  if (!runner?.go) {
+    continue; // Otherwise not implemented
+  }
+
   test(`store and retrieve literal ${
     runner ? 'using ' + runner : ''
   }, specific columns`, async () => {

desktop/panel/columns.ts

@@ -2,13 +2,11 @@ import fs from 'fs';
 import { PanelBody } from '../../shared/rpc';
 import {
   GraphPanelInfo,
-  LiteralPanelInfo,
   PanelInfo,
   ProjectState,
   TablePanelInfo,
 } from '../../shared/state';
 import { columnsFromObject } from '../../shared/table';
-import { parseArrayBuffer } from '../../shared/text';
 import { Dispatch, RPCHandler } from '../rpc';
 import { getProjectResultsFile } from '../store';
 import { getPanelResult, getProjectAndPanel } from './shared';
@@ -54,16 +52,6 @@ export async function evalColumns(
   };
 }
 
-export async function evalLiteral(
-  project: ProjectState,
-  panel: PanelInfo
-): Promise<EvalHandlerResponse> {
-  const {
-    literal: { contentTypeInfo },
-  } = guardPanel<LiteralPanelInfo>(panel, 'literal');
-  return await parseArrayBuffer(contentTypeInfo, '', panel.content);
-}
-
 export const fetchResultsHandler: RPCHandler<PanelBody, EvalHandlerResponse> = {
   resource: 'fetchResults',
   handler: async function (

desktop/panel/eval.ts

@@ -43,15 +43,21 @@ type EvalHandler = (
   dispatch: Dispatch
 ) => Promise<EvalHandlerResponse>;
 
+function unimplementedInJavaScript(): EvalHandler {
+  return function () {
+    throw new Error('There is a bug, this condition should not be possible.');
+  };
+}
+
 const EVAL_HANDLERS: { [k in PanelInfoType]: () => EvalHandler } = {
-  filagg: () => require('./filagg').evalFilterAggregate,
-  file: () => require('./file').evalFile,
-  http: () => require('./http').evalHTTP,
-  program: () => require('./program').evalProgram,
   database: () => require('./database').evalDatabase,
   table: () => require('./columns').evalColumns,
   graph: () => require('./columns').evalColumns,
-  literal: () => require('./columns').evalLiteral,
+  literal: unimplementedInJavaScript,
+  file: unimplementedInJavaScript,
+  http: unimplementedInJavaScript,
+  program: unimplementedInJavaScript,
+  filagg: unimplementedInJavaScript,
 };
 
 const runningProcesses: Record<string, Set<number>> = {};

desktop/panel/filagg.ts

@@ -28,6 +28,10 @@ const baseline = JSON.parse(
 );
 
 for (const subprocessName of RUNNERS) {
+  if (!subprocessName?.go) {
+    continue; // Otherwise not implemented
+  }
+
   describe(
     'eval generic file via ' +
       (subprocessName ? subprocessName.go || subprocessName.node : 'memory'),

desktop/panel/file.test.js

@@ -4,6 +4,8 @@ const { ensureSigningKey } = require('../secret');
 const { spawn } = require('child_process');
 const { CODE_ROOT } = require('../constants');
 const path = require('path');
+const cp = require('child_process');
+const os = require('os');
 const fs = require('fs');
 const { getProjectResultsFile } = require('../store');
 const {
@@ -27,17 +29,54 @@ const baseline = JSON.parse(
 );
 
 const USERDATA_FILES = ['json', 'xlsx', 'csv', 'parquet', 'jsonl'];
+const PORT = '9799';
 
-const cp = spawn('python3', ['-m', 'http.server', '9799']);
-cp.stdout.on('data', (data) => {
-  console.log(data.toString());
-});
+let server;
+beforeAll(async () => {
+  // TODO: port this logic to other platforms...
+  if (process.platform === 'linux') {
+    try {
+      cp.execSync(
+        `bash -c "ps aux | grep 'http.server ${PORT}' | grep -v grep | awk '{print \\$2}' | xargs -I {} kill {}"`
+      );
+    } catch (e) {
+      console.error(e);
+    }
+  }
 
-cp.stderr.on('data', (data) => {
-  console.warn(data.toString());
+  server = spawn('python3', ['-m', 'http.server', PORT]);
+  let ready = false;
+  server.on('spawn', () => {
+    ready = true;
+  });
+
+  server.stdout.on('data', (data) => {
+    console.log(data.toString());
+  });
+
+  server.stderr.on('data', (data) => {
+    console.warn(data.toString());
+  });
+
+  return new Promise(async (resolve, reject) => {
+    try {
+      while (!ready) {
+        console.log('still waiting');
+        await new Promise((resolve) => setTimeout(resolve, 300));
+      }
+
+      resolve();
+    } catch (e) {
+      reject(e);
+    }
+  });
 });
 
 for (const subprocessName of RUNNERS) {
+  if (!subprocessName?.go) {
+    continue; // Otherwise not implemented
+  }
+
   describe(
     'eval generic file via ' +
       (subprocessName ? subprocessName.go || subprocessName.node : 'memory'),
@@ -208,5 +247,5 @@ for (const subprocessName of RUNNERS) {
 }
 
 afterAll(() => {
-  process.kill(cp.pid);
+  process.kill(server.pid);
 });