Skip to content

Commit 69a8137

Browse files
eatonphileatonphil
authored
Require pagination on table panels, fall back to regular input when shapes aren't valid in FieldPicker, better Show/Hide Options UI, allow Google Sheets (#235)
* Basic pagination for table * Try a new options styling * Increase graph max * Started supporting google sheets * Google sheets working * Cleanup * Fixes for fmt * Fix for gofmt * Fix for tests * Fixes for panel names * Unnecessary neo4j step * Move everything to bufio * Fixes for eslint * Tests passing locally * Start moving fetch results to runner * Sketch out fetch * Add tests for graphtable * Graph and table panels return values * Fixes for tests * No expand when no body * Cleanup page buttons * Fix google sheet test * Select so that we can break
1 parent 5a50d7f commit 69a8137

40 files changed

+1603
-933
lines changed

‎.eslintignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
1-
ui/scripts/*
2-
ui/state.test.js
1+
ui/scripts/*

‎desktop/panel/columns.test.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ for (const runner of RUNNERS) {
3232
const tp = new TablePanelInfo(null, {
3333
columns: [{ field: 'a' }],
3434
panelSource: lp.id,
35+
name: 'Table',
3536
});
3637

3738
let finished = false;
@@ -59,7 +60,7 @@ for (const runner of RUNNERS) {
5960
expect(result.preview).toStrictEqual(preview(testData));
6061
expect(result.contentType).toBe('application/json');
6162

62-
const p = await makeEvalHandler().handler(
63+
const p = await makeEvalHandler(runner).handler(
6364
project.projectName,
6465
{ panelId: tp.id },
6566
dispatch

‎desktop/panel/columns.ts‎

Lines changed: 4 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,11 @@
11
import fs from 'fs';
22
import { PanelBody } from '../../shared/rpc';
3-
import {
4-
GraphPanelInfo,
5-
PanelInfo,
6-
ProjectState,
7-
TablePanelInfo,
8-
} from '../../shared/state';
9-
import { columnsFromObject } from '../../shared/table';
103
import { Dispatch, RPCHandler } from '../rpc';
114
import { getProjectResultsFile } from '../store';
12-
import { getPanelResult, getProjectAndPanel } from './shared';
13-
import { EvalHandlerExtra, EvalHandlerResponse, guardPanel } from './types';
14-
15-
export async function evalColumns(
16-
project: ProjectState,
17-
panel: PanelInfo,
18-
{ idMap }: EvalHandlerExtra,
19-
dispatch: Dispatch
20-
): Promise<EvalHandlerResponse> {
21-
let columns: Array<string>;
22-
let panelSource: string;
23-
if (panel.type === 'graph') {
24-
const gp = panel as GraphPanelInfo;
25-
columns = [
26-
gp.graph.x,
27-
gp.graph.uniqueBy,
28-
...gp.graph.ys.map((y) => y.field),
29-
].filter(Boolean);
30-
panelSource = gp.graph.panelSource;
31-
} else if (panel.type === 'table') {
32-
const tp = panel as TablePanelInfo;
33-
columns = tp.table.columns.map((c) => c.field);
34-
panelSource = tp.table.panelSource;
35-
} else {
36-
// Let guardPanel throw a nice error.
37-
guardPanel<GraphPanelInfo>(panel, 'graph');
38-
}
39-
40-
if (!panelSource) {
41-
throw new Error('Panel source not specified, cannot eval.');
42-
}
43-
44-
const { value } = await getPanelResult(
45-
dispatch,
46-
project.projectName,
47-
panelSource
48-
);
49-
50-
const valueWithRequestedColumns = columnsFromObject(
51-
value,
52-
columns,
53-
// Assumes that position always comes before panel name in the idmap
54-
+Object.keys(idMap).find((key) => idMap[key] === panelSource)
55-
);
56-
57-
return {
58-
value: valueWithRequestedColumns,
59-
returnValue: true,
60-
};
61-
}
5+
import { getProjectAndPanel } from './shared';
6+
import { EvalHandlerResponse } from './types';
627

8+
// TODO: this needs to be ported to go
639
export const fetchResultsHandler: RPCHandler<PanelBody, EvalHandlerResponse> = {
6410
resource: 'fetchResults',
6511
handler: async function (
@@ -75,6 +21,7 @@ export const fetchResultsHandler: RPCHandler<PanelBody, EvalHandlerResponse> = {
7521

7622
// Maybe the only appropriate place to call this in this package?
7723
const projectResultsFile = getProjectResultsFile(projectId);
24+
// TODO: this is a 4GB file limit!
7825
const f = fs.readFileSync(projectResultsFile + panel.id);
7926

8027
// Everything gets stored as JSON on disk. Even literals and files get rewritten as JSON.

‎desktop/panel/database.test.js‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -470,9 +470,6 @@ for (const subprocess of RUNNERS) {
470470
return;
471471
}
472472

473-
// Mongo doesn't work yet.
474-
return;
475-
476473
const connectors = [
477474
new DatabaseConnectorInfo({
478475
type: 'google-sheets',
@@ -496,11 +493,11 @@ for (const subprocess of RUNNERS) {
496493

497494
const v = JSON.parse(panelValueBuffer.toString());
498495
expect(v).toStrictEqual([
499-
{ age: 52, name: 'Emma' },
500-
{ age: 50, name: 'Karl' },
501-
{ age: 43, name: 'Garry' },
502-
{ age: 41, name: 'Nile' },
503-
{ age: 39, name: 'Mina' },
496+
{ age: '43', name: 'Garry' },
497+
{ age: '39', name: 'Mina' },
498+
{ age: '50', name: 'Karl' },
499+
{ age: '41', name: 'Nile' },
500+
{ age: '52', name: 'Emma' },
504501
]);
505502

506503
finished = true;

‎desktop/panel/eval.ts‎

Lines changed: 18 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,14 @@
11
import { execFile } from 'child_process';
22
import fs from 'fs';
3-
import jsesc from 'jsesc';
43
import circularSafeStringify from 'json-stringify-safe';
54
import { EOL } from 'os';
65
import path from 'path';
7-
import { preview } from 'preview';
8-
import { shape, Shape } from 'shape';
96
import { file as makeTmpFile } from 'tmp-promise';
10-
import {
11-
Cancelled,
12-
EVAL_ERRORS,
13-
InvalidDependentPanelError,
14-
NoResultError,
15-
} from '../../shared/errors';
7+
import { Cancelled, EVAL_ERRORS, NoResultError } from '../../shared/errors';
168
import log from '../../shared/log';
179
import { newId } from '../../shared/object';
1810
import { PanelBody } from '../../shared/rpc';
19-
import {
20-
ConnectorInfo,
21-
PanelInfo,
22-
PanelInfoType,
23-
PanelResult,
24-
ProjectState,
25-
} from '../../shared/state';
11+
import { ConnectorInfo, PanelInfo, PanelResult } from '../../shared/state';
2612
import {
2713
CODE_ROOT,
2814
DISK_ROOT,
@@ -37,31 +23,6 @@ import { parsePartialJSONFile } from '../partial';
3723
import { Dispatch, RPCHandler } from '../rpc';
3824
import { getProjectResultsFile } from '../store';
3925
import { getProjectAndPanel } from './shared';
40-
import { EvalHandlerExtra, EvalHandlerResponse } from './types';
41-
42-
type EvalHandler = (
43-
project: ProjectState,
44-
panel: PanelInfo,
45-
extra: EvalHandlerExtra,
46-
dispatch: Dispatch
47-
) => Promise<EvalHandlerResponse>;
48-
49-
function unimplementedInJavaScript(): EvalHandler {
50-
return function () {
51-
throw new Error('There is a bug, this condition should not be possible.');
52-
};
53-
}
54-
55-
const EVAL_HANDLERS: { [k in PanelInfoType]: () => EvalHandler } = {
56-
table: () => require('./columns').evalColumns,
57-
graph: () => require('./columns').evalColumns,
58-
literal: unimplementedInJavaScript,
59-
database: unimplementedInJavaScript,
60-
file: unimplementedInJavaScript,
61-
http: unimplementedInJavaScript,
62-
program: unimplementedInJavaScript,
63-
filagg: unimplementedInJavaScript,
64-
};
6526

6627
const runningProcesses: Record<string, Set<number>> = {};
6728
const cancelledPids = new Set<number>();
@@ -84,10 +45,6 @@ function killAllByPanelId(panelId: string) {
8445
}
8546
}
8647

87-
function canUseGoRunner(panel: PanelInfo, connectors: ConnectorInfo[]) {
88-
return !['table', 'graph'].includes(panel.type);
89-
}
90-
9148
export async function evalInSubprocess(
9249
subprocess: {
9350
node: string;
@@ -122,7 +79,7 @@ export async function evalInSubprocess(
12279
args.push(SETTINGS_FILE_FLAG, subprocess.settingsFileOverride);
12380
}
12481

125-
if (subprocess.go && canUseGoRunner(panel, connectors)) {
82+
if (subprocess.go) {
12683
base = subprocess.go;
12784
args.shift();
12885
}
@@ -213,6 +170,14 @@ export async function evalInSubprocess(
213170
throw e;
214171
}
215172
}
173+
174+
// Table and graph panels get their results passed back to me displayed in the UI
175+
if (['table', 'graph'].includes(panel.type)) {
176+
const projectResultsFile = getProjectResultsFile(projectName);
177+
const bytes = fs.readFileSync(projectResultsFile + panel.id);
178+
rm.value = JSON.parse(bytes.toString());
179+
}
180+
216181
return [{ ...rm, ...resultMeta }, stderr];
217182
} finally {
218183
try {
@@ -227,30 +192,6 @@ export async function evalInSubprocess(
227192
}
228193
}
229194

230-
function assertValidDependentPanels(
231-
projectId: string,
232-
content: string,
233-
idMap: Record<string | number, string>
234-
) {
235-
const projectResultsFile = getProjectResultsFile(projectId);
236-
const re =
237-
/(DM_getPanel\((?<number>[0-9]+)\))|(DM_getPanel\((?<singlequote>'(?:[^'\\]|\\.)*\')\))|(DM_getPanel\((?<doublequote>"(?:[^"\\]|\\.)*\")\))/g;
238-
let match = null;
239-
while ((match = re.exec(content)) !== null) {
240-
if (match && match.groups) {
241-
const { number, singlequote, doublequote } = match.groups;
242-
let m = doublequote || singlequote || number;
243-
if (["'", '"'].includes(m.charAt(0))) {
244-
m = m.slice(1, m.length - 1);
245-
}
246-
247-
if (!fs.existsSync(projectResultsFile + idMap[m])) {
248-
throw new InvalidDependentPanelError(m);
249-
}
250-
}
251-
}
252-
}
253-
254195
async function evalNoUpdate(
255196
projectId: string,
256197
body: PanelBody,
@@ -260,7 +201,7 @@ async function evalNoUpdate(
260201
go?: string;
261202
}
262203
): Promise<[Partial<PanelResult>, string]> {
263-
const { project, panel, panelPage } = await getProjectAndPanel(
204+
const { project, panel } = await getProjectAndPanel(
264205
dispatch,
265206
projectId,
266207
body.panelId
@@ -273,66 +214,16 @@ async function evalNoUpdate(
273214
body: { data: new PanelResult(), panelId: panel.id },
274215
});
275216

276-
if (subprocessEval) {
277-
return evalInSubprocess(
278-
subprocessEval,
279-
project.projectName,
280-
panel,
281-
project.connectors
282-
);
217+
if (!subprocessEval) {
218+
throw new Error('Developer error: all eval must use subprocess');
283219
}
284220

285-
const idMap: Record<string | number, string> = {};
286-
const idShapeMap: Record<string | number, Shape> = {};
287-
project.pages[panelPage].panels.forEach((p, i) => {
288-
idMap[i] = p.id;
289-
idMap[p.name] = p.id;
290-
idShapeMap[i] = p.resultMeta.shape;
291-
idShapeMap[p.name] = p.resultMeta.shape;
292-
});
293-
294-
assertValidDependentPanels(projectId, panel.content, idMap);
295-
296-
const evalHandler = EVAL_HANDLERS[panel.type]();
297-
const res = await evalHandler(
298-
project,
221+
return evalInSubprocess(
222+
subprocessEval,
223+
project.projectName,
299224
panel,
300-
{
301-
idMap,
302-
idShapeMap,
303-
},
304-
dispatch
225+
project.connectors
305226
);
306-
307-
// TODO: is it a problem panels like Program skip this escaping?
308-
// This library is important for escaping responses otherwise some
309-
// characters can blow up various panel processes.
310-
const json = jsesc(res.value, { quotes: 'double', json: true });
311-
312-
if (!res.skipWrite) {
313-
const projectResultsFile = getProjectResultsFile(projectId);
314-
fs.writeFileSync(projectResultsFile + panel.id, json);
315-
}
316-
317-
const s = shape(res.value);
318-
319-
return [
320-
{
321-
stdout: res.stdout || '',
322-
preview: preview(res.value),
323-
shape: s,
324-
value: res.returnValue ? res.value : null,
325-
size: res.size === undefined ? json.length : res.size,
326-
arrayCount:
327-
res.arrayCount === undefined
328-
? s.kind === 'array'
329-
? (res.value || []).length
330-
: null
331-
: res.arrayCount,
332-
contentType: res.contentType || 'application/json',
333-
},
334-
'',
335-
];
336227
}
337228

338229
export const makeEvalHandler = (subprocessEval?: {

0 commit comments

Comments
 (0)