Add basic autocomplete support (#213)

* Start basic autocomplete

* Basic autocomplete working

* Fix for height

* Fix for fmt

* Fix for nested fields

* Formats

* Bump coverage

* Duplicate ace type definition

* Add copyright

Author: eatonphil

.github/workflows/docker.yml

@@ -12,7 +12,8 @@
     "typeRoots": [
       "../node_modules/@types",
       "../type-overrides",
-      "./node_modules/@types"
+      "./node_modules/@types",
+      "./type-overrides"
     ],
     "resolveJsonModule": true
   },

ee/tsconfig.json

@@ -0,0 +1,4 @@
+// Copyright 2022 Multiprocess Labs LLC
+
+declare module 'ace-builds/src-min-noconflict/ace';
+declare module 'ace-builds/src-min-noconflict/ext-language_tools';

ee/type-overrides/ace.d.ts

@@ -4,10 +4,10 @@ module.exports = {
     process.platform === 'linux'
       ? {
           global: {
-            statements: 55,
+            statements: 54,
             branches: 41,
-            functions: 36,
-            lines: 55,
+            functions: 35,
+            lines: 54,
           },
         }
       : undefined,

jest.config.js

@@ -28,7 +28,7 @@
   "dependencies": {
     "@tabler/icons": "^1.56.0",
     "ace-builds": "^1.4.13",
-    "better-sqlite3": "multiprocessio/better-sqlite3#11866f1",
+    "better-sqlite3": "^7.5.1",
     "chart.js": "^3.5.1",
     "cookie-parser": "^1.4.5",
     "core-js": "^3.21.1",

package.json

@@ -0,0 +1,2 @@
+declare module 'ace-builds/src-min-noconflict/ace';
+declare module 'ace-builds/src-min-noconflict/ext-language_tools';

type-overrides/ace.d.ts

@@ -1,7 +1,16 @@
+import log from '../../shared/log';
 import { useDebouncedCallback } from 'use-debounce';
-// organize-imports-ignore
+import * as React from 'react';
+import { SettingsContext } from '../Settings';
+import { Tooltip } from './Tooltip';
+import { INPUT_SYNC_PERIOD } from './Input';
+
 // Must be loaded before other ace-builds imports
 import AceEditor from 'react-ace';
+// organize-imports-ignore
+import { Ace } from 'ace-builds';
+import ace from 'ace-builds/src-min-noconflict/ace';
+import langTools from 'ace-builds/src-min-noconflict/ext-language_tools';
 // Enables Ctrl-f
 import 'ace-builds/src-min-noconflict/ext-searchbox';
 // Enables syntax highlighting
@@ -15,13 +24,25 @@ import 'ace-builds/src-min-noconflict/mode-sql';
 // UI theme
 import 'ace-builds/src-min-noconflict/theme-github';
 import 'ace-builds/src-min-noconflict/theme-dracula';
-import * as React from 'react';
 // Shortcuts support, TODO: support non-emacs
 // This steals Ctrl-a so this should not be a default
 //import 'ace-builds/src-min-noconflict/keybinding-emacs';
-import { SettingsContext } from '../Settings';
-import { Tooltip } from './Tooltip';
-import { INPUT_SYNC_PERIOD } from './Input';
+
+export function skipWhitespaceBackward(it: Ace.TokenIterator) {
+  while (!it.getCurrentToken().value.trim()) {
+    if (!it.stepBackward()) {
+      return;
+    }
+  }
+}
+
+export function skipWhitespaceForward(it: Ace.TokenIterator) {
+  while (!it.getCurrentToken().value.trim()) {
+    if (!it.stepForward()) {
+      return;
+    }
+  }
+}
 
 export function CodeEditor({
   value,
@@ -30,6 +51,7 @@ export function CodeEditor({
   placeholder,
   disabled,
   onKeyDown,
+  autocomplete,
   language,
   id,
   singleLine,
@@ -42,6 +64,10 @@ export function CodeEditor({
   disabled?: boolean;
   onKeyDown?: (e: React.KeyboardEvent) => void;
   placeholder?: string;
+  autocomplete?: (
+    tokenIteratorFactory: () => Ace.TokenIterator,
+    prefix: string
+  ) => Array<Ace.Completion>;
   language: string;
   id: string;
   singleLine?: boolean;
@@ -52,7 +78,7 @@ export function CodeEditor({
     state: { theme },
   } = React.useContext(SettingsContext);
 
-  const [editorNode, editorRef] = React.useState<AceEditor>(null);
+  const [editorRef, setEditorRef] = React.useState<AceEditor>(null);
   const debounced = useDebouncedCallback(onChange, INPUT_SYNC_PERIOD);
   // Flush on unmount
   React.useEffect(
@@ -65,35 +91,70 @@ export function CodeEditor({
   // Make sure editor resizes if the overall panel changes size. For
   // example this happens when the preview height changes.
   React.useEffect(() => {
-    if (!editorNode) {
+    if (!editorRef) {
       return;
     }
 
-    const panel = editorNode.editor.container.closest('.panel');
+    const panel = editorRef.editor.container.closest('.panel');
     const obs = new ResizeObserver(function handleEditorResize() {
-      editorNode.editor?.resize();
+      editorRef.editor?.resize();
     });
     obs.observe(panel);
 
     return () => obs.disconnect();
-  }, [editorNode]);
+  }, [editorRef]);
 
   // Resync value when outer changes
   React.useEffect(() => {
-    if (!editorNode || value == editorNode.editor.getValue()) {
+    if (!editorRef || value == editorRef.editor.getValue()) {
       return;
     }
 
     // Without this the clearSelection call below moves the cursor to the end of the textarea destroying in-action edits
-    if (editorNode.editor.container.contains(document.activeElement)) {
+    if (editorRef.editor.container.contains(document.activeElement)) {
       return;
     }
 
-    editorNode.editor.setValue(value);
+    editorRef.editor.setValue(value);
     // setValue() also highlights the inserted values so this gets rid
     // of the highlight. Kind of a weird API really
-    editorNode.editor.clearSelection();
-  }, [value, editorNode]);
+    editorRef.editor.clearSelection();
+  }, [value, editorRef]);
+
+  React.useEffect(() => {
+    if (!autocomplete) {
+      return;
+    }
+
+    const { TokenIterator } = ace.require('ace/token_iterator');
+
+    const completer = {
+      getCompletions: (
+        editor: AceEditor,
+        session: Ace.EditSession,
+        pos: Ace.Point,
+        prefix: string,
+        callback: Ace.CompleterCallback
+      ) => {
+        // This gets registered globally which is kind of weird.  //
+        // So it needs to check again that the currently editing editor
+        // is the one attached to this callback.
+        if (!autocomplete || (editorRef.editor as unknown) !== editor) {
+          return callback(null, []);
+        }
+
+        try {
+          const factory = () => new TokenIterator(session, pos.row, pos.column);
+          return callback(null, autocomplete(factory, prefix));
+        } catch (e) {
+          log.error(e);
+          return callback(null, []);
+        }
+      },
+    };
+
+    langTools.setCompleters([completer]);
+  }, [autocomplete, editorRef]);
 
   return (
     <div
@@ -103,15 +164,14 @@ export function CodeEditor({
     >
       {label && <label className="label input-label">{label}</label>}
       <AceEditor
-        ref={editorRef}
+        ref={setEditorRef}
         mode={language}
         theme={theme === 'dark' ? 'dracula' : 'github'}
         maxLines={singleLine ? 1 : undefined}
         wrapEnabled={true}
-        onBlur={
-          () =>
-            debounced.flush() /* Simplifying this to onBlur={debounced.flush} doesn't work. */
-        }
+        onBlur={() => {
+          debounced.flush(); /* Simplifying this to onBlur={debounced.flush} doesn't work. */
+        }}
         name={id}
         defaultValue={String(value)}
         onChange={(v) => debounced(v)}
@@ -155,7 +215,11 @@ export function CodeEditor({
         setOptions={
           singleLine
             ? { showLineNumbers: false, highlightActiveLine: false }
-            : undefined
+            : {
+                enableBasicAutocompletion: Boolean(autocomplete),
+                enableLiveAutocompletion: Boolean(autocomplete),
+                enableSnippets: Boolean(autocomplete),
+              }
         }
       />
       {tooltip && <Tooltip>{tooltip}</Tooltip>}

ui/components/CodeEditor.tsx

@@ -67,9 +67,6 @@ export function ContentTypePicker({
             </option>
             <option value="text/apache2error">Apache2 Error Logs</option>
             <option value="text/nginxaccess">Nginx Access Logs</option>
-            <option value="application/jsonlines">
-              Newline-delimited JSON
-            </option>
             <option value="text/regexplines">Newline-delimited Regex</option>
           </optgroup>
         </Select>

ui/components/ContentTypePicker.tsx

@@ -1,3 +1,4 @@
+import { Ace } from 'ace-builds';
 import * as React from 'react';
 import { DOCS_ROOT } from '../../shared/constants';
 import { NoConnectorError } from '../../shared/errors';
@@ -6,6 +7,7 @@ import {
   ConnectorInfo,
   DatabaseConnectorInfo,
   DatabasePanelInfo,
+  PanelInfo,
   TimeSeriesRange as TimeSeriesRangeT,
 } from '../../shared/state';
 import { panelRPC } from '../asyncRPC';
@@ -16,6 +18,12 @@ import { ServerPicker } from '../components/ServerPicker';
 import { TimeSeriesRange } from '../components/TimeSeriesRange';
 import { VENDORS } from '../connectors';
 import { ProjectContext } from '../state';
+import {
+  builtinCompletions,
+  dotAccessPanelShapeCompletions,
+  panelNameCompletions,
+  stringPanelShapeCompletions,
+} from './ProgramPanel';
 import { PanelBodyProps, PanelDetailsProps, PanelUIDetails } from './types';
 
 export async function evalDatabasePanel(
@@ -217,11 +225,13 @@ export function DatabasePanelDetails({
 export function DatabasePanelBody({
   updatePanel,
   panel,
+  panels,
   keyboardShortcuts,
 }: PanelBodyProps<DatabasePanelInfo>) {
   return (
     <CodeEditor
-      id={panel.id}
+      autocomplete={makeAutocomplete(panels.filter((p) => p.id !== panel.id))}
+      id={'editor-' + panel.id}
       onKeyDown={keyboardShortcuts}
       value={panel.content}
       onChange={(value: string) => {
@@ -234,6 +244,21 @@ export function DatabasePanelBody({
   );
 }
 
+export function makeAutocomplete(panels: Array<PanelInfo>) {
+  return (tokenIteratorFactory: () => Ace.TokenIterator, prefix: string) => {
+    return [
+      ...builtinCompletions(tokenIteratorFactory).filter(
+        (c) => !c.value.startsWith('DM_setPanel')
+      ),
+      ...panelNameCompletions(tokenIteratorFactory, panels),
+      ...dotAccessPanelShapeCompletions(tokenIteratorFactory, panels),
+      ...stringPanelShapeCompletions(tokenIteratorFactory, panels),
+    ]
+      .flat()
+      .filter((c) => c && c.value.startsWith(prefix));
+  };
+}
+
 export function DatabaseInfo({ panel }: { panel: DatabasePanelInfo }) {
   const { connectors } = React.useContext(ProjectContext).state;
   const connector = connectors.find(

ui/panels/DatabasePanel.tsx

@@ -137,7 +137,7 @@ export function FilterAggregatePanelDetails({
         <div className="form-row">
           <CodeEditor
             singleLine
-            id={panel.id + 'filter'}
+            id={'filter-' + panel.id}
             label="Filter"
             placeholder="x LIKE '%town%' AND y IN (1, 2)"
             value={panel.filagg.filter}