[draft] enable "all custom fields"/"all fields" wildcards in SearchKit

Civi/Api4/Generic/Traits/SavedSearchInspectorTrait.php

@@ -181,6 +181,8 @@ private function getEntityFields() {
   public function getSelectClause() {
     if (!isset($this->_selectClause)) {
       $this->_selectClause = [];
+
+      $this->expandSelectParamWildcards();
       foreach ($this->_apiParams['select'] as $selectExpr) {
         $expr = SqlExpression::convert($selectExpr, TRUE);
         $item = [
@@ -487,4 +489,53 @@ protected function getJoinLabel($joinAlias) {
     return $this->_joinMap[$joinAlias];
   }
 
+  protected function expandSelectParamWildcards(): void {
+    $select = $this->_apiParams['select'];
+    $expanded = [];
+
+    foreach ($select as $expr) {
+      if (!\str_contains($expr, '*')) {
+        $expanded[] = $expr;
+        continue;
+      }
+
+      $parts = explode('.', $expr, 2);
+      $prefix = count($parts) > 1 ? $parts[0] : NULL;
+      $wildcard = count($parts) > 1 ? $parts[1] : $parts[0];
+
+      $entity = $prefix ? $this->getJoinEntity($prefix) : $this->savedSearch['api_entity'];
+
+      $fieldTypes = match ($wildcard) {
+        '*' => ['Field'],
+        'custom_*' => ['Custom'],
+      };
+      $getFields = civicrm_api4($entity, 'getFields', [
+        'checkPermissions' => FALSE,
+        'where' => [['type', 'IN', $fieldTypes]],
+        'select' => ['name'],
+      ])->column('name');
+      foreach ($getFields as $field) {
+        if ($prefix) {
+          $field = $prefix . '.' . $field;
+        }
+        if (!in_array($field, $expanded)) {
+          $expanded[] = $field;
+        }
+      }
+    }
+
+    // replace property
+    $this->_apiParams['select'] = $expanded;
+  }
+
+  private function getJoinEntity(string $alias): string {
+    foreach ($this->savedSearch['api_params']['join'] ?? [] as $join) {
+      [$entityName, $joinAlias] = explode(' AS ', $join[0]);
+      if ($joinAlias === $alias) {
+        return $entityName;
+      }
+    }
+    throw new \CRM_Core_Exception("Could not determine join entity for {$alias}");
+  }
+
 }

ext/search_kit/Civi/Api4/Action/SearchDisplay/AbstractRunAction.php

@@ -5,6 +5,7 @@
 use Civi\API\Exception\UnauthorizedException;
 use Civi\API\Request;
 use Civi\Api4\Query\SqlField;
+use Civi\Api4\SavedSearch;
 use Civi\Api4\SearchDisplay;
 use Civi\Api4\Utils\CoreUtil;
 use Civi\Api4\Utils\FormattingUtil;

ext/search_kit/Civi/Api4/Action/SearchDisplay/Run.php

@@ -53,6 +53,7 @@ protected function processResult(SearchDisplayRunResult $result) {
     $pagerMode = 'page';
 
     $this->preprocessLinks();
+    $this->expandSelectParamWildcards();
     $this->augmentSelectClause($apiParams);
     $this->applyFilters();
 

ext/search_kit/ang/crmSearchAdmin.module.js

@@ -354,6 +354,38 @@
           }
         }
       }
+      const parseWildcardArgs = (expr) => {
+        // note: valid wildcards are *,custom_*,[join_alias].*,[join_alias].custom_*
+        const parts = expr.split('.', 2);
+        const prefix = parts.length > 1 ? parts[0] : null;
+        const wildcard = parts.length > 1 ? parts[1] : parts[0];
+
+        let label = null;
+        let fieldTypes = null;
+        switch (wildcard) {
+          case '*':
+            label = ts('All Fields');
+            fieldTypes = ['Field', 'Custom'];
+            break;
+
+          case 'custom_*':
+            label = ts('All Custom Fields');
+            fieldTypes = ['Custom'];
+            break;
+
+          default:
+            throw new Error('Unrecognised wildcard in select expression');
+        }
+
+        const arg = {
+          wildcardFieldTypes: fieldTypes,
+          value: label,
+        };
+        if (prefix) {
+          arg.join = {alias: prefix};
+        }
+        return [arg];
+      }
       function parseExpr(expr) {
         if (!expr) {
           return;
@@ -364,6 +396,10 @@
           parseFnArgs(info, splitAs[0]);
           return info;
         }
+        if (expr.includes('*')) {
+          info.args = parseWildcardArgs(expr);
+          return info;
+        }
         const arg = parseArg(splitAs[0]);
         if (arg) {
           arg.param = 0;

ext/search_kit/ang/crmSearchAdmin/crmSearch-conditions.html

@@ -2,7 +2,7 @@
   <crm-search-clause clauses="$ctrl.savedSearch.api_params.where" format="string" op="AND" label="{{:: ts('Where') }}" fields="fieldsForWhere" allow-functions="$ctrl.paramExists('groupBy')" ></crm-search-clause>
 </fieldset>
 <fieldset ng-if="$ctrl.paramExists('having')" class="api4-clause-fieldset crm-search-havings">
-  <crm-search-clause clauses="$ctrl.savedSearch.api_params.having" format="string" op="AND" label="{{:: ts('Having') }}" help="having" fields="fieldsForHaving" aliases="$ctrl.savedSearch.api_params.select" ></crm-search-clause>
+  <crm-search-clause clauses="$ctrl.savedSearch.api_params.having" format="string" op="AND" label="{{:: ts('Having') }}" help="having" fields="fieldsForHaving" aliases="$ctrl.getExpandedSelect()" ></crm-search-clause>
 </fieldset>
 
 

ext/search_kit/ang/crmSearchAdmin/crmSearch-group.html

@@ -6,7 +6,7 @@
   <label for="crm-search-admin-group-title">{{:: ts('Group Title') }} <span class="crm-marker">*</span></label>
   <input id="crm-search-admin-group-title" class="form-control" placeholder="{{:: ts('Untitled') }}" ng-model="$ctrl.savedSearch.groups[0].title" ng-disabled="!smartGroupColumns.length" ng-required="smartGroupColumns.length">
   <label for="api-save-search-select-column">{{:: ts('Contact Column') }}</label>
-  <input id="api-save-search-select-column" ng-model="$ctrl.savedSearch.api_params.select[0]" class="form-control" crm-ui-select="{data: smartGroupColumns}"/>
+  <input id="api-save-search-select-column" ng-model="$ctrl.getExpandedSelect()[0]" class="form-control" crm-ui-select="{data: smartGroupColumns}"/>
 </div>
 <fieldset ng-show="smartGroupColumns.length">
   <label>{{:: getField('description', 'Group').label }}</label>

ext/search_kit/ang/crmSearchAdmin/crmSearchAdmin.component.js

@@ -717,6 +717,26 @@
         }
 
         result.push(...getFieldOptionsForFields(searchMeta.getEntity(entityName).fields, prefix));
+
+        // if there are any fields for this entity, add "All Fields" psuedofield
+        if (result.length && allowedTypes.includes('Extra')) {
+          const wildcards = [
+            {
+              id: prefix + '*',
+              text: ts('All %1 Fields', {1: searchMeta.getEntity(entityName).title}),
+            },
+            {
+              id: prefix + 'custom_*',
+              text: ts('All Custom Fields for %1', {1: searchMeta.getEntity(entityName).title}),
+            }
+          ];
+
+          wildcards.forEach((wildcard) => {
+            wildcard.disabled = disabledIf(wildcard.id);
+            result.push(wildcard);
+          });
+        }
+
         return result;
       };
 
@@ -741,7 +761,8 @@
       result.push({
         text: mainEntity.title_plural,
         icon: mainEntity.icon,
-        children: getFieldOptionsForEntity(ctrl.savedSearch.api_entity)
+        children: getFieldOptionsForEntity(ctrl.savedSearch.api_entity),
+        alias: '__sk_main_entity__',
       });
 
       // Include SearchKit's pseudo-fields if specifically requested
@@ -765,7 +786,7 @@
 
     this.getSelectFields = (disabledIf) => {
       disabledIf = disabledIf || (() => false);
-      return ctrl.savedSearch.api_params.select.map((fieldExpr) => {
+      return this.getExpandedSelect().map((fieldExpr) => {
         const info = searchMeta.parseExpr(fieldExpr);
         return {
           id: info.alias,
@@ -801,6 +822,51 @@
       searchMeta.loadFieldOptions(entitiesToLoad);
     }
 
+    /**
+     * Get the default columns for a saved_search
+     *
+     * TODO: if https://github.com/civicrm/civicrm-core/pull/34178 is merged then
+     * we dont need to reimplement this client side
+     * @param Object search
+     * @returns Object[]
+     */
+    this.getDefaultSearchColumns = () => {
+      const keys = this.getExpandedSelect();
+      const columns = keys.map((key) => searchMeta.fieldToColumn(key, {label: true, sortable: true}));
+      // add the defaultDisplay columns (= menu)
+      columns.push(...CRM.crmSearchAdmin.defaultDisplay.settings.columns);
+      return columns;
+    };
+
+    this.getExpandedSelect = () => {
+      const select = this.savedSearch.api_params.select;
+      const expanded = [];
+
+      select.forEach((selectExpr) => {
+        // simple field, just add the single field
+        if (!selectExpr.includes('*')) {
+          expanded.push(selectExpr);
+          return;
+        }
+
+        const info = searchMeta.parseExpr(selectExpr);
+        const arg = info.args[0];
+        const fields = this.getAllFields(':label', arg.wildcardFieldTypes);
+        const fieldGroupId = arg.join ? arg.join.alias : '__sk_main_entity__';
+
+        // use the prefix to get fields for the right entity
+        const entityFields = fields.find((group) => group.id === fieldGroupId).children;
+        const keys = entityFields.map((f) => f.id)
+          // filter explicitly excluded fields
+          // (this handily also filters the wildcard itself)
+          .filter((id) => !select.includes(id));
+
+        expanded.push(...keys);
+      });
+
+      return expanded;
+    }
+
     // Build a list of all possible links to main entity & join entities
     // @return {Array}
     this.buildLinks = function(isRow) {

ext/search_kit/ang/crmSearchAdmin/crmSearchAdminDisplay.component.js

@@ -142,7 +142,7 @@
 
       this.getExprFromSelect = function(key) {
         let fieldKey = key.split(':')[0];
-        let match = ctrl.savedSearch.api_params.select.find((expr) => {
+        let match = ctrl.crmSearchAdmin.getExpandedSelect().find((expr) => {
           let parts = expr.split(' AS ');
           return (parts[1] === fieldKey || parts[0].split(':')[0] === fieldKey);
         });

ext/search_kit/ang/crmSearchAdmin/crmSearchAdminFields.component.js

@@ -30,7 +30,7 @@
             ctrl.select[index] = {
               key: key,
               label: ctrl.crmSearchAdmin.getFieldLabel(key),
-              isPseudoField: ctrl.crmSearchAdmin.isPseudoField(key),
+              isPseudoField: (key.includes('*') || ctrl.crmSearchAdmin.isPseudoField(key)),
               rawKey: key.split(':')[0],
               suffixOptions: ctrl.crmSearchAdmin.getSuffixOptions(key),
             };

ext/search_kit/ang/crmSearchAdmin/displays/common/addColMenu.html

@@ -4,7 +4,7 @@
     {{:: ts('Add/Remove Fields') }} <span class="caret"></span>
   </button>
   <ul class="dropdown-menu">
-    <li ng-repeat="item in $ctrl.parent.savedSearch.api_params.select track by $index">
+    <li ng-repeat="item in $ctrl.parent.crmSearchAdmin.getExpandedSelect() track by $index">
       <a href ng-click="$ctrl.parent.toggleColumn(item); $event.stopPropagation()">
         <i class="crm-i fa-{{ $ctrl.parent.columnExists(item) ? 'check-' : '' }}square-o" role="img" aria-hidden="true"></i>
         {{ $ctrl.parent.getFieldLabel(item) }}

ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.component.js

@@ -20,10 +20,10 @@
         ctrl.apiEntity = ctrl.search.api_entity;
         ctrl.settings = _.cloneDeep(CRM.crmSearchAdmin.defaultDisplay.settings);
         ctrl.settings.button = ts('Search');
-        // The default-display settings contain just one column (the last one, with the links menu)
-        ctrl.settings.columns = _.transform(ctrl.search.api_params.select, function(columns, fieldExpr) {
-          columns.push(searchMeta.fieldToColumn(fieldExpr, {label: true, sortable: true}));
-        }).concat(ctrl.settings.columns);
+        // TODO if https://github.com/civicrm/civicrm-core/pull/34178 is merged
+        // we can just set ctrl.settings.column_mode = 'auto' and the columns
+        // will be autoloaded
+        ctrl.settings.columns = ctrl.crmSearchAdmin.getDefaultSearchColumns(ctrl.search);
         ctrl.columns = _.cloneDeep(ctrl.settings.columns);
         ctrl.columns.forEach((col) => {
           col.enabled = true;
@@ -79,9 +79,19 @@
         ctrl.crmSearchAdmin.clearParam('select', index);
       };
 
-      $scope.getColumnLabel = function(index) {
-        return searchMeta.getDefaultLabel(ctrl.search.api_params.select[index], ctrl.search);
-      };
+      this.getColumnDecoration = (index) => {
+        if (this.crmSearchAdmin.groupExists && !index) {
+          return 'locked';
+        }
+        const key = this.columns[index].key;
+        if (key && !this.search.api_params.select.includes(key)) {
+          return 'wildcard';
+        }
+        else if (key) {
+          // explicitly included field, can be removed
+          return 'removable';
+        }
+      }
 
     }
   });

ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.html

@@ -9,12 +9,13 @@
       <tr ng-model="$ctrl.search.api_params.select" ui-sortable="sortableColumnOptions">
         <th class="crm-search-result-select" ng-include="'~/crmSearchDisplayTable/crmSearchDisplayTaskHeader.html'" ng-if=":: $ctrl.settings.actions">
         </th>
-        <th ng-repeat="item in $ctrl.search.api_params.select" ng-click="$ctrl.setSort($ctrl.columns[$index], $event)" title="{{$index || !$ctrl.crmSearchAdmin.groupExists ? ts('Drag to reorder columns, click to sort results (shift-click to sort by multiple).') : ts('Column reserved for smart group.')}}">
+        <th ng-repeat="col in $ctrl.columns" ng-click="$ctrl.setSort($ctrl.columns[$index], $event)" title="{{$index || !$ctrl.crmSearchAdmin.groupExists ? ts('Drag to reorder columns, click to sort results (shift-click to sort by multiple).') : ts('Column reserved for smart group.')}}">
           <i ng-if=":: $ctrl.isSortable($ctrl.columns[$index])" class="crm-i {{ $ctrl.getSort($ctrl.columns[$index]) }}" role="img" aria-hidden="true"></i>
-          <span ng-class="{'crm-draggable': $index || !$ctrl.crmSearchAdmin.groupExists}">{{ getColumnLabel($index) }}</span>
-          <span ng-switch="$index || !$ctrl.crmSearchAdmin.groupExists ? 'sortable' : 'locked'">
+          <span ng-class="{'crm-draggable': $index || !$ctrl.crmSearchAdmin.groupExists}">{{ col.label }}</span>
+          <span ng-switch="$ctrl.getColumnDecoration($index)">
             <i ng-switch-when="locked" class="crm-i fa-lock" role="img" aria-hidden="true"></i>
-            <a href ng-switch-default class="crm-hover-button" title="{{:: ts('Clear') }}" ng-click="removeColumn($index); $event.stopPropagation();"><i class="crm-i fa-times" role="img" aria-hidden="true"></i></a>
+            <button ng-switch-when="removable" class="crm-hover-button" title="{{:: ts('Clear') }}" ng-click="removeColumn($index)"><i class="crm-i fa-times" role="img" aria-hidden="true"></i></button>
+            <i ng-switch-when="wildcard" class="crm-i fa-bolt" role="img" title="{{:: ts('Included by wildcard') }}"></i>
           </span>
         </th>
         <th class="form-inline text-right">