SearchKit - Add download CSV action

ext/search_kit/Civi/Api4/Action/SearchDisplay/AbstractRunAction.php

@@ -123,10 +123,11 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction {
     $formatted = [];
     foreach ($result as $data) {
       $row = [];
-      foreach ($data as $key => $raw) {
+      foreach ($select as $key => $item) {
+        $raw = $data[$key] ?? NULL;
         $row[$key] = [
           'raw' => $raw,
-          'view' => $this->formatViewValue($select[$key]['dataType'], $raw),
+          'view' => $this->formatViewValue($item['dataType'], $raw),
         ];
       }
       $formatted[] = $row;
@@ -318,6 +319,11 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction {
    * @param array $apiParams
    */
   protected function augmentSelectClause(&$apiParams): void {
+    $additions = [];
+    // Add primary key field if actions are enabled
+    if (!empty($this->display['settings']['actions'])) {
+      $additions = CoreUtil::getInfoItem($this->savedSearch['api_entity'], 'primary_key');
+    }
     $possibleTokens = '';
     foreach ($this->display['settings']['columns'] as $column) {
       // Collect display values in which a token is allowed
@@ -335,7 +341,7 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction {
     // Add fields referenced via token
     $tokens = [];
     preg_match_all('/\\[([^]]+)\\]/', $possibleTokens, $tokens);
-    $apiParams['select'] = array_unique(array_merge($apiParams['select'], $tokens[1]));
+    $apiParams['select'] = array_unique(array_merge($apiParams['select'], $additions, $tokens[1]));
   }
 
   /**

ext/search_kit/Civi/Api4/Action/SearchDisplay/Download.php

+<?php
+
+namespace Civi\Api4\Action\SearchDisplay;
+
+use League\Csv\Writer;
+
+/**
+ * Download the results of a SearchDisplay as a spreadsheet.
+ *
+ * Note: unlike other APIs this action directly outputs a file.
+ *
+ * @package Civi\Api4\Action\SearchDisplay
+ */
+class Download extends AbstractRunAction {
+
+  /**
+   * Requested file format
+   * @var string
+   * @required
+   * @options csv
+   */
+  protected $format;
+
+  /**
+   * @param \Civi\Api4\Generic\Result $result
+   * @throws \API_Exception
+   */
+  protected function processResult(\Civi\Api4\Generic\Result $result) {
+    $entityName = $this->savedSearch['api_entity'];
+    $apiParams =& $this->savedSearch['api_params'];
+    $settings = $this->display['settings'];
+
+    // Displays are only exportable if they have actions enabled
+    if (empty($settings['actions'])) {
+      \CRM_Utils_System::permissionDenied();
+    }
+
+    // Force limit if the display has no pager
+    if (!isset($settings['pager']) && !empty($settings['limit'])) {
+      $apiParams['limit'] = $settings['limit'];
+    }
+    $apiParams['orderBy'] = $this->getOrderByFromSort();
+    $this->augmentSelectClause($apiParams);
+
+    $this->applyFilters();
+
+    $apiResult = civicrm_api4($entityName, 'get', $apiParams);
+
+    $rows = $this->formatResult($apiResult);
+
+    $columns = [];
+    foreach ($this->display['settings']['columns'] as $col) {
+      $col += ['type' => NULL, 'label' => '', 'rewrite' => FALSE];
+      if ($col['type'] === 'field' && !empty($col['key'])) {
+        $columns[] = $col;
+      }
+    }
+
+    // This weird little API spits out a file and exits instead of returning a result
+    $fileName = \CRM_Utils_File::makeFilenameWithUnicode($this->display['label']) . '.' . $this->format;
+
+    switch ($this->format) {
+      case 'csv':
+        $this->outputCSV($rows, $columns, $fileName);
+        break;
+    }
+
+    \CRM_Utils_System::civiExit();
+  }
+
+  /**
+   * Outputs csv format directly to browser for download
+   * @param array $rows
+   * @param array $columns
+   * @param string $fileName
+   */
+  private function outputCSV(array $rows, array $columns, string $fileName) {
+    $csv = Writer::createFromFileObject(new \SplTempFileObject());
+    $csv->setOutputBOM(Writer::BOM_UTF8);
+
+    // Header row
+    $csv->insertOne(array_column($columns, 'label'));
+
+    foreach ($rows as $data) {
+      $row = [];
+      foreach ($columns as $col) {
+        $row[] = $this->formatColumnValue($col, $data);
+      }
+      $csv->insertOne($row);
+    }
+    // Echo headers and content directly to browser
+    $csv->output($fileName);
+  }
+
+  /**
+   * Returns final formatted column value
+   *
+   * @param array $col
+   * @param array $data
+   * @return string
+   */
+  protected function formatColumnValue(array $col, array $data) {
+    $val = $col['rewrite'] ?: $data[$col['key']]['view'] ?? '';
+    if ($col['rewrite']) {
+      foreach ($data as $k => $v) {
+        $val = str_replace("[$k]", $v['view'], $val);
+      }
+    }
+    return is_array($val) ? implode(', ', $val) : $val;
+  }
+
+}

ext/search_kit/Civi/Api4/Action/SearchDisplay/GetSearchTasks.php

@@ -45,6 +45,15 @@ class GetSearchTasks extends \Civi\Api4\Generic\AbstractAction {
       ];
     }
 
+    $tasks[$entity['name']]['download'] = [
+      'module' => 'crmSearchTasks',
+      'title' => E::ts('Download Spreadsheet'),
+      'icon' => 'fa-download',
+      'uiDialog' => ['templateUrl' => '~/crmSearchTasks/crmSearchTaskDownload.html'],
+      // Does not require any rows to be selected
+      'number' => '>= 0',
+    ];
+
     if (array_key_exists('update', $entity['actions'])) {
       $tasks[$entity['name']]['update'] = [
         'module' => 'crmSearchTasks',
@@ -126,6 +135,8 @@ class GetSearchTasks extends \Civi\Api4\Generic\AbstractAction {
 
     foreach ($tasks[$entity['name']] as $name => &$task) {
       $task['name'] = $name;
+      // Add default for number of rows action requires
+      $task += ['number' => '> 0'];
     }
 
     $result->exchangeArray(array_values($tasks[$entity['name']]));

ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.component.js

@@ -72,6 +72,11 @@
           apiParams: JSON.stringify(ctrl.search.api_params, null, 2)
         };
         ctrl.settings = ctrl.display.settings;
+        setLabel();
+      }
+
+      function setLabel() {
+        ctrl.display.label = ctrl.search.label || searchMeta.getEntity(ctrl.search.api_entity).title_plural;
       }
 
       this.$onInit = function() {
@@ -79,6 +84,7 @@
         this.initializeDisplay($scope, $element);
         $scope.$watch('$ctrl.search.api_entity', buildSettings);
         $scope.$watch('$ctrl.search.api_params', buildSettings, true);
+        $scope.$watch('$ctrl.search.label', setLabel);
       };
 
       // Add callbacks for pre & post run

ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.html

@@ -2,7 +2,7 @@
   <div ng-include="'~/crmSearchAdmin/resultsTable/debug.html'"></div>
   <div class="form-inline">
     <div class="btn-group" ng-include="'~/crmSearchDisplay/SearchButton.html'"></div>
-    <crm-search-tasks entity="$ctrl.apiEntity" ids="$ctrl.selectedRows" refresh="$ctrl.refreshAfterTask()"></crm-search-tasks>
+    <crm-search-tasks entity="$ctrl.apiEntity" ids="$ctrl.selectedRows" search="$ctrl.search" display="$ctrl.display" display-controller="$ctrl" refresh="$ctrl.refreshAfterTask()"></crm-search-tasks>
   </div>
   <table>
     <thead>

ext/search_kit/ang/crmSearchDisplayTable/crmSearchDisplayTable.html

 <div class="crm-search-display crm-search-display-table">
   <div class="form-inline">
     <div class="btn-group" ng-include="'~/crmSearchDisplay/SearchButton.html'" ng-if="$ctrl.settings.button"></div>
-    <crm-search-tasks ng-if="$ctrl.settings.actions" entity="$ctrl.apiEntity" ids="$ctrl.selectedRows" refresh="$ctrl.refreshAfterTask()"></crm-search-tasks>
+    <crm-search-tasks ng-if="$ctrl.settings.actions" entity="$ctrl.apiEntity" ids="$ctrl.selectedRows" search="$ctrl.search" display="$ctrl.display" display-controller="$ctrl" refresh="$ctrl.refreshAfterTask()"></crm-search-tasks>
   </div>
   <table>
     <thead>

ext/search_kit/ang/crmSearchTasks/crmSearchTaskDownload.ctrl.js

+(function(angular, $, _) {
+  "use strict";
+
+  angular.module('crmSearchTasks').controller('crmSearchTaskDownload', function($scope, $http, searchTaskBaseTrait, $timeout, $interval) {
+    var ts = $scope.ts = CRM.ts('org.civicrm.search_kit'),
+      // Combine this controller with model properties (ids, entity, entityInfo) and searchTaskBaseTrait
+      ctrl = angular.extend(this, $scope.model, searchTaskBaseTrait);
+
+    this.entityTitle = this.getEntityTitle();
+    this.format = 'csv';
+    this.progress = null;
+
+    this.download = function() {
+      ctrl.progress = 0;
+      $('.ui-dialog-titlebar button').hide();
+      // Show the user something is happening (even though it doesn't accurately reflect progress)
+      var incrementer = $interval(function() {
+        if (ctrl.progress < 90) {
+          ctrl.progress += 10;
+        }
+      }, 1000);
+      var apiParams = ctrl.displayController.getApiParams();
+      delete apiParams.return;
+      delete apiParams.limit;
+      apiParams.filters.id = ctrl.ids || null;
+      apiParams.format = ctrl.format;
+      // Use AJAX to fetch file with arrayBuffer
+      var httpConfig = {
+        responseType: 'arraybuffer',
+        headers: {'X-Requested-With': 'XMLHttpRequest', 'Content-Type': 'application/x-www-form-urlencoded'}
+      };
+      $http.post(CRM.url('civicrm/ajax/api4/SearchDisplay/download'), $.param({
+        params: JSON.stringify(apiParams)
+      }), httpConfig)
+        .then(function(response) {
+          $interval.cancel(incrementer);
+          ctrl.progress = 100;
+          // Convert arrayBuffer response to blob
+          var blob = new Blob([response.data], {
+            type: response.headers('Content-Type')
+          }),
+            a = document.createElement("a"),
+            url = a.href = window.URL.createObjectURL(blob),
+            fileName = getFileNameFromHeader(response.headers('Content-Disposition'));
+          a.download = fileName;
+          // Trigger file download
+          a.click();
+          // Free browser memory
+          window.URL.revokeObjectURL(url);
+          $timeout(function() {
+            CRM.alert(ts('%1 has been downloaded to your computer.', {1: fileName}), ts('Download Complete'), 'success');
+            // This action does not update data so don't trigger a refresh
+            ctrl.cancel();
+          }, 1000);
+        });
+    };
+
+    // Parse and decode fileName from Content-Disposition header
+    function getFileNameFromHeader(contentDisposition) {
+      var utf8FilenameRegex = /filename\*=utf-8''([\w%\-\.]+)(?:; ?|$)/i,
+        asciiFilenameRegex = /filename=(["']?)(.*?[^\\])\1(?:; ?|$)/;
+
+      if (contentDisposition && contentDisposition.length) {
+        if (utf8FilenameRegex.test(contentDisposition)) {
+          return decodeURIComponent(utf8FilenameRegex.exec(contentDisposition)[1]);
+        } else {
+          var matches = asciiFilenameRegex.exec(contentDisposition);
+          if (matches != null && matches[2]) {
+            return matches[2];
+          }
+        }
+      }
+      // Fallback in case header could not be parsed
+      return ctrl.entityTitle + '.' + ctrl.format;
+    }
+
+  });
+})(angular, CRM.$, CRM._);

ext/search_kit/ang/crmSearchTasks/crmSearchTaskDownload.html

+<div id="bootstrap-theme">
+  <form ng-controller="crmSearchTaskDownload as $ctrl">
+    <p>
+      <strong ng-if="$ctrl.ids.length">{{:: ts('Download %1 %2', {1: $ctrl.ids.length, 2: $ctrl.entityTitle}) }}</strong>
+      <strong ng-if="!$ctrl.ids.length">{{:: ts('Download %1 %2', {1: $ctrl.displayController.rowCount, 2: $ctrl.entityTitle}) }}</strong>
+    </p>
+    <div class="form-inline">
+      <label for="crmSearchTaskDownload-format">{{:: ts('Format') }}</label>
+      <select id="crmSearchTaskDownload-format" class="form-control" ng-model="$ctrl.format">
+        <option value="csv">{{:: ts('CSV File') }}</option>
+      </select>
+    </div>
+    <hr />
+    <div ng-if="$ctrl.progress !== null" class="crm-search-task-progress">
+      <h5>{{:: ts('Downloading...') }}</h5>
+      <div class="progress">
+        <div class="progress-bar progress-bar-striped active" role="progressbar" ng-style="{width: '' + $ctrl.progress + '%'}"></div>
+      </div>
+    </div>
+    <hr />
+    <div class="buttons text-right">
+      <button type="button" ng-click="$ctrl.cancel()" class="btn btn-danger" ng-hide="$ctrl.run">
+        <i class="crm-i fa-times"></i>
+        {{:: ts('Cancel') }}
+      </button>
+      <button ng-click="$ctrl.download()" class="btn btn-primary" ng-disabled="$ctrl.run">
+        <i class="crm-i fa-{{ $ctrl.run ? 'spin fa-spinner' : 'download' }}"></i>
+        {{:: ts('Download') }}
+      </button>
+    </div>
+  </form>
+</div>

ext/search_kit/ang/crmSearchTasks/crmSearchTasks.component.js

@@ -5,6 +5,9 @@
     bindings: {
       entity: '<',
       refresh: '&',
+      search: '<',
+      display: '<',
+      displayController: '<',
       ids: '<'
     },
     templateUrl: '~/crmSearchTasks/crmSearchTasks.html',
@@ -15,14 +18,17 @@
         unwatchIDs = $scope.$watch('$ctrl.ids.length', watchIDs);
 
       function watchIDs() {
-        if (ctrl.ids && ctrl.ids.length && !initialized) {
+        if (ctrl.ids && ctrl.ids.length) {
           unwatchIDs();
-          initialized = true;
-          initialize();
+          ctrl.getTasks();
         }
       }
 
-      function initialize() {
+      this.getTasks = function() {
+        if (initialized) {
+          return;
+        }
+        initialized = true;
         crmApi4({
           entityInfo: ['Entity', 'get', {select: ['name', 'title', 'title_plural'], where: [['name', '=', ctrl.entity]]}, 0],
           tasks: ['SearchDisplay', 'getSearchTasks', {entity: ctrl.entity}]
@@ -30,19 +36,22 @@
           ctrl.entityInfo = result.entityInfo;
           ctrl.tasks = result.tasks;
         });
-      }
+      };
 
       this.isActionAllowed = function(action) {
-        return !action.number || $scope.eval('' + $ctrl.ids.length + action.number);
+        return $scope.$eval('' + ctrl.ids.length + action.number);
       };
 
       this.doAction = function(action) {
-        if (!ctrl.isActionAllowed(action) || !ctrl.ids.length) {
+        if (!ctrl.isActionAllowed(action)) {
           return;
         }
         var data = {
           ids: ctrl.ids,
           entity: ctrl.entity,
+          search: ctrl.search,
+          display: ctrl.display,
+          displayController: ctrl.displayController,
           entityInfo: ctrl.entityInfo
         };
         // If action uses a crmPopup form
@@ -59,7 +68,8 @@
             title: action.title
           });
           dialogService.open('crmSearchTask', action.uiDialog.templateUrl, data, options)
-            .then(ctrl.refresh);
+            // Reload results on success, do nothing on cancel
+            .then(ctrl.refresh, _.noop);
         }
       };
     }

ext/search_kit/ang/crmSearchTasks/crmSearchTasks.html

 <div class="btn-group" title="{{:: ts('Perform action on selected items.') }}">
-  <button type="button" ng-disabled="!$ctrl.ids.length" class="btn dropdown-toggle btn-default" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+  <button type="button" ng-disabled="$ctrl.displayController.loading || !$ctrl.displayController.results.length" ng-click="$ctrl.getTasks()" class="btn dropdown-toggle btn-default" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
     <i class="crm-i fa-pencil"></i>
     {{:: ts('Action') }} <span class="caret"></span>
   </button>
   <ul class="dropdown-menu">
-    <li ng-disabled="!$ctrl.isActionAllowed(action)" ng-repeat="action in $ctrl.tasks">
+    <li ng-class="{disabled: !$ctrl.isActionAllowed(action)}" ng-repeat="action in $ctrl.tasks">
       <a href ng-click="$ctrl.doAction(action)"><i class="fa {{:: action.icon }}"></i> {{:: action.title }}</a>
     </li>
     <li class="disabled" ng-if="!$ctrl.tasks">

ext/search_kit/tests/phpunit/api/v4/SearchDisplay/SearchDownloadTest.php

+<?php
+namespace api\v4\SearchDisplay;
+
+use Civi\Api4\Contact;
+use Civi\Test\HeadlessInterface;
+use Civi\Test\TransactionalInterface;
+
+/**
+ * @group headless
+ */
+class SearchDownloadTest extends \PHPUnit\Framework\TestCase implements HeadlessInterface, TransactionalInterface {
+
+  public function setUpHeadless() {
+    // Civi\Test has many helpers, like install(), uninstall(), sql(), and sqlFile().
+    // See: https://docs.civicrm.org/dev/en/latest/testing/phpunit/#civitest
+    return \Civi\Test::headless()
+      ->installMe(__DIR__)
+      ->apply();
+  }
+
+  /**
+   * Test downloading CSV format.
+   *
+   * Must run in separate process to capture direct output to browser
+   *
+   * @runInSeparateProcess
+   * @preserveGlobalState disabled
+   */
+  public function testDownloadCSV() {
+    $this->markTestIncomplete('Unable to get this test working in separate process, probably due to being in an extension');
+
+    // Re-enable because this test has to run in a separate process
+    \CRM_Extension_System::singleton()->getManager()->install('org.civicrm.search_kit');
+
+    $lastName = uniqid(__FUNCTION__);
+    $sampleData = [
+      ['first_name' => 'One', 'last_name' => $lastName],
+      ['first_name' => 'Two', 'last_name' => $lastName],
+      ['first_name' => 'Three', 'last_name' => $lastName],
+      ['first_name' => 'Four', 'last_name' => $lastName],
+    ];
+    Contact::save(FALSE)->setRecords($sampleData)->execute();
+
+    $params = [
+      'checkPermissions' => FALSE,
+      'format' => 'csv',
+      'savedSearch' => [
+        'api_entity' => 'Contact',
+        'api_params' => [
+          'version' => 4,
+          'select' => ['last_name'],
+          'where' => [],
+        ],
+      ],
+      'display' => [
+        'type' => 'table',
+        'label' => '',
+        'settings' => [
+          'limit' => 2,
+          'actions' => TRUE,
+          'pager' => [],
+          'columns' => [
+            [
+              'key' => 'last_name',
+              'label' => 'First Last',
+              'dataType' => 'String',
+              'type' => 'field',
+              'rewrite' => '[first_name] [last_name]',
+            ],
+          ],
+          'sort' => [
+            ['id', 'ASC'],
+          ],
+        ],
+      ],
+      'filters' => ['last_name' => $lastName],
+      'afform' => NULL,
+    ];
+
+    // UTF-8 BOM
+    $expectedOut = preg_quote("\xEF\xBB\xBF");
+    $expectedOut .= preg_quote('"First Last"');
+    foreach ($sampleData as $row) {
+      $expectedOut .= '\s+' . preg_quote('"' . $row['first_name'] . ' ' . $lastName . '"');
+    }
+    $this->expectOutputRegex('#' . $expectedOut . '#');
+
+    try {
+      civicrm_api4('SearchDisplay', 'download', $params);
+      $this->fail();
+    }
+    catch (\CRM_Core_Exception_PrematureExitException $e) {
+      // All good, we expected the api to exit
+    }
+  }
+
+}