Added aggregation functions as a field

CHANGELOG.md

@@ -13,6 +13,7 @@
 * Added filter for searching contacts with a certain type.
 * Added filter to respect the ACL. So that a user only sees the contacts he is allowed to see.
 * Removed the title attribute from the outputs as those don't make sense.
+* Added fields with aggregation.
 
 # Version 1.0.7
 

CRM/Dataprocessor/Utils/Cache.php

@@ -53,7 +53,11 @@ class CRM_Dataprocessor_Utils_Cache {
    * @return mixed
    */
   public function get($key, $default=NULL) {
-    return $this->cache->get($key, $default);
+    $environment = CRM_Core_BAO_Setting::getItem('', 'environment');
+    if ($environment == 'Production') {
+      return $this->cache->get($key, $default);
+    }
+    return $default;
   }
 
   /**
@@ -67,4 +71,4 @@ class CRM_Dataprocessor_Utils_Cache {
     return $this->cache->set($key, $value, $ttl);
   }
 
-}
+}
\ No newline at end of file

Civi/DataProcessor/DataFlow/CombinedDataFlow/CombinedSqlDataFlow.php

@@ -133,6 +133,9 @@ class CombinedSqlDataFlow extends SqlDataFlow implements MultipleSourceDataFlows
    */
   public function getFieldsForGroupByStatement() {
     $fields = array();
+    foreach($this->aggregateFields as $field) {
+      $fields[] = "`{$this->primary_table_alias}`.`{$field->name}`";
+    }
     foreach($this->sourceDataFlowDescriptions as $sourceDataFlowDescription) {
       $fields = array_merge($fields, $sourceDataFlowDescription->getDataFlow()->getFieldsForGroupByStatement());
     }
@@ -235,4 +238,4 @@ class CombinedSqlDataFlow extends SqlDataFlow implements MultipleSourceDataFlows
   }
 
 
-}
+}
\ No newline at end of file

Civi/DataProcessor/DataFlow/SqlDataFlow.php

@@ -284,4 +284,4 @@ abstract class SqlDataFlow extends AbstractDataFlow {
     return $this->dao;
   }
 
-}
+}
\ No newline at end of file

Civi/DataProcessor/DataSpecification/AggregateFunctionFieldSpecification.php

+<?php
+/**
+ * @author Jaap Jansma <jaap.jansma@civicoop.org>
+ * @license AGPL-3.0
+ */
+
+namespace Civi\DataProcessor\DataSpecification;
+
+use Civi\DataProcessor\Exception\InvalidConfigurationException;
+use CRM_Dataprocessor_ExtensionUtil as E;
+
+/**
+ * Class used to add aggregate functions (such as SUM) to a database query.
+ *
+ * @package Civi\DataProcessor\DataSpecification
+ */
+class AggregateFunctionFieldSpecification extends FieldSpecification {
+
+  protected $function;
+
+  public function setAggregateFunction($function) {
+    $functions = self::functionList();
+    if (!isset($functions[$function])) {
+      throw new InvalidConfigurationException(E::ts('Field %1 has an invalid aggregate function.', [1=>$this->title]));
+    }
+    $this->function = $function;
+  }
+
+  /**
+   * Convert a fieldSpecification into the aggregate function field specification.
+   *
+   * @param \Civi\DataProcessor\DataSpecification\FieldSpecification $fieldSpecification
+   * @param $function
+   *
+   * @return \Civi\DataProcessor\DataSpecification\AggregateFunctionFieldSpecification
+   */
+  public static function convertFromFieldSpecification(FieldSpecification $fieldSpecification, $function) {
+    $return = new AggregateFunctionFieldSpecification($fieldSpecification->name, $fieldSpecification->type, $fieldSpecification->title, $fieldSpecification->options, $fieldSpecification->alias);
+    $return->setAggregateFunction($function);
+    return $return;
+  }
+
+  /**
+   * Returns the select statement for this field.
+   * E.g. COUNT(civicrm_contact.id) AS contact_id_count
+   *
+   * @param String $table_alias
+   * @return string
+   */
+  public function getSqlSelectStatement($table_alias) {
+    $function_arg = "";
+    $function = $this->function;
+    if ($function == 'COUNT_DISTINCT') {
+      $function = 'COUNT';
+      $function_arg = 'DISTINCT';
+    }
+    return "{$function}({$function_arg}`{$table_alias}`.`{$this->name}`) AS `{$this->alias}`";
+  }
+
+  /**
+   * Returns the valid aggregate functions
+   *
+   * @return array
+   */
+  public static function functionList() {
+    return [
+      'SUM' => E::ts('Sum'),
+      'AVG' => E::ts('Average'),
+      'MIN' => E::ts('Minimum'),
+      'MAX' => E::ts('Maximum'),
+      'STDDEV_POP' => E::ts('Standard deviation'),
+      'STDDEV_SAMP' => E::ts('Sample standard deviation'),
+      'VAR_POP' => E::ts('Standard variance'),
+      'VAR_SAMP' => E::ts('Sample variance'),
+      'COUNT' => E::ts('Count'),
+      'COUNT_DISTINCT' => E::ts('Distinct count')
+    ];
+  }
+
+}

Civi/DataProcessor/DataSpecification/AggregationField.php

@@ -25,4 +25,6 @@ class AggregationField {
     $this->fieldSpecification = $fieldSpecification;
   }
 
-}
+
\ No newline at end of file
+
+}

Civi/DataProcessor/DataSpecification/FieldSpecification.php

@@ -71,4 +71,4 @@ class FieldSpecification implements SqlFieldSpecification {
     return "`{$table_alias}`.`{$this->name}`";
   }
 
-}
+}
\ No newline at end of file

Civi/DataProcessor/DataSpecification/SqlFieldSpecification.php

@@ -17,4 +17,4 @@ interface SqlFieldSpecification {
    */
   public function getSqlSelectStatement($table_alias);
 
-}
+}
\ No newline at end of file

Civi/DataProcessor/Factory.php

@@ -147,8 +147,9 @@ class Factory {
     $this->addOutputHandler('groups_of_contact', 'Civi\DataProcessor\FieldOutputHandler\GroupsOfContactFieldOutputHandler', E::ts('Display the groups of a contact'));
     $this->addOutputHandler('event_repeating_info', 'Civi\DataProcessor\FieldOutputHandler\EventRepeatingInfoFieldOutputHandler', E::ts('Display info about repeating event'));
     $this->addOutputHandler('event_participants', 'Civi\DataProcessor\FieldOutputHandler\EventParticipantsFieldOutputHandler', E::ts('List participants'));
-    $this->addOutputHandler('calculations_substract', 'Civi\DataProcessor\FieldOutputHandler\Calculations\SubtractFieldOutputHandler', E::ts('Calculation: Subtract'));
-    $this->addOutputHandler('calculations_total', 'Civi\DataProcessor\FieldOutputHandler\Calculations\TotalFieldOutputHandler', E::ts('Calculation: Total'));
+    $this->addOutputHandler('calculations_substract', 'Civi\DataProcessor\FieldOutputHandler\Calculations\SubtractFieldOutputHandler', E::ts('Calculations (on multiple fields): Subtract'));
+    $this->addOutputHandler('calculations_total', 'Civi\DataProcessor\FieldOutputHandler\Calculations\TotalFieldOutputHandler', E::ts('Calculations (on multiple fields): Adding up'));
+    $this->addOutputHandler('aggregation_function', 'Civi\DataProcessor\FieldOutputHandler\AggregateFunctionFieldOutputHandler', E::ts('Aggregation function'));
   }
 
   /**

Civi/DataProcessor/FieldOutputHandler/AggregateFunctionFieldOutputHandler.php

+<?php
+/**
+ * @author Jaap Jansma <jaap.jansma@civicoop.org>
+ * @license AGPL-3.0
+ */
+
+namespace Civi\DataProcessor\FieldOutputHandler;
+
+use Civi\DataProcessor\DataSpecification\AggregateFunctionFieldSpecification;
+use Civi\DataProcessor\DataSpecification\FieldSpecification;
+use Civi\DataProcessor\Exception\DataSourceNotFoundException;
+use Civi\DataProcessor\Exception\FieldNotFoundException;
+use Civi\DataProcessor\FieldOutputHandler\FieldOutput;
+use Civi\DataProcessor\FieldOutputHandler\RawFieldOutputHandler;
+use CRM_Dataprocessor_ExtensionUtil as E;
+
+class AggregateFunctionFieldOutputHandler extends RawFieldOutputHandler {
+
+  /**
+   * @var \Civi\DataProcessor\DataSpecification\FieldSpecification
+   */
+  protected $aggregateField;
+
+  protected $prefix = '';
+
+  protected $suffix = '';
+
+  protected $number_of_decimals = '';
+
+  protected $decimal_sep = '';
+
+  protected $thousand_sep = '';
+
+  /**
+   * @return \Civi\DataProcessor\DataSpecification\FieldSpecification
+   */
+  public function getSortableInputFieldSpec() {
+    return $this->aggregateField;
+  }
+
+  /**
+   * Returns the data type of this field
+   *
+   * @return String
+   */
+  protected function getType() {
+    return 'Float';
+  }
+
+  /**
+   * Returns the formatted value
+   *
+   * @param $rawRecord
+   * @param $formattedRecord
+   *
+   * @return \Civi\DataProcessor\FieldOutputHandler\FieldOutput
+   */
+  public function formatField($rawRecord, $formattedRecord) {
+    $value = $rawRecord[$this->aggregateField->alias];
+
+    $formattedValue = $value;
+    if (is_numeric($this->number_of_decimals) && $value != null) {
+      $formattedValue = number_format($value, $this->number_of_decimals, $this->decimal_sep, $this->thousand_sep);
+    }
+    if ($formattedValue != null) {
+      $formattedValue = $this->prefix . $formattedValue . $this->suffix;
+    }
+
+    $output = new FieldOutput($rawRecord[$this->aggregateField->alias]);
+    $output->formattedValue = $formattedValue;
+    return $output;
+  }
+
+  /**
+   * Initialize the processor
+   *
+   * @param String $alias
+   * @param String $title
+   * @param array $configuration
+   * @param \Civi\DataProcessor\ProcessorType\AbstractProcessorType $processorType
+   */
+  public function initialize($alias, $title, $configuration) {
+    $this->dataSource = $this->dataProcessor->getDataSourceByName($configuration['datasource']);
+    if (!$this->dataSource) {
+      throw new DataSourceNotFoundException(E::ts("Field %1 requires data source '%2' which could not be found. Did you rename or deleted the data source?", array(1=>$title, 2=>$configuration['datasource'])));
+    }
+    $this->inputFieldSpec = $this->dataSource->getAvailableFields()->getFieldSpecificationByName($configuration['field']);
+    if (!$this->inputFieldSpec) {
+      throw new FieldNotFoundException(E::ts("Field %1 requires a field with the name '%2' in the data source '%3'. Did you change the data source type?", array(
+        1 => $title,
+        2 => $configuration['field'],
+        3 => $configuration['datasource']
+      )));
+    }
+    $this->aggregateField = AggregateFunctionFieldSpecification::convertFromFieldSpecification($this->inputFieldSpec, $configuration['function']);
+    $this->aggregateField->alias = $alias;
+    $this->dataSource->ensureFieldInSource($this->aggregateField);
+
+    $this->outputFieldSpec = clone $this->inputFieldSpec;
+    $this->outputFieldSpec->alias = $alias;
+    $this->outputFieldSpec->title = $title;
+    $this->outputFieldSpec->type = 'Float';
+
+    if (isset($configuration['number_of_decimals'])) {
+      $this->number_of_decimals = $configuration['number_of_decimals'];
+    }
+    if (isset($configuration['decimal_separator'])) {
+      $this->decimal_sep = $configuration['decimal_separator'];
+    }
+    if (isset($configuration['thousand_separator'])) {
+      $this->thousand_sep = $configuration['thousand_separator'];
+    }
+    if (isset($configuration['prefix'])) {
+      $this->prefix = $configuration['prefix'];
+    }
+    if (isset($configuration['suffix'])) {
+      $this->suffix = $configuration['suffix'];
+    }
+  }
+
+  /**
+   * When this handler has additional configuration you can add
+   * the fields on the form with this function.
+   *
+   * @param \CRM_Core_Form $form
+   * @param array $field
+   */
+  public function buildConfigurationForm(\CRM_Core_Form $form, $field=array()) {
+    $fieldSelect = $this->getFieldOptions($field['data_processor_id']);
+
+    $form->add('select', 'field', E::ts('Field'), $fieldSelect, true, array(
+      'style' => 'min-width:250px',
+      'class' => 'crm-select2 huge data-processor-field-for-name',
+      'placeholder' => E::ts('- select -'),
+    ));
+    $form->add('select', 'function', E::ts('Function'), AggregateFunctionFieldSpecification::functionList(), true, array(
+      'style' => 'min-width:250px',
+      'class' => 'crm-select2 huge',
+      'placeholder' => E::ts('- select -'),
+    ));
+
+    $form->add('text', 'number_of_decimals', E::ts('Number of decimals'), false);
+    $form->add('text', 'decimal_separator', E::ts('Decimal separator'), false);
+    $form->add('text', 'thousand_separator', E::ts('Thousand separator'), false);
+    $form->add('text', 'prefix', E::ts('Prefix (e.g. &euro;)'), false);
+    $form->add('text', 'suffix', E::ts('Suffix (e.g. $)'), false);
+
+    if (isset($field['configuration'])) {
+      $configuration = $field['configuration'];
+      $defaults = array();
+      if (isset($configuration['field']) && isset($configuration['datasource'])) {
+        $defaults['field'] = $configuration['datasource'] . '::' . $configuration['field'];
+      }
+      if (isset($configuration['function'])) {
+        $defaults['function'] = $configuration['function'];
+      }
+      if (isset($configuration['number_of_decimals'])) {
+        $defaults['number_of_decimals'] = $configuration['number_of_decimals'];
+      }
+      if (isset($configuration['decimal_separator'])) {
+        $defaults['decimal_separator'] = $configuration['decimal_separator'];
+      }
+      if (isset($configuration['thousand_separator'])) {
+        $defaults['thousand_separator'] = $configuration['thousand_separator'];
+      }
+      if (isset($configuration['prefix'])) {
+        $defaults['prefix'] = $configuration['prefix'];
+      }
+      if (isset($configuration['suffix'])) {
+        $defaults['suffix'] = $configuration['suffix'];
+      }
+      $form->setDefaults($defaults);
+    }
+  }
+
+  /**
+   * When this handler has configuration specify the template file name
+   * for the configuration form.
+   *
+   * @return false|string
+   */
+  public function getConfigurationTemplateFileName() {
+    return "CRM/Dataprocessor/Form/Field/Configuration/AggregateFunctionFieldOutputHandler.tpl";
+  }
+
+
+  /**
+   * Process the submitted values and create a configuration array
+   *
+   * @param $submittedValues
+   * @return array
+   */
+  public function processConfiguration($submittedValues) {
+    list($datasource, $field) = explode('::', $submittedValues['field'], 2);
+    $configuration['field'] = $field;
+    $configuration['datasource'] = $datasource;
+    $configuration['function'] = $submittedValues['function'];
+    $configuration['number_of_decimals'] = $submittedValues['number_of_decimals'];
+    $configuration['decimal_separator'] = $submittedValues['decimal_separator'];
+    $configuration['thousand_separator'] = $submittedValues['thousand_separator'];
+    $configuration['prefix'] = $submittedValues['prefix'];
+    $configuration['suffix'] = $submittedValues['suffix'];
+    return $configuration;
+  }
+
+  /**
+   * Callback function for determining whether this field could be handled by this output handler.
+   *
+   * @param \Civi\DataProcessor\DataSpecification\FieldSpecification $field
+   * @return bool
+   */
+  public function isFieldValid(FieldSpecification $field) {
+    switch ($field->type){
+      case 'Int':
+      case 'Integer':
+      case 'Float':
+      case 'Money':
+        return true;
+        break;
+    }
+    return false;
+  }
+
+}

Civi/DataProcessor/Source/AbstractCivicrmEntitySource.php

@@ -330,10 +330,10 @@ abstract class AbstractCivicrmEntitySource extends AbstractSource {
    */
   public function ensureFieldInSource(FieldSpecification $fieldSpecification) {
     try {
-      if ($this->getAvailableFields()
-        ->doesFieldExist($fieldSpecification->name)) {
-        if ($fieldSpecification instanceof CustomFieldSpecification) {
-          $customGroupDataFlow = $this->ensureCustomGroup($fieldSpecification->customGroupTableName, $fieldSpecification->customGroupName);
+      if ($this->getAvailableFields()->doesFieldExist($fieldSpecification->name)) {
+        $originalFieldSpecification = $this->getAvailableFields()->getFieldSpecificationByName($fieldSpecification->name);
+        if ($originalFieldSpecification instanceof CustomFieldSpecification) {
+          $customGroupDataFlow = $this->ensureCustomGroup($originalFieldSpecification->customGroupTableName, $originalFieldSpecification->customGroupName);
           if (!$customGroupDataFlow->getDataSpecification()
             ->doesFieldExist($fieldSpecification->alias)) {
             $customGroupDataFlow->getDataSpecification()
@@ -377,4 +377,4 @@ abstract class AbstractCivicrmEntitySource extends AbstractSource {
     return $this->primaryDataFlow;
   }
 
-}
+}
\ No newline at end of file

Civi/DataProcessor/Source/Contribution/ContributionSource.php

@@ -10,6 +10,7 @@ use Civi\DataProcessor\DataFlow\CombinedDataFlow\CombinedSqlDataFlow;
 use Civi\DataProcessor\DataFlow\MultipleDataFlows\DataFlowDescription;
 use Civi\DataProcessor\DataFlow\MultipleDataFlows\SimpleJoin;
 use Civi\DataProcessor\DataFlow\CombinedDataFlow\SubqueryDataFlow;
+use Civi\DataProcessor\DataFlow\MultipleDataFlows\SimpleNonRequiredJoin;
 use Civi\DataProcessor\DataFlow\SqlTableDataFlow;
 use Civi\DataProcessor\DataSpecification\DataSpecification;
 use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
@@ -93,7 +94,7 @@ class ContributionSource extends AbstractCivicrmEntitySource {
   protected function getEntityDataFlow() {
     $contributionDataDescription = new DataFlowDescription($this->contributionDataFlow);
 
-    $join = new SimpleJoin($this->contributionDataFlow->getTableAlias(), 'id', $this->contributionSoftDataFlow->getTableAlias(), 'contribution_id');
+    $join = new SimpleJoin($this->contributionDataFlow->getTableAlias(), 'id', $this->contributionSoftDataFlow->getTableAlias(), 'contribution_id', 'LEFT');
     $join->setDataProcessor($this->dataProcessor);
     $contributionSoftDataDescription = new DataFlowDescription($this->contributionSoftDataFlow, $join);
 
@@ -144,4 +145,20 @@ class ContributionSource extends AbstractCivicrmEntitySource {
     DataSpecificationUtils::addDAOFieldsToDataSpecification($daoClass, $dataSpecification, $fieldsToSkip, '', $aliasPrefix);
     DataSpecificationUtils::addDAOFieldsToDataSpecification('CRM_Contribute_DAO_ContributionSoft', $dataSpecification, array('id', 'contribution_id'), 'contribution_soft_', $aliasPrefix, E::ts('Soft :: '));
   }
-}
+
\ No newline at end of file
+  /**
+   * Returns the default configuration for this data source
+   *
+   * @return array
+   */
+  public function getDefaultConfiguration() {
+    return array(
+      'filter' => array(
+        'is_test' => array (
+          'op' => '=',
+          'value' => '0',
+        )
+      )
+    );
+  }
+}

templates/CRM/Dataprocessor/Form/Field/Configuration/AggregateFunctionFieldOutputHandler.tpl

+{crmScope extensionKey='dataprocessor'}
+  <div class="crm-section">
+    <div class="label">{$form.function.label}</div>
+    <div class="content">{$form.function.html}</div>
+    <div class="clear"></div>
+  </div>
+  <div class="crm-section">
+      <div class="label">{$form.field.label}</div>
+      <div class="content">{$form.field.html}</div>
+      <div class="clear"></div>
+  </div>
+
+  <div class="crm-section">
+    <div class="label">{$form.number_of_decimals.label}</div>
+    <div class="content">{$form.number_of_decimals.html}
+      <p class="description">{ts}Leave empty for no formatting{/ts}</p>
+    </div>
+    <div class="clear"></div>
+  </div>
+  <div class="crm-section">
+    <div class="label">{$form.decimal_separator.label}</div>
+    <div class="content">{$form.decimal_separator.html}</div>
+    <div class="clear"></div>
+  </div>
+  <div class="crm-section">
+    <div class="label">{$form.thousand_separator.label}</div>
+    <div class="content">{$form.thousand_separator.html}</div>
+    <div class="clear"></div>
+  </div>
+  <div class="crm-section">
+    <div class="label">{$form.prefix.label}</div>
+    <div class="content">{$form.prefix.html}</div>
+    <div class="clear"></div>
+  </div>
+  <div class="crm-section">
+    <div class="label">{$form.suffix.label}</div>
+    <div class="content">{$form.suffix.html}</div>
+    <div class="clear"></div>
+  </div>
+
+{/crmScope}