Improvements for count query and preparation for php 8 compatibility

CHANGELOG.md

 # Version 1.63 (not yet released)
 
+* Preparation for PHP 8 compatibility
+* Improved count query by removing group by and functions (such as AVG and SUM).
+
 # Version 1.62
 
 * Optimization of the contact data source and contact type filter.

Civi/DataProcessor/DataFlow/CombinedDataFlow/CombinedSqlDataFlow.php

@@ -134,12 +134,13 @@ class CombinedSqlDataFlow extends SqlDataFlow implements MultipleSourceDataFlows
   /**
    * Returns an array with the fields for in the select statement in the sql query.
    *
+   * @param bool $isCountQuery
    * @return string[]
    */
-  public function getFieldsForSelectStatement(): array {
+  public function getFieldsForSelectStatement(bool $isCountQuery=false): array {
     $fields = array();
     foreach($this->sourceDataFlowDescriptions as $sourceDataFlowDescription) {
-      $fields = array_merge($fields, $sourceDataFlowDescription->getDataFlow()->getFieldsForSelectStatement());
+      $fields = array_merge($fields, $sourceDataFlowDescription->getDataFlow()->getFieldsForSelectStatement($isCountQuery));
     }
     return $fields;
   }

Civi/DataProcessor/DataFlow/CombinedDataFlow/UnionQueryDataFlow.php

@@ -103,13 +103,17 @@ class UnionQueryDataFlow extends SqlDataFlow implements MultipleSourceDataFlows
   /**
    * Returns an array with the fields for in the select statement in the sql query.
    *
+   * @param bool $isCountQuery
    * @return string[]
    */
-  public function getFieldsForSelectStatement(): array {
+  public function getFieldsForSelectStatement(bool $isCountQuery = false): array {
     $fields = array();
     foreach($this->getDataSpecification()->getFields() as $field) {
       if ($field instanceof SqlFieldSpecification) {
-        $fields[] = $field->getSqlSelectStatement($this->getName());
+        $fieldStatement = $field->getSqlSelectStatement($this->getName(), $isCountQuery);
+        if (!empty($fieldStatement)) {
+          $fields[] = $fieldStatement;
+        }
       } else {
         $fields[] = "`{$this->getName()}`.`$field->name` AS `$field->alias`";
       }

Civi/DataProcessor/DataFlow/SqlDataFlow.php

@@ -51,10 +51,11 @@ abstract class SqlDataFlow extends AbstractDataFlow {
   /**
    * Returns an array with the fields for in the select statement in the sql query.
    *
+   * @param bool $isCountQuery
    * @return string[]
    * @throws \Civi\DataProcessor\DataSpecification\FieldExistsException
    */
-  abstract public function getFieldsForSelectStatement(): array;
+  abstract public function getFieldsForSelectStatement(bool $isCountQuery=false): array;
 
   /**
    * Returns an array with the fields for in the group by statement in the sql query.
@@ -132,12 +133,13 @@ abstract class SqlDataFlow extends AbstractDataFlow {
 
     try {
       $selectAndFrom = $this->getSelectQueryStatement();
+      $selectAndFromForCountQuery = $this->getSelectQueryStatement(true);
       $where = $this->getWhereStatement();
       $groupBy = $this->getGroupByStatement();
       $orderBy = $this->getOrderByStatement();
       $countName = 'count_'.$this->getName();
       $sql = "$selectAndFrom $where $groupBy $orderBy";
-      $countSql = "SELECT COUNT(*) AS `count` FROM ($selectAndFrom $where $groupBy) `$countName`";
+      $countSql = "SELECT COUNT(*) AS `count` FROM ($selectAndFromForCountQuery $where) `$countName`";
       $this->sqlCountStatements[] = $countSql;
       $startTime = microtime(true);
       $countDao = CRM_Core_DAO::executeQuery($countSql, [], true, NULL, false, true);
@@ -241,11 +243,13 @@ abstract class SqlDataFlow extends AbstractDataFlow {
   }
 
   /**
+   * @param bool $isCountQuery
+   *
    * @return string
    * @throws \Civi\DataProcessor\DataSpecification\FieldExistsException
    */
-  public function getSelectQueryStatement(): string {
-    $select = implode(", ", $this->getFieldsForSelectStatement());
+  public function getSelectQueryStatement(bool $isCountQuery = false): string {
+    $select = implode(", ", $this->getFieldsForSelectStatement($isCountQuery));
     $from = $this->getFromStatement();
     return "SELECT DISTINCT $select $from";
   }

Civi/DataProcessor/DataFlow/SqlTableDataFlow.php

@@ -57,14 +57,18 @@ class SqlTableDataFlow extends SqlDataFlow {
   /**
    * Returns an array with the fields for in the select statement in the sql query.
    *
+   * @param bool $isCountQuery
    * @return string[]
    * @throws \Civi\DataProcessor\DataSpecification\FieldExistsException
    */
-  public function getFieldsForSelectStatement(): array {
+  public function getFieldsForSelectStatement(bool $isCountQuery = false): array {
     $fields = array();
     foreach($this->getDataSpecification()->getFields() as $field) {
       if ($field instanceof SqlFieldSpecification) {
-        $fields[] = $field->getSqlSelectStatement($this->table_alias);
+        $fieldStatement = $field->getSqlSelectStatement($this->table_alias, $isCountQuery);
+        if (!empty($fieldStatement)) {
+          $fields[] = $fieldStatement;
+        }
       } else {
         $fields[] = "`{$this->table_alias}`.`{$field->name}` AS `{$field->alias}`";
       }

Civi/DataProcessor/DataSpecification/AggregateFunctionFieldSpecification.php

@@ -107,16 +107,20 @@ class AggregateFunctionFieldSpecification extends FieldSpecification implements
    * Returns the select statement for this field.
    * E.g. COUNT(civicrm_contact.id) AS contact_id_count
    *
+   * @param bool $isCountQuery
    * @param String $table_alias
    * @return string
    */
-  public function getSqlSelectStatement($table_alias) {
+  public function getSqlSelectStatement($table_alias, bool $isCountQuery = false): string {
     $function_arg = "";
     $function = $this->function;
     if ($function == 'COUNT_DISTINCT') {
       $function = 'COUNT';
       $function_arg = 'DISTINCT';
     }
+    if ($isCountQuery) {
+      return "";
+    }
     return "{$function}({$function_arg}`{$table_alias}`.`{$this->name}`) AS `{$this->alias}`";
   }
 

Civi/DataProcessor/DataSpecification/FieldSpecification.php

@@ -93,11 +93,15 @@ class FieldSpecification implements SqlFieldSpecification {
    * Returns the select statement for this field.
    * E.g. COUNT(civicrm_contact.id) AS contact_id_count
    *
+   * @param bool $isCountQuery
    * @param String $table_alias
    * @return string
    */
-  public function getSqlSelectStatement($table_alias) {
+  public function getSqlSelectStatement($table_alias, bool $isCountQuery = false): string {
     if ($this->sqlValueFormatFunction) {
+      if ($isCountQuery) {
+        return "";
+      }
       if (stripos($this->sqlValueFormatFunction, '%1') !== false && stripos($this->sqlValueFormatFunction, '%2') !== false) {
         return "(".str_replace(['%1', '%2'], [$table_alias, $this->getName()], $this->sqlValueFormatFunction). ") AS `{$this->alias}`";
       } else {

Civi/DataProcessor/DataSpecification/FixedValueSqlFieldSpecification.php

@@ -29,11 +29,12 @@ class FixedValueSqlFieldSpecification extends FieldSpecification {
    * Returns the select statement for this field.
    * E.g. COUNT(civicrm_contact.id) AS contact_id_count
    *
+   * @param bool $isCountQuery
    * @param String $table_alias
    *
    * @return String
    */
-  public function getSqlSelectStatement($table_alias) {
+  public function getSqlSelectStatement($table_alias, bool $isCountQuery = false): string {
     return "{$this->value} as `{$this->alias}`";
   }
 

Civi/DataProcessor/DataSpecification/SqlFieldSpecification.php

@@ -12,10 +12,11 @@ interface SqlFieldSpecification {
    * Returns the select statement for this field.
    * E.g. COUNT(civicrm_contact.id) AS contact_id_count
    *
+   * @param bool $isCountQuery
    * @param String $table_alias
    * @return String
    */
-  public function getSqlSelectStatement($table_alias);
+  public function getSqlSelectStatement($table_alias, bool $isCountQuery = false);
 
   /**
    * Returns the group by statement for this field.

Civi/DataProcessor/Source/Contact/AbstractContactSource.php

@@ -53,14 +53,12 @@ abstract class AbstractContactSource extends AbstractCivicrmEntitySource {
     if (count($contactTypes)) {
       $this->ensureEntity();
       $tableAlias = $this->getEntityTableAlias();
-      $spec = $this->getAvailableFields()->getFieldSpecificationByName('id');
       if (count($contactTypes) == 1) {
-        $contactTypeFilter = new SimpleWhereClause($tableAlias . '_contact_type', 'contact_type', '=', reset($contactTypes));
+        $contactTypeFilter = new SimpleWhereClause($tableAlias, 'contact_type', '=', reset($contactTypes));
       } else {
-        $contactTypeFilter = new SimpleWhereClause($tableAlias . '_contact_type', 'contact_type', 'IN', $contactTypes);
+        $contactTypeFilter = new SimpleWhereClause($tableAlias, 'contact_type', 'IN', $contactTypes);
       }
-      $inTableContactTypeFilter = new InTableWhereClause('id', 'civicrm_contact', $tableAlias . '_contact_type', [$contactTypeFilter], $tableAlias, $spec->name, $op, TRUE);
-      $this->entityDataFlow->addWhereClause($inTableContactTypeFilter);
+      $this->entityDataFlow->addWhereClause($contactTypeFilter);
     }
   }