Optimization of the contact data source and contact type filter.

CHANGELOG.md

 # Version 1.62 (not yet released)
 
+* Optimization of the contact data source and contact type filter.
+
 # Version 1.61
 
 * Improved on !98 filter parameter for SQL Table sources.

Civi/DataProcessor/Source/Contact/AbstractContactSource.php

+<?php
+/**
+ * Copyright (C) 2023  Jaap Jansma (jaap.jansma@civicoop.org)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace Civi\DataProcessor\Source\Contact;
+
+use Civi\DataProcessor\DataFlow\SqlDataFlow\AndClause;
+use Civi\DataProcessor\DataFlow\SqlDataFlow\InTableWhereClause;
+use Civi\DataProcessor\DataFlow\SqlDataFlow\OrClause;
+use Civi\DataProcessor\DataFlow\SqlDataFlow\SimpleWhereClause;
+use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
+
+abstract class AbstractContactSource extends AbstractCivicrmEntitySource {
+
+  /**
+   * Add a sub query filter for the contact type.
+   *
+   * The subquery performance better than a direct where statement on the contact_type
+   * field.
+   *
+   * @param array $contactTypes
+   * @param string $op
+   *
+   * @return void
+   * @throws \Exception
+   */
+  protected function addContactTypeFilter(array $contactTypes, string $op = 'IN') {
+    switch ($op) {
+      case '=':
+        $op = 'IN';
+        break;
+      case '!=':
+        $op = 'NOT IN';
+        break;
+    }
+    if ($op != 'IN' && $op != 'NOT IN') {
+      return;
+    }
+    if (count($contactTypes)) {
+      $this->ensureEntity();
+      $tableAlias = $this->getEntityTableAlias();
+      $spec = $this->getAvailableFields()->getFieldSpecificationByName('id');
+      if (count($contactTypes) == 1) {
+        $contactTypeFilter = new SimpleWhereClause($tableAlias . '_contact_type', 'contact_type', '=', reset($contactTypes));
+      } else {
+        $contactTypeFilter = new SimpleWhereClause($tableAlias . '_contact_type', 'contact_type', 'IN', $contactTypes);
+      }
+      $inTableContactTypeFilter = new InTableWhereClause('id', 'civicrm_contact', $tableAlias . '_contact_type', [$contactTypeFilter], $tableAlias, $spec->name, $op, TRUE);
+      $this->entityDataFlow->addWhereClause($inTableContactTypeFilter);
+    }
+  }
+
+  /**
+   * Adds an inidvidual filter to the data source
+   *
+   * @param $filter_field_alias
+   * @param $op
+   * @param $values
+   *
+   * @throws \Exception
+   */
+  protected function addFilter($filter_field_alias, $op, $values) {
+    if ($filter_field_alias == 'contact_type' && !in_array($op, ['IS NULL', 'IS NOT NULL'])) {
+      $contactTypes = $values;
+      if (!is_array($contactTypes)) {
+        $contactTypes = [$contactTypes];
+      }
+      $this->addContactTypeFilter($contactTypes, $op);
+    } elseif ($filter_field_alias == 'contact_sub_type' && $op == 'IN') {
+      $contactTypeClauses = [];
+      foreach ($values as $value) {
+        $contactTypeSearchName = '%' . \CRM_Core_DAO::VALUE_SEPARATOR . $value . \CRM_Core_DAO::VALUE_SEPARATOR . '%';
+        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'LIKE', $contactTypeSearchName, 'String', TRUE);
+      }
+      if (count($contactTypeClauses)) {
+        $contactTypeClause = new OrClause($contactTypeClauses, TRUE);
+        $entityDataFlow = $this->ensureEntity();
+        $entityDataFlow->addWhereClause($contactTypeClause);
+      }
+    } elseif ($filter_field_alias == 'contact_sub_type' && $op == 'NOT IN') {
+      $contactTypeClauses = [];
+      foreach($values as $value) {
+        $contactTypeSearchName = '%'.\CRM_Core_DAO::VALUE_SEPARATOR.$value.\CRM_Core_DAO::VALUE_SEPARATOR.'%';
+        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'NOT LIKE', $contactTypeSearchName, 'String',TRUE);
+      }
+      if (count($contactTypeClauses)) {
+        $contactTypeClause = new AndClause($contactTypeClauses, TRUE);
+        $entityDataFlow = $this->ensureEntity();
+        $entityDataFlow->addWhereClause($contactTypeClause);
+      }
+    } else {
+      parent::addFilter($filter_field_alias, $op, $values);
+    }
+  }
+
+}

Civi/DataProcessor/Source/Contact/ContactSource.php

@@ -6,23 +6,16 @@
 
 namespace Civi\DataProcessor\Source\Contact;
 
-use Civi\DataProcessor\DataFlow\SqlDataFlow\AndClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\OrClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\SimpleWhereClause;
 use Civi\DataProcessor\DataSpecification\DataSpecification;
-use Civi\DataProcessor\DataSpecification\Utils as DataSpecificationUtils;
-use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
 
-use CRM_Dataprocessor_ExtensionUtil as E;
-
-class ContactSource extends AbstractCivicrmEntitySource {
+class ContactSource extends AbstractContactSource {
 
   /**
    * Returns the entity name
    *
    * @return String
    */
-  protected function getEntity() {
+  protected function getEntity(): string {
     return 'Contact';
   }
 
@@ -31,7 +24,7 @@ class ContactSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getTable() {
+  protected function getTable(): string {
     return 'civicrm_contact';
   }
 
@@ -39,7 +32,7 @@ class ContactSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFilterFields() {
+  public function getAvailableFilterFields(): DataSpecification {
     if (!$this->availableFilterFields) {
       $this->availableFilterFields = new DataSpecification();
       $this->loadFields($this->availableFilterFields);
@@ -55,10 +48,10 @@ class ContactSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFields() {
+  public function getAvailableFields(): DataSpecification {
     if (!$this->availableFields) {
       $this->availableFields = new DataSpecification();
-      $this->loadFields($this->availableFields, array());
+      $this->loadFields($this->availableFields);
       $this->loadCustomGroupsAndFields($this->availableFields, false);
       $this->loadCustomGroupsAndFields($this->availableFields, false, 'Individual');
       $this->loadCustomGroupsAndFields($this->availableFields, false, 'Household');
@@ -73,7 +66,7 @@ class ContactSource extends AbstractCivicrmEntitySource {
    *
    * @return array
    */
-  public function getDefaultConfiguration() {
+  public function getDefaultConfiguration(): array {
     return array(
       'filter' => array(
         'is_deleted' => array (
@@ -84,41 +77,4 @@ class ContactSource extends AbstractCivicrmEntitySource {
     );
   }
 
-  /**
-   * Adds an inidvidual filter to the data source
-   *
-   * @param $filter_field_alias
-   * @param $op
-   * @param $values
-   *
-   * @throws \Exception
-   */
-  protected function addFilter($filter_field_alias, $op, $values) {
-    if ($filter_field_alias == 'contact_sub_type' && $op == 'IN') {
-      $contactTypeClauses = [];
-      foreach ($values as $value) {
-        $contactTypeSearchName = '%' . \CRM_Core_DAO::VALUE_SEPARATOR . $value . \CRM_Core_DAO::VALUE_SEPARATOR . '%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'LIKE', $contactTypeSearchName, 'String', TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new OrClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } elseif ($filter_field_alias == 'contact_sub_type' && $op == 'NOT IN') {
-      $contactTypeClauses = [];
-      foreach($values as $value) {
-        $contactTypeSearchName = '%'.\CRM_Core_DAO::VALUE_SEPARATOR.$value.\CRM_Core_DAO::VALUE_SEPARATOR.'%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'NOT LIKE', $contactTypeSearchName, 'String',TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new AndClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } else {
-      parent::addFilter($filter_field_alias, $op, $values);
-    }
-  }
-
 }

Civi/DataProcessor/Source/Contact/HouseholdSource.php

@@ -6,15 +6,9 @@
 
 namespace Civi\DataProcessor\Source\Contact;
 
-use Civi\DataProcessor\DataFlow\SqlDataFlow\AndClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\OrClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\SimpleWhereClause;
 use Civi\DataProcessor\DataSpecification\DataSpecification;
-use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
 
-use CRM_Dataprocessor_ExtensionUtil as E;
-
-class HouseholdSource extends AbstractCivicrmEntitySource {
+class HouseholdSource extends AbstractContactSource {
 
   protected $skipFields = array(
     'first_name',
@@ -60,7 +54,7 @@ class HouseholdSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getEntity() {
+  protected function getEntity(): string {
     return 'Contact';
   }
 
@@ -69,7 +63,7 @@ class HouseholdSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getTable() {
+  protected function getTable(): string {
     return 'civicrm_contact';
   }
 
@@ -78,7 +72,7 @@ class HouseholdSource extends AbstractCivicrmEntitySource {
    *
    * @return array
    */
-  public function getDefaultConfiguration() {
+  public function getDefaultConfiguration(): array {
     return array(
       'filter' => array(
         'is_deleted' => array (
@@ -93,7 +87,7 @@ class HouseholdSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFilterFields() {
+  public function getAvailableFilterFields(): DataSpecification {
     if (!$this->availableFilterFields) {
       $this->availableFilterFields = new DataSpecification();
       $this->loadFields($this->availableFilterFields, $this->skipFilterFields);
@@ -106,7 +100,7 @@ class HouseholdSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFields() {
+  public function getAvailableFields(): DataSpecification {
     if (!$this->availableFields) {
       $this->availableFields = new DataSpecification();
       $this->loadFields($this->availableFields, $this->skipFields);
@@ -126,42 +120,5 @@ class HouseholdSource extends AbstractCivicrmEntitySource {
     $this->addFilter('contact_type', '=', 'Household');
   }
 
-  /**
-   * Adds an inidvidual filter to the data source
-   *
-   * @param $filter_field_alias
-   * @param $op
-   * @param $values
-   *
-   * @throws \Exception
-   */
-  protected function addFilter($filter_field_alias, $op, $values) {
-    if ($filter_field_alias == 'contact_sub_type' && $op == 'IN') {
-      $contactTypeClauses = [];
-      foreach ($values as $value) {
-        $contactTypeSearchName = '%' . \CRM_Core_DAO::VALUE_SEPARATOR . $value . \CRM_Core_DAO::VALUE_SEPARATOR . '%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'LIKE', $contactTypeSearchName, 'String', TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new OrClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } elseif ($filter_field_alias == 'contact_sub_type' && $op == 'NOT IN') {
-      $contactTypeClauses = [];
-      foreach($values as $value) {
-        $contactTypeSearchName = '%'.\CRM_Core_DAO::VALUE_SEPARATOR.$value.\CRM_Core_DAO::VALUE_SEPARATOR.'%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'NOT LIKE', $contactTypeSearchName, 'String',TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new AndClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } else {
-      parent::addFilter($filter_field_alias, $op, $values);
-    }
-  }
-
 
 }

Civi/DataProcessor/Source/Contact/IndividualSource.php

@@ -6,15 +6,9 @@
 
 namespace Civi\DataProcessor\Source\Contact;
 
-use Civi\DataProcessor\DataFlow\SqlDataFlow\AndClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\OrClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\SimpleWhereClause;
 use Civi\DataProcessor\DataSpecification\DataSpecification;
-use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
 
-use CRM_Dataprocessor_ExtensionUtil as E;
-
-class IndividualSource extends AbstractCivicrmEntitySource {
+class IndividualSource extends AbstractContactSource {
 
   protected $skipFields = array(
     'household_name',
@@ -36,7 +30,7 @@ class IndividualSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getEntity() {
+  protected function getEntity(): string {
     return 'Contact';
   }
 
@@ -45,7 +39,7 @@ class IndividualSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getTable() {
+  protected function getTable(): string {
     return 'civicrm_contact';
   }
 
@@ -54,7 +48,7 @@ class IndividualSource extends AbstractCivicrmEntitySource {
    *
    * @return array
    */
-  public function getDefaultConfiguration() {
+  public function getDefaultConfiguration(): array {
     return array(
       'filter' => array(
         'is_deleted' => array (
@@ -73,7 +67,7 @@ class IndividualSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFilterFields() {
+  public function getAvailableFilterFields(): DataSpecification {
     if (!$this->availableFilterFields) {
       $this->availableFilterFields = new DataSpecification();
       $this->loadFields($this->availableFilterFields, $this->skipFilterFields);
@@ -86,7 +80,7 @@ class IndividualSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFields() {
+  public function getAvailableFields(): DataSpecification {
     if (!$this->availableFields) {
       $this->availableFields = new DataSpecification();
       $this->loadFields($this->availableFields, $this->skipFields);
@@ -106,42 +100,5 @@ class IndividualSource extends AbstractCivicrmEntitySource {
     $this->addFilter('contact_type', '=', 'Individual');
   }
 
-  /**
-   * Adds an inidvidual filter to the data source
-   *
-   * @param $filter_field_alias
-   * @param $op
-   * @param $values
-   *
-   * @throws \Exception
-   */
-  protected function addFilter($filter_field_alias, $op, $values) {
-    if ($filter_field_alias == 'contact_sub_type' && $op == 'IN') {
-      $contactTypeClauses = [];
-      foreach ($values as $value) {
-        $contactTypeSearchName = '%' . \CRM_Core_DAO::VALUE_SEPARATOR . $value . \CRM_Core_DAO::VALUE_SEPARATOR . '%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'LIKE', $contactTypeSearchName, 'String', TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new OrClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } elseif ($filter_field_alias == 'contact_sub_type' && $op == 'NOT IN') {
-      $contactTypeClauses = [];
-      foreach($values as $value) {
-        $contactTypeSearchName = '%'.\CRM_Core_DAO::VALUE_SEPARATOR.$value.\CRM_Core_DAO::VALUE_SEPARATOR.'%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'NOT LIKE', $contactTypeSearchName, 'String',TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new AndClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } else {
-      parent::addFilter($filter_field_alias, $op, $values);
-    }
-  }
-
 
 }

Civi/DataProcessor/Source/Contact/OrganizationSource.php

@@ -6,15 +6,9 @@
 
 namespace Civi\DataProcessor\Source\Contact;
 
-use Civi\DataProcessor\DataFlow\SqlDataFlow\AndClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\OrClause;
-use Civi\DataProcessor\DataFlow\SqlDataFlow\SimpleWhereClause;
 use Civi\DataProcessor\DataSpecification\DataSpecification;
-use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
 
-use CRM_Dataprocessor_ExtensionUtil as E;
-
-class OrganizationSource extends AbstractCivicrmEntitySource {
+class OrganizationSource extends AbstractContactSource {
 
   protected $skipFields = array(
     'first_name',
@@ -52,7 +46,7 @@ class OrganizationSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getEntity() {
+  protected function getEntity(): string {
     return 'Contact';
   }
 
@@ -61,7 +55,7 @@ class OrganizationSource extends AbstractCivicrmEntitySource {
    *
    * @return String
    */
-  protected function getTable() {
+  protected function getTable(): string {
     return 'civicrm_contact';
   }
 
@@ -70,7 +64,7 @@ class OrganizationSource extends AbstractCivicrmEntitySource {
    *
    * @return array
    */
-  public function getDefaultConfiguration() {
+  public function getDefaultConfiguration(): array {
     return array(
       'filter' => array(
         'is_deleted' => array (
@@ -85,7 +79,7 @@ class OrganizationSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFilterFields() {
+  public function getAvailableFilterFields(): DataSpecification {
     if (!$this->availableFilterFields) {
       $this->availableFilterFields = new DataSpecification();
       $this->loadFields($this->availableFilterFields, $this->skipFilterFields);
@@ -98,7 +92,7 @@ class OrganizationSource extends AbstractCivicrmEntitySource {
    * @return \Civi\DataProcessor\DataSpecification\DataSpecification
    * @throws \Exception
    */
-  public function getAvailableFields() {
+  public function getAvailableFields(): DataSpecification {
     if (!$this->availableFields) {
       $this->availableFields = new DataSpecification();
       $this->loadFields($this->availableFields, $this->skipFields);
@@ -118,42 +112,5 @@ class OrganizationSource extends AbstractCivicrmEntitySource {
     $this->addFilter('contact_type', '=', 'Organization');
   }
 
-  /**
-   * Adds an inidvidual filter to the data source
-   *
-   * @param $filter_field_alias
-   * @param $op
-   * @param $values
-   *
-   * @throws \Exception
-   */
-  protected function addFilter($filter_field_alias, $op, $values) {
-    if ($filter_field_alias == 'contact_sub_type' && $op == 'IN') {
-      $contactTypeClauses = [];
-      foreach ($values as $value) {
-        $contactTypeSearchName = '%' . \CRM_Core_DAO::VALUE_SEPARATOR . $value . \CRM_Core_DAO::VALUE_SEPARATOR . '%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'LIKE', $contactTypeSearchName, 'String', TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new OrClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } elseif ($filter_field_alias == 'contact_sub_type' && $op == 'NOT IN') {
-      $contactTypeClauses = [];
-      foreach($values as $value) {
-        $contactTypeSearchName = '%'.\CRM_Core_DAO::VALUE_SEPARATOR.$value.\CRM_Core_DAO::VALUE_SEPARATOR.'%';
-        $contactTypeClauses[] = new SimpleWhereClause($this->getSourceName(), 'contact_sub_type', 'NOT LIKE', $contactTypeSearchName, 'String',TRUE);
-      }
-      if (count($contactTypeClauses)) {
-        $contactTypeClause = new AndClause($contactTypeClauses, TRUE);
-        $entityDataFlow = $this->ensureEntity();
-        $entityDataFlow->addWhereClause($contactTypeClause);
-      }
-    } else {
-      parent::addFilter($filter_field_alias, $op, $values);
-    }
-  }
-
 
 }