diff --git a/CHANGELOG.md b/CHANGELOG.md
index f620cadb99a278291fe993bebd5b719dd6a75273..565f7ec686d293845842682e79557cb4b0f3aca5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
* Regression fix dashlets not working anymore #122
* Fixed contact Summary tab not loading due to fatal error. See !115 and #122
+* Performance improvements.
# Version 1.69
diff --git a/CRM/Dataprocessor/Utils/Smartgroup.php b/CRM/Dataprocessor/Utils/Smartgroup.php
new file mode 100644
index 0000000000000000000000000000000000000000..bcc6ab5af3d00a467119521cadee0d1eadc0cf77
--- /dev/null
+++ b/CRM/Dataprocessor/Utils/Smartgroup.php
@@ -0,0 +1,32 @@
+<?php
+/**
+ * Copyright (C) 2023 Jaap Jansma (jaap.jansma@civicoop.org)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+class CRM_Dataprocessor_Utils_Smartgroup {
+
+ public static function checkOrRefresh(array $groupIds, $timeOut=120) {
+ $key = 'dataprocessor_smartgroup_check_'.md5(json_encode($groupIds));
+ $cache = CRM_Dataprocessor_Utils_Cache::singleton();
+ $lastCheckTime = $cache->get($key);
+ $timeOut = time() - $timeOut;
+ if (empty($toCheck) || $lastCheckTime < $timeOut) {
+ CRM_Contact_BAO_GroupContactCache::check($groupIds);
+ $cache->set($key, time());
+ }
+ }
+
+}
diff --git a/CRM/DataprocessorSearch/Form/AbstractSearch.php b/CRM/DataprocessorSearch/Form/AbstractSearch.php
index 374862244b058ec1a62909db4ac1794e78f8291d..b957a77b308575324e9c261303c01e46b45bda26 100644
--- a/CRM/DataprocessorSearch/Form/AbstractSearch.php
+++ b/CRM/DataprocessorSearch/Form/AbstractSearch.php
@@ -645,7 +645,7 @@ abstract class CRM_DataprocessorSearch_Form_AbstractSearch extends CRM_Dataproce
public function allRecordsSelected(): bool {
if (isset($this->_formValues['radio_ts']) && $this->_formValues['radio_ts'] == 'ts_all') {
return true;
- } elseif (!isset($this->_formValues['radio_ts'])) {
+ } elseif (!isset($this->_formValues['radio_ts']) || $this->_formValues['radio_ts'] === "") {
return true;
}
return false;
diff --git a/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php b/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php
index f449ba82f6c802e404c2b1f64a43104fd5faa24b..dc103b6c1b04f039a16b0359d7397062166c4b4c 100644
--- a/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php
+++ b/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php
@@ -22,6 +22,8 @@ class InTableWhereClause extends AbstractWhereClause implements WhereClauseInter
protected $operator;
+ protected $indexStatement = "";
+
/**
* @var bool
*/
@@ -38,6 +40,11 @@ class InTableWhereClause extends AbstractWhereClause implements WhereClauseInter
$this->source_table_alias = $source_table_alias;
}
+ public function setIndexStatement(string $indexStatement): InTableWhereClause {
+ $this->indexStatement = $indexStatement;
+ return $this;
+ }
+
/**
* Returns the where clause
* E.g. contact_type = 'Individual'
@@ -55,7 +62,7 @@ class InTableWhereClause extends AbstractWhereClause implements WhereClauseInter
return "`$this->source_table_alias`.`$this->source_field` $this->operator (
SELECT `$this->table_alias`.`$this->select_field`
- FROM `$this->table` `$this->table_alias`
+ FROM `$this->table` `$this->table_alias` $this->indexStatement
WHERE $whereClause
)";
}
diff --git a/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php b/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php
index 0e77b74e862d8754b8072d8634f41b18153688cb..5970c6aaf283a3a728adc1118f70a3fb94248c43 100644
--- a/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php
+++ b/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php
@@ -268,13 +268,14 @@ class ContactHasContributionInPeriod extends AbstractFieldInPeriodFilter {
SELECT `contact_id`
FROM `civicrm_contribution` `c_$fieldAlias`
WHERE 1";
- if (isset($filterParams['status_ids']) && is_array($filterParams['status_ids']) && count($filterParams['status_ids'])) {
- $baseSqlStatement .= " AND `c_$fieldAlias`.`contribution_status_id` IN (" . implode(",", $filterParams['status_ids']) . ")";
- }
if (isset($filterParams['financial_type_ids']) && is_array($filterParams['financial_type_ids']) && count($filterParams['financial_type_ids'])) {
$baseSqlStatement .= " AND `c_$fieldAlias`.`financial_type_id` IN (" . implode(",", $filterParams['financial_type_ids']) . ")";
}
$baseSqlStatement .= " AND " . $this->getDateSqlStatement('c_'.$fieldAlias, 'receive_date', $filterParams);
+ $baseSqlStatement .= " AND `c_$fieldAlias`.`is_test` = 0";
+ if (isset($filterParams['status_ids']) && is_array($filterParams['status_ids']) && count($filterParams['status_ids'])) {
+ $baseSqlStatement .= " AND `c_$fieldAlias`.`contribution_status_id` IN (" . implode(",", $filterParams['status_ids']) . ")";
+ }
$includeCampaignIds = [];
$excludeCampaignIds = [];
if (isset($filterParams['campaign_ids']) && is_array($filterParams['campaign_ids']) && count($filterParams['campaign_ids'])) {
diff --git a/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php b/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php
index cc21de1756e3583d4b1d9ba21b336d216e9e09e5..52dfa9044c37a8e2a0d694c42eeb0c9789c539a1 100644
--- a/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php
+++ b/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php
@@ -67,18 +67,19 @@ class ContactInGroupFilter extends AbstractFieldFilterHandler {
// If the groups are smartgroups (saved searches) they may be out of date.
// This triggers a check (and rebuild if necessary).
- \CRM_Contact_BAO_GroupContactCache::check($group_ids);
+ \CRM_Dataprocessor_Utils_Smartgroup::checkOrRefresh($group_ids);
+
- // Look in the group contact table
- $groupTableAlias = 'civicrm_group_contact_'.$this->inputFieldSpecification->alias;
- $groupFilters = array(
- new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'status', '=', 'Added'),
- new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'group_id', 'IN', $group_ids),
- );
if ($dataFlow && $dataFlow instanceof SqlDataFlow) {
+ // Look in the group contact table
+ $groupTableAlias = 'civicrm_group_contact_'.$this->inputFieldSpecification->alias;
+ $groupFilters = array(
+ new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'status', '=', 'Added'),
+ new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'group_id', 'IN', $group_ids),
+ );
$tableAlias = $this->getTableAlias($dataFlow);
- $this->whereClause = new SqlDataFlow\InTableWhereClause(
+ $groupWhereClause = new SqlDataFlow\InTableWhereClause(
'contact_id',
'civicrm_group_contact',
$groupTableAlias,
@@ -87,28 +88,25 @@ class ContactInGroupFilter extends AbstractFieldFilterHandler {
$this->inputFieldSpecification->getName(),
$filter['op']
);
- $whereClauses[] = $this->whereClause;
- }
-
- // Now look in the smartgroup group contact table
- $groupTableAlias = 'civicrm_group_contact_cache_'.$this->inputFieldSpecification->alias;
- $groupFilters = array(
- new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'group_id', 'IN', $group_ids),
- );
+ $whereClauses[] = $groupWhereClause;
- if ($dataFlow && $dataFlow instanceof SqlDataFlow) {
+ // Now look in the smartgroup group contact table
+ $smartGroupTableAlias = 'civicrm_group_contact_cache_'.$this->inputFieldSpecification->alias;
+ $smartGroupFilters = array(
+ new SqlDataFlow\SimpleWhereClause($smartGroupTableAlias, 'group_id', 'IN', $group_ids),
+ );
$tableAlias = $this->getTableAlias($dataFlow);
- $this->whereClause = new SqlDataFlow\InTableWhereClause(
+ $smartGroupWhereClause = new SqlDataFlow\InTableWhereClause(
'contact_id',
'civicrm_group_contact_cache',
- $groupTableAlias,
- $groupFilters,
+ $smartGroupTableAlias,
+ $smartGroupFilters,
$tableAlias,
$this->inputFieldSpecification->getName(),
$filter['op']
);
-
- $whereClauses[] = $this->whereClause;
+ $smartGroupWhereClause->setIndexStatement('USE INDEX (`UI_contact_group`)');
+ $whereClauses[] = $smartGroupWhereClause;
}
switch ($filter['op']) {
case 'IN':