From 544b76759b912eb8d6211cdec80b630bbdea2819 Mon Sep 17 00:00:00 2001
From: Jaap Jansma <jaap.jansma@civicoop.org>
Date: Tue, 16 May 2023 10:21:50 +0200
Subject: [PATCH] Performance improvements.
---
CHANGELOG.md | 1 +
CRM/Dataprocessor/Utils/Smartgroup.php | 32 ++++++++++++++
.../Form/AbstractSearch.php | 2 +-
.../SqlDataFlow/InTableWhereClause.php | 9 +++-
.../ContactHasContributionInPeriod.php | 7 ++--
.../FilterHandler/ContactInGroupFilter.php | 42 +++++++++----------
6 files changed, 66 insertions(+), 27 deletions(-)
create mode 100644 CRM/Dataprocessor/Utils/Smartgroup.php
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f620cadb..565f7ec6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
* Regression fix dashlets not working anymore #122
* Fixed contact Summary tab not loading due to fatal error. See !115 and #122
+* Performance improvements.
# Version 1.69
diff --git a/CRM/Dataprocessor/Utils/Smartgroup.php b/CRM/Dataprocessor/Utils/Smartgroup.php
new file mode 100644
index 00000000..bcc6ab5a
--- /dev/null
+++ b/CRM/Dataprocessor/Utils/Smartgroup.php
@@ -0,0 +1,32 @@
+<?php
+/**
+ * Copyright (C) 2023 Jaap Jansma (jaap.jansma@civicoop.org)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+class CRM_Dataprocessor_Utils_Smartgroup {
+
+ public static function checkOrRefresh(array $groupIds, $timeOut=120) {
+ $key = 'dataprocessor_smartgroup_check_'.md5(json_encode($groupIds));
+ $cache = CRM_Dataprocessor_Utils_Cache::singleton();
+ $lastCheckTime = $cache->get($key);
+ $timeOut = time() - $timeOut;
+ if (empty($toCheck) || $lastCheckTime < $timeOut) {
+ CRM_Contact_BAO_GroupContactCache::check($groupIds);
+ $cache->set($key, time());
+ }
+ }
+
+}
diff --git a/CRM/DataprocessorSearch/Form/AbstractSearch.php b/CRM/DataprocessorSearch/Form/AbstractSearch.php
index 37486224..b957a77b 100644
--- a/CRM/DataprocessorSearch/Form/AbstractSearch.php
+++ b/CRM/DataprocessorSearch/Form/AbstractSearch.php
@@ -645,7 +645,7 @@ abstract class CRM_DataprocessorSearch_Form_AbstractSearch extends CRM_Dataproce
public function allRecordsSelected(): bool {
if (isset($this->_formValues['radio_ts']) && $this->_formValues['radio_ts'] == 'ts_all') {
return true;
- } elseif (!isset($this->_formValues['radio_ts'])) {
+ } elseif (!isset($this->_formValues['radio_ts']) || $this->_formValues['radio_ts'] === "") {
return true;
}
return false;
diff --git a/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php b/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php
index f449ba82..dc103b6c 100644
--- a/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php
+++ b/Civi/DataProcessor/DataFlow/SqlDataFlow/InTableWhereClause.php
@@ -22,6 +22,8 @@ class InTableWhereClause extends AbstractWhereClause implements WhereClauseInter
protected $operator;
+ protected $indexStatement = "";
+
/**
* @var bool
*/
@@ -38,6 +40,11 @@ class InTableWhereClause extends AbstractWhereClause implements WhereClauseInter
$this->source_table_alias = $source_table_alias;
}
+ public function setIndexStatement(string $indexStatement): InTableWhereClause {
+ $this->indexStatement = $indexStatement;
+ return $this;
+ }
+
/**
* Returns the where clause
* E.g. contact_type = 'Individual'
@@ -55,7 +62,7 @@ class InTableWhereClause extends AbstractWhereClause implements WhereClauseInter
return "`$this->source_table_alias`.`$this->source_field` $this->operator (
SELECT `$this->table_alias`.`$this->select_field`
- FROM `$this->table` `$this->table_alias`
+ FROM `$this->table` `$this->table_alias` $this->indexStatement
WHERE $whereClause
)";
}
diff --git a/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php b/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php
index 0e77b74e..5970c6aa 100644
--- a/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php
+++ b/Civi/DataProcessor/FilterHandler/ContactHasContributionInPeriod.php
@@ -268,13 +268,14 @@ class ContactHasContributionInPeriod extends AbstractFieldInPeriodFilter {
SELECT `contact_id`
FROM `civicrm_contribution` `c_$fieldAlias`
WHERE 1";
- if (isset($filterParams['status_ids']) && is_array($filterParams['status_ids']) && count($filterParams['status_ids'])) {
- $baseSqlStatement .= " AND `c_$fieldAlias`.`contribution_status_id` IN (" . implode(",", $filterParams['status_ids']) . ")";
- }
if (isset($filterParams['financial_type_ids']) && is_array($filterParams['financial_type_ids']) && count($filterParams['financial_type_ids'])) {
$baseSqlStatement .= " AND `c_$fieldAlias`.`financial_type_id` IN (" . implode(",", $filterParams['financial_type_ids']) . ")";
}
$baseSqlStatement .= " AND " . $this->getDateSqlStatement('c_'.$fieldAlias, 'receive_date', $filterParams);
+ $baseSqlStatement .= " AND `c_$fieldAlias`.`is_test` = 0";
+ if (isset($filterParams['status_ids']) && is_array($filterParams['status_ids']) && count($filterParams['status_ids'])) {
+ $baseSqlStatement .= " AND `c_$fieldAlias`.`contribution_status_id` IN (" . implode(",", $filterParams['status_ids']) . ")";
+ }
$includeCampaignIds = [];
$excludeCampaignIds = [];
if (isset($filterParams['campaign_ids']) && is_array($filterParams['campaign_ids']) && count($filterParams['campaign_ids'])) {
diff --git a/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php b/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php
index cc21de17..52dfa904 100644
--- a/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php
+++ b/Civi/DataProcessor/FilterHandler/ContactInGroupFilter.php
@@ -67,18 +67,19 @@ class ContactInGroupFilter extends AbstractFieldFilterHandler {
// If the groups are smartgroups (saved searches) they may be out of date.
// This triggers a check (and rebuild if necessary).
- \CRM_Contact_BAO_GroupContactCache::check($group_ids);
+ \CRM_Dataprocessor_Utils_Smartgroup::checkOrRefresh($group_ids);
+
- // Look in the group contact table
- $groupTableAlias = 'civicrm_group_contact_'.$this->inputFieldSpecification->alias;
- $groupFilters = array(
- new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'status', '=', 'Added'),
- new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'group_id', 'IN', $group_ids),
- );
if ($dataFlow && $dataFlow instanceof SqlDataFlow) {
+ // Look in the group contact table
+ $groupTableAlias = 'civicrm_group_contact_'.$this->inputFieldSpecification->alias;
+ $groupFilters = array(
+ new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'status', '=', 'Added'),
+ new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'group_id', 'IN', $group_ids),
+ );
$tableAlias = $this->getTableAlias($dataFlow);
- $this->whereClause = new SqlDataFlow\InTableWhereClause(
+ $groupWhereClause = new SqlDataFlow\InTableWhereClause(
'contact_id',
'civicrm_group_contact',
$groupTableAlias,
@@ -87,28 +88,25 @@ class ContactInGroupFilter extends AbstractFieldFilterHandler {
$this->inputFieldSpecification->getName(),
$filter['op']
);
- $whereClauses[] = $this->whereClause;
- }
-
- // Now look in the smartgroup group contact table
- $groupTableAlias = 'civicrm_group_contact_cache_'.$this->inputFieldSpecification->alias;
- $groupFilters = array(
- new SqlDataFlow\SimpleWhereClause($groupTableAlias, 'group_id', 'IN', $group_ids),
- );
+ $whereClauses[] = $groupWhereClause;
- if ($dataFlow && $dataFlow instanceof SqlDataFlow) {
+ // Now look in the smartgroup group contact table
+ $smartGroupTableAlias = 'civicrm_group_contact_cache_'.$this->inputFieldSpecification->alias;
+ $smartGroupFilters = array(
+ new SqlDataFlow\SimpleWhereClause($smartGroupTableAlias, 'group_id', 'IN', $group_ids),
+ );
$tableAlias = $this->getTableAlias($dataFlow);
- $this->whereClause = new SqlDataFlow\InTableWhereClause(
+ $smartGroupWhereClause = new SqlDataFlow\InTableWhereClause(
'contact_id',
'civicrm_group_contact_cache',
- $groupTableAlias,
- $groupFilters,
+ $smartGroupTableAlias,
+ $smartGroupFilters,
$tableAlias,
$this->inputFieldSpecification->getName(),
$filter['op']
);
-
- $whereClauses[] = $this->whereClause;
+ $smartGroupWhereClause->setIndexStatement('USE INDEX (`UI_contact_group`)');
+ $whereClauses[] = $smartGroupWhereClause;
}
switch ($filter['op']) {
case 'IN':
--
GitLab