From b81df9e3bfcc52c38a39f46f3ee8663c5315f863 Mon Sep 17 00:00:00 2001
From: Jaap Jansma <jaap.jansma@civicoop.org>
Date: Thu, 19 Mar 2020 15:44:22 +0100
Subject: [PATCH] Improved export/import functionality
---
CHANGELOG.md | 1 +
CRM/Dataprocessor/Utils/Importer.php | 41 +++++++-
.../DataProcessor/DataSpecification/Utils.php | 1 +
.../Source/Event/EventSource.php | 67 ++++++++++++-
.../Source/Event/ParticipantSource.php | 99 ++++++++++++++++++-
.../Utils/AlterExportInterface.php | 31 ++++++
6 files changed, 234 insertions(+), 6 deletions(-)
create mode 100644 Civi/DataProcessor/Utils/AlterExportInterface.php
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e65247c9..cdcc687a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@
* Added Union Query Data Flow.
* Added a field specification for a fixed value
* Fixed #24
+* Improved export/import functionality.
# Version 1.3.0
diff --git a/CRM/Dataprocessor/Utils/Importer.php b/CRM/Dataprocessor/Utils/Importer.php
index 87d8b6ea..dd2f68b2 100644
--- a/CRM/Dataprocessor/Utils/Importer.php
+++ b/CRM/Dataprocessor/Utils/Importer.php
@@ -16,6 +16,8 @@ class CRM_Dataprocessor_Utils_Importer {
* @throws \Exception
*/
public static function export($data_processor_id) {
+ $factory = dataprocessor_get_factory();
+
$dataProcessor = civicrm_api3('DataProcessor', 'getsingle', array('id' => $data_processor_id));
unset($dataProcessor['id']);
unset($dataProcessor['status']);
@@ -26,6 +28,10 @@ class CRM_Dataprocessor_Utils_Importer {
foreach($sources['values'] as $i => $datasource) {
unset($datasource['id']);
unset($datasource['data_processor_id']);
+ $sourceClass = $factory->getDataSourceByName($datasource['type']);
+ if ($sourceClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $datasource = $sourceClass->alterExportData($datasource);
+ }
$dataProcessor['data_sources'][] = $datasource;
}
$filters = civicrm_api3('DataProcessorFilter', 'get', array('data_processor_id' => $data_processor_id, 'options' => array('limit' => 0)));
@@ -33,6 +39,10 @@ class CRM_Dataprocessor_Utils_Importer {
foreach($filters['values'] as $i => $filter) {
unset($filter['id']);
unset($filter['data_processor_id']);
+ $filterClass = $factory->getFilterByName($filter['type']);
+ if ($filterClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $filter = $filterClass->alterExportData($filter);
+ }
$dataProcessor['filters'][] = $filter;
}
$fields = civicrm_api3('DataProcessorField', 'get', array('data_processor_id' => $data_processor_id, 'options' => array('limit' => 0)));
@@ -40,6 +50,10 @@ class CRM_Dataprocessor_Utils_Importer {
foreach($fields['values'] as $i => $field) {
unset($field['id']);
unset($field['data_processor_id']);
+ $fieldClass = $factory->getOutputHandlerByName($field['type']);
+ if ($fieldClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $field = $filterClass->alterExportData($field);
+ }
$dataProcessor['fields'][] = $field;
}
$outputs = $outputs = civicrm_api3('DataProcessorOutput', 'get', array('data_processor_id' => $data_processor_id, 'options' => array('limit' => 0)));
@@ -47,6 +61,10 @@ class CRM_Dataprocessor_Utils_Importer {
foreach($outputs['values'] as $i => $output) {
unset($output['id']);
unset($output['data_processor_id']);
+ $outputClass = $factory->getOutputByName($output['type']);
+ if ($outputClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $output = $outputClass->alterExportData($output);
+ }
$dataProcessor['outputs'][] = $output;
}
@@ -148,6 +166,7 @@ class CRM_Dataprocessor_Utils_Importer {
* @throws \Exception
*/
public static function importDataProcessor($data, $filename, $data_processor_id, $status) {
+ $factory = dataprocessor_get_factory();
$params = $data;
unset($params['data_sources']);
unset($params['outputs']);
@@ -174,6 +193,10 @@ class CRM_Dataprocessor_Utils_Importer {
CRM_Dataprocessor_BAO_DataProcessorOutput::deleteWithDataProcessorId($id);
foreach($data['data_sources'] as $data_source) {
+ $sourceClass = $factory->getDataSourceByName($data_source['type']);
+ if ($sourceClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $data_source = $sourceClass->alterImportData($data_source);
+ }
$params = $data_source;
$params['data_processor_id'] = $id;
$params['debug'] = 1;
@@ -184,16 +207,28 @@ class CRM_Dataprocessor_Utils_Importer {
}
}
foreach($data['filters'] as $filter) {
+ $filterClass = $factory->getFilterByName($filter['type']);
+ if ($filterClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $filter = $filterClass->alterImportData($filter);
+ }
$params = $filter;
$params['data_processor_id'] = $id;
civicrm_api3('DataProcessorFilter', 'create', $params);
}
foreach($data['fields'] as $field) {
+ $fieldClass = $factory->getFilterByName($field['type']);
+ if ($fieldClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $field = $fieldClass->alterImportData($field);
+ }
$params = $field;
$params['data_processor_id'] = $id;
civicrm_api3('DataProcessorField', 'create', $params);
}
foreach($data['outputs'] as $output) {
+ $outputClass = $factory->getFilterByName($output['type']);
+ if ($outputClass instanceof \Civi\DataProcessor\Utils\AlterExportInterface) {
+ $output = $outputClass->alterImportData($output);
+ }
$params = $output;
$params['data_processor_id'] = $id;
civicrm_api3('DataProcessorOutput', 'create', $params);
@@ -222,9 +257,9 @@ class CRM_Dataprocessor_Utils_Importer {
// Remove all data processors which are in code or overridden but not imported
$removeSql = "
- SELECT id, name, status
- FROM civicrm_data_processor
- WHERE status IN (".CRM_Dataprocessor_Status::STATUS_IN_CODE.", ".CRM_Dataprocessor_Status::STATUS_OVERRIDDEN.")
+ SELECT id, name, status
+ FROM civicrm_data_processor
+ WHERE status IN (".CRM_Dataprocessor_Status::STATUS_IN_CODE.", ".CRM_Dataprocessor_Status::STATUS_OVERRIDDEN.")
AND source_file IS NOT NULL";
if (count($importedIds)) {
$removeSql .= " AND id NOT IN (".implode($importedIds, ",").")";
diff --git a/Civi/DataProcessor/DataSpecification/Utils.php b/Civi/DataProcessor/DataSpecification/Utils.php
index 8258012a..4f39ca20 100644
--- a/Civi/DataProcessor/DataSpecification/Utils.php
+++ b/Civi/DataProcessor/DataSpecification/Utils.php
@@ -26,6 +26,7 @@ class Utils {
if (in_array($field['name'], $fieldsToSkip)) {
continue;
}
+
$type = \CRM_Utils_Type::typeToString($field['type']);
$options = $daoClass::buildOptions($field['name']);
$alias = $aliasPrefix.$field['name'];
diff --git a/Civi/DataProcessor/Source/Event/EventSource.php b/Civi/DataProcessor/Source/Event/EventSource.php
index 31048dfd..be0126cd 100644
--- a/Civi/DataProcessor/Source/Event/EventSource.php
+++ b/Civi/DataProcessor/Source/Event/EventSource.php
@@ -14,9 +14,10 @@ use Civi\DataProcessor\DataSpecification\FieldSpecification;
use Civi\DataProcessor\DataSpecification\Utils;
use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
+use Civi\DataProcessor\Utils\AlterExportInterface;
use CRM_Dataprocessor_ExtensionUtil as E;
-class EventSource extends AbstractCivicrmEntitySource {
+class EventSource extends AbstractCivicrmEntitySource implements AlterExportInterface {
/**
* @var SqlTableDataFlow
@@ -41,6 +42,70 @@ class EventSource extends AbstractCivicrmEntitySource {
return 'civicrm_event';
}
+ /**
+ * Function to alter the export data.
+ * E.g. use this to convert ids to names
+ *
+ * @param array $data
+ *
+ * @return array
+ */
+ public function alterExportData($data) {
+ if (isset($data['configuration']) && is_array($data['configuration'])) {
+ $configuration = $data['configuration'];
+
+ if (isset($configuration['filter']['event_type_id'])) {
+ $event_types = [];
+ foreach ($configuration['filter']['event_type_id']['value'] as $event_type_id) {
+ try {
+ $event_types[] = civicrm_api3('OptionValue', 'getvalue', [
+ 'option_group_id' => 'event_type',
+ 'value' => $event_type_id,
+ 'return' => 'name'
+ ]);
+ } catch (\CiviCRM_API3_Exception $ex) {
+ $event_types[] = $event_type_id;
+ }
+ }
+ $configuration['filter']['event_type_id']['value'] = $event_types;
+ }
+
+ $data['configuration'] = $configuration;
+ }
+ return $data;
+ }
+
+ /**
+ * Function to alter the export data.
+ * E.g. use this to convert names to ids
+ *
+ * @param array $data
+ *
+ * @return array
+ */
+ public function alterImportData($data) {
+ if (isset($data['configuration']) && is_array($data['configuration'])) {
+ $configuration = $data['configuration'];
+ if (isset($configuration['filter']['event_type_id'])) {
+ $event_types = [];
+ foreach ($configuration['filter']['event_type_id']['value'] as $event_type_id) {
+ try {
+ $event_types[] = civicrm_api3('OptionValue', 'getvalue', [
+ 'option_group_id' => 'event_type',
+ 'name' => $event_type_id,
+ 'return' => 'value'
+ ]);
+ } catch (\CiviCRM_API3_Exception $ex) {
+ $event_types[] = $event_type_id;
+ }
+ }
+ $configuration['filter']['event_type_id']['value'] = $event_types;
+ }
+ $data['configuration'] = $configuration;
+ }
+ return $data;
+ }
+
/**
* Load the fields from this entity.
*
diff --git a/Civi/DataProcessor/Source/Event/ParticipantSource.php b/Civi/DataProcessor/Source/Event/ParticipantSource.php
index e9f10238..b0080db8 100644
--- a/Civi/DataProcessor/Source/Event/ParticipantSource.php
+++ b/Civi/DataProcessor/Source/Event/ParticipantSource.php
@@ -8,9 +8,10 @@ namespace Civi\DataProcessor\Source\Event;
use Civi\DataProcessor\Source\AbstractCivicrmEntitySource;
+use Civi\DataProcessor\Utils\AlterExportInterface;
use CRM_Dataprocessor_ExtensionUtil as E;
-class ParticipantSource extends AbstractCivicrmEntitySource {
+class ParticipantSource extends AbstractCivicrmEntitySource implements AlterExportInterface {
/**
* Returns the entity name
@@ -30,4 +31,98 @@ class ParticipantSource extends AbstractCivicrmEntitySource {
return 'civicrm_participant';
}
-}
\ No newline at end of file
+ /**
+ * Function to alter the export data.
+ * E.g. use this to convert ids to names
+ *
+ * @param array $data
+ *
+ * @return array
+ */
+ public function alterExportData($data) {
+ if (isset($data['configuration']) && is_array($data['configuration'])) {
+ $configuration = $data['configuration'];
+
+ if (isset($configuration['filter']['status_id'])) {
+ $status_ids = [];
+ foreach ($configuration['filter']['status_id']['value'] as $status_id) {
+ try {
+ $status_ids[] = civicrm_api3('ParticipantStatusType', 'getvalue', [
+ 'id' => $status_id,
+ 'return' => 'name'
+ ]);
+ } catch (\CiviCRM_API3_Exception $ex) {
+ $status_ids[] = $status_id;
+ }
+ }
+ $configuration['filter']['status_id']['value'] = $status_ids;
+ }
+
+ if (isset($configuration['filter']['role_id'])) {
+ $roles = [];
+ foreach ($configuration['filter']['role_id']['value'] as $role_id) {
+ try {
+ $roles[] = civicrm_api3('OptionValue', 'getvalue', [
+ 'option_group_id' => 'participant_role',
+ 'value' => $role_id,
+ 'return' => 'name'
+ ]);
+ } catch (\CiviCRM_API3_Exception $ex) {
+ $roles[] = $role_id;
+ }
+ }
+ $configuration['filter']['role_id']['value'] = $roles;
+ }
+ $data['configuration'] = $configuration;
+ }
+ return $data;
+ }
+
+ /**
+ * Function to alter the export data.
+ * E.g. use this to convert names to ids
+ *
+ * @param array $data
+ *
+ * @return array
+ */
+ public function alterImportData($data) {
+ if (isset($data['configuration']) && is_array($data['configuration'])) {
+ $configuration = $data['configuration'];
+ if (isset($configuration['filter']['status_id'])) {
+ $status_ids = [];
+ foreach ($configuration['filter']['status_id']['value'] as $status_name) {
+ try {
+ $status_ids[] = civicrm_api3('ParticipantStatusType', 'getvalue', [
+ 'name' => $status_name,
+ 'return' => 'id'
+ ]);
+ } catch (\CiviCRM_API3_Exception $ex) {
+ $status_ids[] = $status_name;
+ }
+ }
+ $configuration['filter']['status_id']['value'] = $status_ids;
+ }
+
+ if (isset($configuration['filter']['role_id'])) {
+ $roles = [];
+ foreach ($configuration['filter']['role_id']['value'] as $role_name) {
+ try {
+ $roles[] = civicrm_api3('OptionValue', 'getvalue', [
+ 'option_group_id' => 'participant_role',
+ 'name' => $role_name,
+ 'return' => 'value'
+ ]);
+ } catch (\CiviCRM_API3_Exception $ex) {
+ $roles[] = $role_name;
+ }
+ }
+ $configuration['filter']['role_id']['value'] = $roles;
+ }
+ $data['configuration'] = $configuration;
+ }
+ return $data;
+ }
+
+
+}
diff --git a/Civi/DataProcessor/Utils/AlterExportInterface.php b/Civi/DataProcessor/Utils/AlterExportInterface.php
new file mode 100644
index 00000000..524e18ea
--- /dev/null
+++ b/Civi/DataProcessor/Utils/AlterExportInterface.php
@@ -0,0 +1,31 @@
+<?php
+/**
+ * @author Jaap Jansma <jaap.jansma@civicoop.org>
+ * @license AGPL-3.0
+ */
+
+namespace Civi\DataProcessor\Utils;
+
+interface AlterExportInterface {
+
+ /**
+ * Function to alter the export data.
+ * E.g. use this to convert ids to names
+ *
+ * @param array $data
+ *
+ * @return array
+ */
+ public function alterExportData($data);
+
+ /**
+ * Function to alter the export data.
+ * E.g. use this to convert names to ids
+ *
+ * @param array $data
+ *
+ * @return array
+ */
+ public function alterImportData($data);
+
+}
--
GitLab