initial load

CRM/Pcvmigratie/CsvFile.php

+<?php
+use CRM_Pcvmigratie_ExtensionUtil as E;
+
+/**
+ * Class voor Pax Christi Vlaanderen migratie CSV import
+ *
+ * @author Erik Hommel (CiviCooP) <erik.hommel@civicoop.org>
+ * @date 13 Oct 2021
+ * @license AGPL-3.0
+ */
+class CRM_Pcvmigratie_CsvFile {
+
+  private $_separator = NULL;
+  private $_fileName = NULL;
+  private $_usesHeaders = NULL;
+  private $_rowNumber = NULL;
+  private $_headers = [];
+  public $csv = NULL;
+
+  /**
+   * Method to initialize
+   *
+   * @param $fileName
+   * @param bool $usesHeaders
+   * @param string $separator
+   */
+  public function initialize($fileName, $usesHeaders = FALSE, $separator = "~") {
+    $this->_fileName = $fileName;
+    $this->_separator = $separator;
+    $this->_usesHeaders = $usesHeaders;
+    $this->_rowNumber = 0;
+  }
+  /**
+   * Method to get the headers (and numbers if none found)
+   *
+   * @throws Exception if file can not be opened
+   */
+  public function getHeaders() {
+    $headers = [];
+    $i = 0;
+    $this->open();
+    if ($this->_usesHeaders) {
+      $data = fgetcsv($this->csv, 0, $this->_separator);
+      foreach ($data as $key => $value) {
+        if ($value) {
+          $headers[] = $value;
+          $i++;
+        }
+        else {
+          $i++;
+          $headers[] = "csv column " . $i;
+        }
+      }
+    }
+    else {
+      $data = fgetcsv($this->csv, 0, $this->_separator);
+      $count = count($data);
+      while ($i < $count) {
+        $i++;
+        $headers[] = "csv column " . $i;
+      }
+    }
+    $this->close();
+    return $headers;
+  }
+
+  /**
+   * Method to open the file for read
+   *
+   * @throws Exception when file can not be opened
+   * @return bool
+   */
+  public function open() {
+    $this->csv = fopen($this->_fileName, 'r');
+    if ($this->csv) {
+      return TRUE;
+    }
+    return FALSE;
+  }
+
+  /**
+   * Method to read the next line and return an array with data
+   *
+   * @return array|bool
+   */
+  public function readNext() {
+    $this->_rowNumber++;
+    // if row 1 complete mapping and if headers, skip
+    if ($this->_rowNumber == 1) {
+      $data = fgetcsv($this->csv, 0, $this->_separator);
+      $this->setHeaders($data);
+      if (!$this->_usesHeaders) {
+        rewind($this->csv);
+      }
+    }
+    $data = fgetcsv($this->csv, 0, $this->_separator);
+    $this->sanitizeData($data);
+    if ($data) {
+      return $data;
+    }
+    return FALSE;
+  }
+
+  /**
+   * Sanitize row: empty data if it contains invalid content
+   *
+   * @param $data
+   */
+  private function sanitizeData(&$data) {
+    if (!empty($data)) {
+      foreach ($data as $key => $value) {
+        $data[$key] = "";
+      }
+      $onlyBlanks = TRUE;
+      foreach ($data as $value) {
+        if (!empty($value)) {
+          $onlyBlanks = FALSE;
+        }
+      }
+      if ($onlyBlanks) {
+        $data = [];
+      }
+    }
+  }
+
+  /**
+   * Method to close the file
+   */
+  public function close() {
+    fclose($this->csv);
+  }
+
+  /**
+   * Method to set the headers of the csv file
+   *
+   * @param $headers
+   */
+  private function setHeaders($headers) {
+    $this->_headers = $headers;
+  }
+
+
+
+}

CRM/Pcvmigratie/Form/CsvSelect.php

+<?php
+
+use CRM_Pcvmigratie_ExtensionUtil as E;
+
+/**
+ * Form controller class
+ *
+ * @see https://docs.civicrm.org/dev/en/latest/framework/quickform/
+ */
+class CRM_Pcvmigratie_Form_CsvSelect extends CRM_Core_Form {
+  /**
+   * Overridden parent method to build the form
+   */
+  public function buildQuickForm() {
+    $this->add('file', 'csv_file', E::ts('Migratie bestand (CSV)'), [], TRUE);
+    $this->addRule('csv_file', E::ts('Bestand moet CSV formaat hebben'), 'utf8File');
+    $this->addRule('csv_file', E::ts('Er moet een geldig bestand geladen worden.'), 'uploadedfile');
+    $this->add('advcheckbox', 'first_row_headers', E::ts('Bevat de eerste rij kolomkoppen?'));
+    $this->add('select', 'separator_id', E::ts('Scheidingsteken voor velden'), Civi::service('pcvmigratie')->getSeparatorList() , TRUE);
+    $this->addButtons([
+      ['type' => 'next', 'name' => E::ts('Next'), 'isDefault' => TRUE],
+      ['type' => 'cancel', 'name' => E::ts('Cancel')],
+    ]);
+    parent::buildQuickForm();
+  }
+
+  /**
+   * Overridden parent method to set default values
+   *
+   * @return array
+   */
+  public function setDefaultValues()   {
+    $defaults = [];
+    $defaults['first_row_headers'] = TRUE;
+    return $defaults;
+  }
+
+  /**
+   * Overridden parent method to add validation rules
+   *
+   */
+  public function addRules() {
+    $this->addFormRule(['CRM_Pcvmigratie_CsvSelect', 'validateCsvFile']);
+  }
+
+  /**
+   * Overridden parent method to prepare the form
+   */
+  public function preProcess() {
+    CRM_Utils_System::setTitle("Pax Christi Vlaanderen - CiviCRM migratie - kies CSV bestand");
+    parent::preProcess(); // TODO: Change the autogenerated stub
+  }
+
+  /**
+   * Overridden parent method to process the form
+   */
+  public function postProcess() {
+    // send csv data to upload table
+    $table = $this->populateTable(Civi::service('pcvmigratie')->getSeparator($this->_submitValues['separator_id']));
+    if ($table) {
+      $mapping = $this->getMapping();
+      if ($mapping) {
+        CRM_Pcvmigratie_Migratie::migreer($this->_tableName);
+      }
+    }
+    parent::postProcess();
+  }
+
+  /**
+   * Method to get only bit before . for filename
+   *
+   * @param $fileName
+   * @return string
+   */
+  private function sanitizeFileName($fileName) {
+    $fileName = trim(strtolower($fileName));
+    $parts = explode(".", $fileName);
+    $fileName = $parts[0];
+    $invalids = [" ", "/", "\\", "#", "~", "`", "@", "%", "^", "&", "*", "(", ")", "{", "}", "[", "]", "<", ">", "?", ",", ":", ";"];
+    foreach ($invalids as $invalid) {
+      $fileName = str_replace($invalid, "_", $fileName);
+    }
+    if (strlen($fileName) > 53) {
+      $fileName = substr($fileName, 0, 53);
+    }
+    return $fileName;
+  }
+
+  /**
+   * Method to populate table with data from selected csv file
+   *
+   * @param $separator
+   * @return bool
+   * @throws Exception
+   */
+  private function populateTable($separator) {
+    $table = new CRM_Pcvmigratie_ImportTable();
+    $table->generateTableName($this->sanitizeFileName($this->_submitFiles['csv_file']['name']));
+    $this->_tableName = $table->getTableName();
+    $csv = new CRM_Pcvmigratie_CsvFile();
+    $csv->initialize($this->_submitFiles['csv_file']['tmp_name'], $this->_submitValues['first_row_headers'], $separator);
+    $newTable = $table->createTable($csv->getHeaders());
+    if ($newTable) {
+      $csv->open();
+      while (!feof($csv->csv)) {
+        $data = $csv->readNext();
+        if (!empty($data)) {
+          $table->writeRow($data);
+        }
+      }
+      $csv->close();
+      return TRUE;
+    }
+    else {
+      CRM_Core_Session::setStatus('Kon geen MySQL tabel aanmaken voor migratie, check of je het juiste verldscheidingsteken gebruikt en of de data lijkt te kloppen en geen rare tekens bevat.', 'Migratie van bestand mislukt', 'error');
+      return FALSE;
+    }
+  }
+  /**
+   * Method to validate if file has the correct ext and can be opened
+   *
+   * @param $fields
+   * @param $files
+   * @return bool|array
+   * @throws
+   */
+  public static function validateCsvFile($fields, $files) {
+    $ext = pathinfo($files['csv_file']['name'], PATHINFO_EXTENSION);
+    if ($ext != "csv" && $ext != "txt") {
+      $errors['csv_file'] = "Alleen CSV bestanden kunnen gemigreerd worden.";
+      return $errors;
+    }
+    if (!isset($files['csv_file']['tmp_name'])) {
+      $errors['csv_file'] = "Er lijkt geen bestand geladen te zijn.";
+      return $errors;
+    }
+    $check = fopen($files['csv_file']['tmp_name'], 'r');
+    if (!$check) {
+      $errors['csv_file'] = "Kan het CSV bestand niet openen.";
+      return $errors;
+    }
+    fclose($check);
+    return TRUE;
+  }
+
+}

CRM/Pcvmigratie/ImportTable.php

+<?php
+
+use CRM_Pcvmigratie_ExtensionUtil as E;
+/**
+ * Class voor de tabel uit de csv import (Pax Christi Vlaanderen migratie)
+ *
+ * @author Erik Hommel (CiviCooP) <erik.hommel@civicoop.org>
+ * @date 13 Oct 2021
+ * @license AGPL-3.0
+ */
+class CRM_Pcvmigratie_ImportTable {
+
+  private $_columnNames;
+  private $_tableName;
+
+  /**
+   * Method to generate the name of the table
+   *
+   * @param string $fileName
+   */
+  public function generateTableName(string $fileName) {
+    $tableName = strtolower(substr($fileName,0,48)) . "_" . date('YmdHis');
+    // check if table already exists and if so, and random number to name
+    if (CRM_Core_DAO::checkTableExists($tableName)) {
+      $tableName .= "_" . rand(0,99);
+    }
+    $this->_tableName = $tableName;
+  }
+
+  /**
+   * Method to set the table name
+   *
+   * @param $tableName
+   */
+  public function setTableName($tableName) {
+    $this->_tableName = $tableName;
+  }
+
+  /**
+   * Method to get the table name
+   *
+   * @return mixed
+   */
+  public function getTableName() {
+    return $this->_tableName;
+  }
+
+  /**
+   * Method to create the table
+   *
+   * @param $headers
+   * @return bool
+   */
+  public function createTable($headers) {
+    if (empty($headers)) {
+      return FALSE;
+    }
+    // check if the number of headers agree with a possible preset mapping
+    if ($this->checkValidHeaders($headers)) {
+      $this->_columnNames = [];
+      $columns = ['id int UNSIGNED NOT NULL AUTO_INCREMENT'];
+      // create a varchar column for each header
+      foreach ($headers as $header) {
+        $columns[] = $this->sanitizeColumnName($header) . " VARCHAR(256)";
+      }
+      // if table exists, drop old one
+      if (CRM_Core_DAO::checkTableExists($this->_tableName)) {
+        $this->destroy();
+      }
+      $create = "CREATE TABLE " . $this->_tableName . " (" . implode(", ", $columns) .", PRIMARY KEY (id) )";
+      CRM_Core_DAO::executeQuery($create);
+      return TRUE;
+    }
+    return FALSE;
+  }
+
+  /**
+   * Method to check if the number of headers are correct comparing with the potential preset mapping
+   *
+   * @param $headers
+   * @return bool
+   */
+  private function checkValidHeaders($headers) {
+    $mapping = CRM_Aivlcsvimports_ImportType::getPresetMapping($this->_importType);
+    if ($mapping) {
+      $countMapping = count(get_object_vars($mapping));
+      $countHeaders = count($headers);
+      if ($countHeaders == 1 && $countMapping != 1) {
+        return FALSE;
+      }
+    }
+    return TRUE;
+  }
+
+  /**
+   * Method to sanitize the header into a column name
+   *
+   * @param $header
+   * @return string
+   */
+  private function sanitizeColumnName($header) {
+    $invalids = [" ", "/", "\\", "#", "~", "`", "@", "%", "^", "&", "*", "(", ")", "{", "}", "[", "]", "<", ">", "?", ",", ":", ";"];
+    $nameParts = [];
+    $x = 1;
+    $parts = explode(" ", $header);
+    foreach ($parts as $key => $value) {
+      $value = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $value);
+      $nameParts[] = strtolower($value);
+    }
+    $columnName = implode("_", $nameParts);
+    if (strlen($columnName) > 64) {
+      $columnName = substr($columnName,0, 64);
+    }
+    // check if column name does not contain invalid characters
+    foreach ($invalids as $invalid) {
+      $columnName = str_replace($invalid, "_", $columnName);
+    }
+    // ID is already taken as column_name
+    if ($columnName == "id") {
+      $columnName = "id" . $x;
+      $x++;
+    }
+    if (isset($this->_columnNames[$columnName])) {
+      $columnName .= $x;
+    }
+    $this->_columnNames[$columnName] = $header;
+    return  $columnName;
+  }
+
+  /**
+   * Function to write row in table using data from incoming array
+   *
+   * @param $data
+   * @return bool
+   */
+  public function writeRow($data) {
+    if (empty($data)) {
+      return FALSE;
+    }
+    if (!is_array($data)) {
+      $data = [$data];
+    }
+    $i = 0;
+    $elements = [];
+    $queryParams = [];
+    foreach ($data as $key => $value) {
+      $value = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $value);
+      $i++;
+      $elements[] = "%" . $i;
+      $queryParams[$i] = [$value, "String"];
+    }
+    $columnNames = [];
+    foreach ($this->_columnNames as $columnName => $header) {
+      $columnNames[] = $columnName;
+    }
+    $query = "INSERT INTO ". $this->_tableName . " (" . implode(",", $columnNames) . ") VALUES (" . implode(",", $elements) . ")";
+    try {
+      CRM_Core_DAO::executeQuery($query,$queryParams);
+    }
+    catch (Exception $ex) {
+      CRM_Core_Session::setStatus("Could not write a record for a csv file row, ask your system administrator to check the CiviCRM error log", "Could not add row from csv file", "error");
+      Civi::log()->error("Could not write table record for csv file row in " . __METHOD__ . ", error message: " . $ex->getMessage());
+      return FALSE;
+    }
+    return TRUE;
+  }
+
+  /**
+   * Method to get all rows
+   *
+   * @return array|false
+   */
+  public function getAllRows($tableName) {
+    if ($tableName) {
+      $result = [];
+      $query = "SELECT * FROM " . $tableName;
+      $dao = CRM_Core_DAO::executeQuery($query);
+      while ($dao->fetch()) {
+        $result[] = Civi::service('aivlcsvimports')->moveDaoToArray($dao);
+      }
+      return $result;
+    }
+    return FALSE;
+  }
+
+  /**
+   * Method to get the column names
+   *
+   * @return array
+   */
+  public function getColumnNames() {
+    $result = [];
+    if ($this->_tableName) {
+      $dao = CRM_Core_DAO::executeQuery("SHOW COLUMNS FROM " . $this->_tableName);
+      while ($dao->fetch()) {
+        if ($dao->Field != "id") {
+          $result[] = $dao->Field;
+        }
+      }
+    }
+    return $result;
+  }
+
+  /**
+   * Method to drop table
+   */
+  public function destroy() {
+    if ($this->_tableName) {
+      CRM_Core_DAO::executeQuery("DROP TABLE IF EXISTS " . $this->_tableName);
+    }
+  }
+
+  /**
+   * Method to appy mapping
+   *
+   * @param $tableName
+   * @param $row
+   * @return array
+   */
+  public function applyMapping($importType, $tableName, $row) {
+    $result = [];
+    $mapping = CRM_Aivlcsvimports_ImportType::getPresetMapping($importType);
+    if (!$mapping) {
+      $mapping = CRM_Aivlcsvimports_ImportType::getMapping($tableName);
+    }
+    foreach ($mapping as $field => $target) {
+      if (isset($row[$target]) && $row[$target]) {
+        $result[$field] = $row[$target];
+      }
+    }
+    return $result;
+  }
+
+
+}
+

CRM/Pcvmigratie/ImportType.php

+<?php
+
+use CRM_Aivlcsvimports_ExtensionUtil as E;
+/**
+ * Class for dealing with the type of import
+ *
+ * @author Erik Hommel (CiviCooP) <erik.hommel@civicoop.org>
+ * @date 27 Feb 2021
+ * @license AGPL-3.0
+ */
+class CRM_Aivlcsvimports_ImportType {
+
+  private $_importTypes = [];
+
+  /**
+   * CRM_Aivlcsvimports_ImportType constructor.
+   */
+  public function __construct() {
+    $this->_importTypes =  [
+      'petition',
+      'tm_mindwize',
+      ];
+  }
+
+  /**
+   * Method to check if importType is valid
+   *
+   * @param $importType
+   * @return bool
+   */
+  public static function isValidImportType($key) {
+    $importType = new CRM_Aivlcsvimports_ImportType();
+    if (in_array($key, $importType->_importTypes)) {
+      return TRUE;
+    }
+    return FALSE;
+  }
+
+  /**
+   * Method to get the valid import types
+   * @return string[]
+   */
+  public static function getValidImportTypes() {
+    $importType = new CRM_Aivlcsvimports_ImportType();
+    return $importType->_importTypes;
+  }
+
+  /**
+   * Method to check if the data row contains valid data depending on import type
+   *
+   * @param $importType
+   * @param $data
+   * @return false
+   */
+  public static function containsValidData($importType, $data) {
+    $methodName = "containsValid" . ucfirst(strtolower($importType)) . "Data";
+    if (method_exists(__CLASS__, $methodName)) {
+      $importType = new CRM_Aivlcsvimports_ImportType();
+      return $importType->$methodName($data);
+    }
+    return FALSE;
+  }
+
+  /**
+   * Method to check if the data row contains valid petition data
+   *
+   * @param $data
+   * @return bool
+   */
+  private function containsValidPetitionData($data) {
+    $mandatories = [
+      'first name',
+      'last name' ,
+      'email',
+      'campaign ID'];
+    foreach ($mandatories as $mandatory) {
+      if (!isset($data[$mandatory]) || empty($data[$mandatory])) {
+        return FALSE;
+      }
+    }
+    return TRUE;
+