Merge pull request #23771 from eileenmcnaughton/cont_update

#3651 #1337 Fix failure to show last column data in output, spaces in names

CRM/Import/DataSource.php

@@ -67,6 +67,30 @@ abstract class CRM_Import_DataSource {
    */
   private $statuses = [];
 
+  /**
+   * Fields to select.
+   *
+   * @var array
+   */
+  private $selectFields;
+
+  /**
+   * @return array|null
+   */
+  public function getSelectFields(): ?array {
+    return $this->selectFields;
+  }
+
+  /**
+   * @param array $selectFields
+   *
+   * @return CRM_Import_DataSource
+   */
+  public function setSelectFields(array $selectFields): CRM_Import_DataSource {
+    $this->selectFields = $selectFields;
+    return $this;
+  }
+
   /**
    * Current row.
    *
@@ -537,13 +561,20 @@ abstract class CRM_Import_DataSource {
    * @throws \CRM_Core_Exception
    */
   private function instantiateQueryObject(): void {
-    $query = 'SELECT * FROM ' . $this->getTableName() . ' ' . $this->getStatusClause();
+    $query = 'SELECT ' . $this->getSelectClause() . ' FROM ' . $this->getTableName() . ' ' . $this->getStatusClause();
     if ($this->limit) {
       $query .= ' LIMIT ' . $this->limit . ($this->offset ? (' OFFSET ' . $this->offset) : NULL);
     }
     $this->queryResultObject = CRM_Core_DAO::executeQuery($query);
   }
 
+  /**
+   * @return string
+   */
+  private function getSelectClause(): string {
+    return $this->getSelectFields() ? implode(', ', $this->getSelectFields()) : '*';
+  }
+
   /**
    * Get the mapping of constants to database status codes.
    *

CRM/Import/DataSource/CSV.php

@@ -124,7 +124,6 @@ class CRM_Import_DataSource_CSV extends CRM_Import_DataSource {
     }
 
     $firstrow = fgetcsv($fd, 0, $fieldSeparator);
-    $result['column_headers'] = array_fill(0, count($firstrow), '');
     // create the column names from the CSV header or as col_0, col_1, etc.
     if ($headers) {
       //need to get original headers.
@@ -169,8 +168,9 @@ class CRM_Import_DataSource_CSV extends CRM_Import_DataSource {
     else {
       $columns = [];
       foreach ($firstrow as $i => $_) {
-        $columns[] = "col_$i";
+        $columns[] = "column_$i";
       }
+      $result['column_headers'] = $columns;
     }
 
     $table = CRM_Utils_SQL_TempTable::build()->setDurable();

CRM/Import/DataSource/SQL.php

@@ -92,7 +92,17 @@ class CRM_Import_DataSource_SQL extends CRM_Import_DataSource {
 
     $columnNames = [];
     while ($columnsResult->fetch()) {
-      $columnNames[] = $columnsResult->Field;
+      if (strpos($columnsResult->Field, ' ') !== FALSE) {
+        // Remove spaces as the Database object does this
+        // $keys = str_replace(array(".", " "), "_", array_keys($array));
+        // https://lab.civicrm.org/dev/core/-/issues/1337
+        $usableColumnName = str_replace(' ', '_', $columnsResult->Field);
+        CRM_Core_DAO::executeQuery('ALTER TABLE ' . $tableName . ' CHANGE `' . $columnsResult->Field . '` ' . $usableColumnName . ' ' . $columnsResult->Type);
+        $columnNames[] = $usableColumnName;
+      }
+      else {
+        $columnNames[] = $columnsResult->Field;
+      }
     }
 
     $this->addTrackingFieldsToTable($tableName);

CRM/Import/Forms.php

@@ -456,6 +456,35 @@ class CRM_Import_Forms extends CRM_Core_Form {
     return $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)->getRows();
   }
 
+  /**
+   * Get the datasource rows ready for csv output.
+   *
+   * @param array $statuses
+   * @param int $limit
+   *
+   * @return array
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   */
+  protected function getOutputRows($statuses = [], int $limit = 0) {
+    $statuses = (array) $statuses;
+    return $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)
+      ->setSelectFields(array_merge(['_id', '_status_message'], $this->getColumnHeaders()))
+      ->setStatuses($statuses)->getRows();
+  }
+
+  /**
+   * Get the column headers for the output csv.
+   *
+   * @return array
+   */
+  protected function getOutputColumnsHeaders(): array {
+    $headers = $this->getColumnHeaders();
+    array_unshift($headers, ts('Reason'));
+    array_unshift($headers, ts('Line Number'));
+    return $headers;
+  }
+
   /**
    * Get the number of rows with the specified status.
    *
@@ -492,16 +521,11 @@ class CRM_Import_Forms extends CRM_Core_Form {
 
     $form->getUserJob();
     $writer = Writer::createFromFileObject(new SplTempFileObject());
-    $headers = $form->getColumnHeaders();
-    if ($headers) {
-      array_unshift($headers, ts('Reason'));
-      array_unshift($headers, ts('Line Number'));
-      $writer->insertOne($headers);
-    }
-    $writer->addFormatter(['CRM_Import_Forms', 'reorderOutput']);
+    $headers = $form->getOutputColumnsHeaders();
+    $writer->insertOne($headers);
     // Note this might be more inefficient that iterating the result
     // set & doing insertOne - possibly something to explore later.
-    $writer->insertAll($form->getDataRows($status));
+    $writer->insertAll($form->getOutputRows($status));
 
     CRM_Utils_System::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0');
     CRM_Utils_System::setHttpHeader('Content-Description', 'File Transfer');
@@ -510,28 +534,6 @@ class CRM_Import_Forms extends CRM_Core_Form {
     CRM_Utils_System::civiExit();
   }
 
-  /**
-   * When outputting the row as a csv, more the last 2 rows to the start.
-   *
-   * This is because the id and status message fields are at the end. It may make sense
-   * to move them to the start later, when order code cleanup has happened...
-   *
-   * @param array $record
-   */
-  public static function reorderOutput(array $record): array {
-    $rowNumber = array_pop($record);
-    $message = array_pop($record);
-    // Also pop off the status - but we are not going to use this at this stage.
-    array_pop($record);
-    // Related entities
-    array_pop($record);
-    // Entity_id
-    array_pop($record);
-    array_unshift($record, $message);
-    array_unshift($record, $rowNumber);
-    return $record;
-  }
-
   /**
    * Get the url to download the relevant csv file.
    * @param string $status
@@ -597,7 +599,7 @@ class CRM_Import_Forms extends CRM_Core_Form {
     $this->_columnNames = $this->getColumnHeaders();
     $this->_dataValues = array_values($this->getDataRows([], 2));
     $this->assign('columnNames', $this->getColumnHeaders());
-    $this->assign('showColumnNames', $this->getSubmittedValue('skipColumnHeader'));
+    $this->assign('showColumnNames', $this->getSubmittedValue('skipColumnHeader') || $this->getSubmittedValue('dataSource') !== 'CRM_Import_DataSource');
     $this->assign('highlightedFields', $this->getHighlightedFields());
     $this->assign('columnCount', $this->_columnCount);
     $this->assign('dataValues', $this->_dataValues);