Merge pull request #15969 from eileenmcnaughton/utfmb8

Add utf8 to utfmb8 conversion api command

CRM/Core/BAO/SchemaHandler.php

@@ -796,4 +796,83 @@ MODIFY      {$columnName} varchar( $length )
     return $sql;
   }
 
+  /**
+   * Performs the utf8mb4 migration.
+   *
+   * @param bool $revert
+   *   Being able to revert if primarily for unit testing.
+   *
+   * @return bool
+   */
+  public static function migrateUtf8mb4($revert = FALSE) {
+    $newCharSet = $revert ? 'utf8' : 'utf8mb4';
+    $newCollation = $revert ? 'utf8_unicode_ci' : 'utf8mb4_unicode_ci';
+    $newBinaryCollation = $revert ? 'utf8_bin' : 'utf8mb4_bin';
+    $tables = [];
+    $dao = new CRM_Core_DAO();
+    $database = $dao->_database;
+    CRM_Core_DAO::executeQuery("ALTER DATABASE $database CHARACTER SET = $newCharSet COLLATE = $newCollation");
+    $dao = CRM_Core_DAO::executeQuery("SHOW TABLE STATUS WHERE Engine = 'InnoDB' AND Name LIKE 'civicrm\_%'");
+    while ($dao->fetch()) {
+      $tables[$dao->Name] = [
+        'Engine' => $dao->Engine,
+      ];
+    }
+    $dsn = defined('CIVICRM_LOGGING_DSN') ? DB::parseDSN(CIVICRM_LOGGING_DSN) : DB::parseDSN(CIVICRM_DSN);
+    $logging_database = $dsn['database'];
+    $dao = CRM_Core_DAO::executeQuery("SHOW TABLE STATUS FROM `$logging_database` WHERE Engine <> 'MyISAM' AND Name LIKE 'log\_civicrm\_%'");
+    while ($dao->fetch()) {
+      $tables["$logging_database.{$dao->Name}"] = [
+        'Engine' => $dao->Engine,
+      ];
+    }
+    foreach ($tables as $table => $param) {
+      $query = "ALTER TABLE $table";
+      $dao = CRM_Core_DAO::executeQuery("SHOW FULL COLUMNS FROM $table", [], TRUE, NULL, FALSE, FALSE);
+      $index = 0;
+      $params = [];
+      $tableCollation = $newCollation;
+      while ($dao->fetch()) {
+        if (!$dao->Collation || $dao->Collation === $newCollation || $dao->Collation === $newBinaryCollation) {
+          continue;
+        }
+        if (strpos($dao->Collation, 'utf8') !== 0) {
+          continue;
+        }
+
+        if (strpos($dao->Collation, '_bin') !== FALSE) {
+          $tableCollation = $newBinaryCollation;
+        }
+        else {
+          $tableCollation = $newCollation;
+        }
+        if ($dao->Null === 'YES') {
+          $null = 'NULL';
+        }
+        else {
+          $null = 'NOT NULL';
+        }
+        $default = '';
+        if ($dao->Default !== NULL) {
+          $index++;
+          $default = "DEFAULT %$index";
+          $params[$index] = [$dao->Default, 'String'];
+        }
+        elseif ($dao->Null === 'YES') {
+          $default = 'DEFAULT NULL';
+        }
+        $index++;
+        $params[$index] = [$dao->Comment, 'String'];
+        $query .= " MODIFY `{$dao->Field}` {$dao->Type} CHARACTER SET $newCharSet COLLATE $tableCollation $null $default {$dao->Extra} COMMENT %$index,";
+      }
+      $query .= " CHARACTER SET = $newCharSet COLLATE = $tableCollation";
+      if ($param['Engine'] === 'InnoDB') {
+        $query .= ' ROW_FORMAT = Dynamic';
+      }
+      // Disable i18n rewrite.
+      CRM_Core_DAO::executeQuery($query, $params, TRUE, NULL, FALSE, FALSE);
+    }
+    return TRUE;
+  }
+
 }

api/v3/System.php

@@ -384,6 +384,38 @@ function civicrm_api3_system_updatelogtables($params) {
   return civicrm_api3_create_success($updatedTablesCount);
 }
 
+/**
+ * Update log table structures.
+ *
+ * This updates the engine type if defined in the hook and changes the field type
+ * for log_conn_id to reflect CRM-18193.
+ *
+ * @param array $params
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ */
+function civicrm_api3_system_utf8conversion($params) {
+  if (CRM_Core_BAO_SchemaHandler::migrateUtf8mb4($params['is_revert'])) {
+    return civicrm_api3_create_success(1);
+  }
+  throw new API_Exception('Conversion failed');
+}
+
+/**
+ * Metadata for conversion function.
+ *
+ * @param array $params
+ */
+function _civicrm_api3_system_utf8conversion_spec(&$params) {
+  $params['is_revert'] = [
+    'title' => ts('Revert back from UTF8MB4 to UTF8?'),
+    'type' => CRM_Utils_Type::T_BOOLEAN,
+    'api.default' => FALSE,
+  ];
+}
+
 /**
  * Adjust Metadata for Flush action.
  *

tests/phpunit/api/v3/SystemTest.php

@@ -83,4 +83,19 @@ class api_v3_SystemTest extends CiviUnitTestCase {
     $this->assertEquals('UnitTests', $result['values'][0]['uf']);
   }
 
+  /**
+   * @throws \CRM_Core_Exception
+   */
+  public function testSystemUTFMB8Conversion() {
+    $this->callAPISuccess('System', 'utf8conversion', []);
+    $table = CRM_Core_DAO::executeQuery('SHOW CREATE TABLE civicrm_contact');
+    $table->fetch();
+    $this->assertStringEndsWith('DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC', $table->Create_Table);
+
+    $this->callAPISuccess('System', 'utf8conversion', ['is_revert' => 1]);
+    $table = CRM_Core_DAO::executeQuery('SHOW CREATE TABLE civicrm_contact');
+    $table->fetch();
+    $this->assertStringEndsWith('DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci ROW_FORMAT=DYNAMIC', $table->Create_Table);
+  }
+
 }