translation#67 - Define "Translation" entity. Add during installation/upgrade.

This creates an entity, `Translation` (`civicrm_translation`), to represent a single translated database value. Loosely speaking,
any field in the database can be designated as translatable -- and then it will be permitted to store values like:

```sql
INSERT INTO civicrm_translation (entity_table, entity_id, entity_field, language, string)
VALUES ('civicrm_event', 100, 'title', 'fr_FR', 'La nouvelle chaine')
```

This is based on a `civi-data-translate` strings table, but with some changes:

* Entity names are usually singular, but `String` is conflicted. I previously used hybrid
  String/Strings (depending on context), but we negotiated `Translation` on tcon.
* The language only needs 5 characters (NN_nn).
* Consolidated `bool is_active` and `bool is_default` into one `int status_id`.
* Added indexing
* Mark dynamic foreign key

This commit includes the BAO with some of the backing-methods required for
API exposure.  However, the API won't really work until we have the
validation-values event, so the API has been kicked to a subsequent PR.

The list of translatable entities/fields will be signficant because it will
determine when/how to redirect data in API calls.  This patch does not
commit to specific translatable fields - but it does provide a hook to
determine them.

When the API PR becomes unblocked, it will include test-coverage that hits the API, BAO, and hook.

CRM/Core/BAO/Translation.php

+<?php
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+/**
+ *
+ * @package CRM
+ * @copyright CiviCRM LLC https://civicrm.org/licensing
+ */
+class CRM_Core_BAO_Translation extends CRM_Core_DAO_Translation {
+
+  /**
+   * Get a list of valid statuses for translated-strings.
+   *
+   * @return string[]
+   */
+  public static function getStatuses($context = NULL) {
+    $options = [
+      ['id' => 1, 'name' => 'active', 'label' => ts('Active')],
+      ['id' => 2, 'name' => 'draft', 'label' => ts('Draft')],
+    ];
+    return self::formatPsuedoconstant($context, $options);
+  }
+
+  /**
+   * Get a list of tables with translatable strings.
+   *
+   * @return string[]
+   *   Ex: ['civicrm_event' => 'civicrm_event']
+   */
+  public static function getEntityTables() {
+    if (!isset(Civi::$statics[__CLASS__]['allTables'])) {
+      $tables = array_keys(self::getTranslatedFields());
+      Civi::$statics[__CLASS__]['allTables'] = array_combine($tables, $tables);
+    }
+    return Civi::$statics[__CLASS__]['allTables'];
+  }
+
+  /**
+   * Get a list of fields with translatable strings.
+   *
+   * @return string[]
+   *   Ex: ['title' => 'title', 'description' => 'description']
+   */
+  public static function getEntityFields() {
+    if (!isset(Civi::$statics[__CLASS__]['allFields'])) {
+      $allFields = [];
+      foreach (self::getTranslatedFields() as $columns) {
+        foreach ($columns as $column => $sqlExpr) {
+          $allFields[$column] = $column;
+        }
+      }
+      Civi::$statics[__CLASS__]['allFields'] = $allFields;
+    }
+    return Civi::$statics[__CLASS__]['allFields'];
+  }
+
+  /**
+   * Given a constant list of of id/name/label options, convert to the
+   * format required by pseudoconstant.
+   *
+   * @param string|NULL $context
+   * @param array $options
+   *   List of options, each as a record of id+name+label.
+   *   Ex: [['id' => 123, 'name' => 'foo_bar', 'label' => 'Foo Bar']]
+   *
+   * @return array|false
+   */
+  private static function formatPsuedoconstant($context, array $options) {
+    // https://docs.civicrm.org/dev/en/latest/framework/pseudoconstant/#context
+    $key = ($context === 'match') ? 'name' : 'id';
+    $value = ($context === 'validate') ? 'name' : 'label';
+    return array_combine(array_column($options, $key), array_column($options, $value));
+  }
+
+  /**
+   * @return array
+   *   List of data fields to translate, organized by table and column.
+   *   Omitted/unlisted fields are not translated. Any listed field may be translated.
+   *   Values should be TRUE.
+   *   Ex: $fields['civicrm_event']['summary'] = TRUE
+   */
+  public static function getTranslatedFields() {
+    $key = 'translatedFields';
+    $cache = Civi::cache('fields');
+    if (($r = $cache->get($key)) !== NULL) {
+      return $r;
+    }
+
+    $f = [];
+    \CRM_Utils_Hook::translateFields($f);
+
+    // Future: Assimilate defaults originating in XML (incl extension-entities)
+    // e.g. CRM_Core_I18n_SchemaStructure::columns() will grab core fields
+
+    $cache->set($key, $f);
+    return $f;
+  }
+
+}

CRM/Core/DAO/AllCoreTables.data.php

@@ -47,6 +47,11 @@ return [
     'class' => 'CRM_Core_DAO_SystemLog',
     'table' => 'civicrm_system_log',
   ],
+  'CRM_Core_DAO_Translation' => [
+    'name' => 'Translation',
+    'class' => 'CRM_Core_DAO_Translation',
+    'table' => 'civicrm_translation',
+  ],
   'CRM_Core_DAO_Worldregion' => [
     'name' => 'Worldregion',
     'class' => 'CRM_Core_DAO_Worldregion',

CRM/Core/DAO/Translation.php

+<?php
+
+/**
+ * @package CRM
+ * @copyright CiviCRM LLC https://civicrm.org/licensing
+ *
+ * Generated from xml/schema/CRM/Core/Translation.xml
+ * DO NOT EDIT.  Generated by CRM_Core_CodeGen
+ * (GenCodeChecksum:750397fc8532cb5f28bd1ff507bb3dd2)
+ */
+
+/**
+ * Database access object for the Translation entity.
+ */
+class CRM_Core_DAO_Translation extends CRM_Core_DAO {
+  const EXT = 'civicrm';
+  const TABLE_ADDED = '5.39';
+
+  /**
+   * Static instance to hold the table name.
+   *
+   * @var string
+   */
+  public static $_tableName = 'civicrm_translation';
+
+  /**
+   * Should CiviCRM log any modifications to this table in the civicrm_log table.
+   *
+   * @var bool
+   */
+  public static $_log = TRUE;
+
+  /**
+   * Unique String ID
+   *
+   * @var int
+   */
+  public $id;
+
+  /**
+   * Table where referenced item is stored
+   *
+   * @var string
+   */
+  public $entity_table;
+
+  /**
+   * Field where referenced item is stored
+   *
+   * @var string
+   */
+  public $entity_field;
+
+  /**
+   * ID of the relevant entity.
+   *
+   * @var int
+   */
+  public $entity_id;
+
+  /**
+   * Relevant language
+   *
+   * @var string
+   */
+  public $language;
+
+  /**
+   * Specify whether the string is active, draft, etc
+   *
+   * @var int
+   */
+  public $status_id;
+
+  /**
+   * Translated string
+   *
+   * @var longtext
+   */
+  public $string;
+
+  /**
+   * Class constructor.
+   */
+  public function __construct() {
+    $this->__table = 'civicrm_translation';
+    parent::__construct();
+  }
+
+  /**
+   * Returns localized title of this entity.
+   *
+   * @param bool $plural
+   *   Whether to return the plural version of the title.
+   */
+  public static function getEntityTitle($plural = FALSE) {
+    return $plural ? ts('Translated Strings') : ts('Translated String');
+  }
+
+  /**
+   * Returns foreign keys and entity references.
+   *
+   * @return array
+   *   [CRM_Core_Reference_Interface]
+   */
+  public static function getReferenceColumns() {
+    if (!isset(Civi::$statics[__CLASS__]['links'])) {
+      Civi::$statics[__CLASS__]['links'] = static::createReferenceColumns(__CLASS__);
+      Civi::$statics[__CLASS__]['links'][] = new CRM_Core_Reference_Dynamic(self::getTableName(), 'entity_id', NULL, 'id', 'entity_table');
+      CRM_Core_DAO_AllCoreTables::invoke(__CLASS__, 'links_callback', Civi::$statics[__CLASS__]['links']);
+    }
+    return Civi::$statics[__CLASS__]['links'];
+  }
+
+  /**
+   * Returns all the column names of this table
+   *
+   * @return array
+   */
+  public static function &fields() {
+    if (!isset(Civi::$statics[__CLASS__]['fields'])) {
+      Civi::$statics[__CLASS__]['fields'] = [
+        'id' => [
+          'name' => 'id',
+          'type' => CRM_Utils_Type::T_INT,
+          'description' => ts('Unique String ID'),
+          'required' => TRUE,
+          'where' => 'civicrm_translation.id',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'readonly' => TRUE,
+          'add' => '5.39',
+        ],
+        'entity_table' => [
+          'name' => 'entity_table',
+          'type' => CRM_Utils_Type::T_STRING,
+          'title' => ts('Entity Table'),
+          'description' => ts('Table where referenced item is stored'),
+          'required' => TRUE,
+          'maxlength' => 64,
+          'size' => CRM_Utils_Type::BIG,
+          'where' => 'civicrm_translation.entity_table',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'pseudoconstant' => [
+            'callback' => 'CRM_Core_BAO_Translation::getEntityTables',
+          ],
+          'add' => '5.39',
+        ],
+        'entity_field' => [
+          'name' => 'entity_field',
+          'type' => CRM_Utils_Type::T_STRING,
+          'title' => ts('Entity Field'),
+          'description' => ts('Field where referenced item is stored'),
+          'required' => TRUE,
+          'maxlength' => 64,
+          'size' => CRM_Utils_Type::BIG,
+          'where' => 'civicrm_translation.entity_field',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'pseudoconstant' => [
+            'callback' => 'CRM_Core_BAO_Translation::getEntityFields',
+          ],
+          'add' => '5.39',
+        ],
+        'entity_id' => [
+          'name' => 'entity_id',
+          'type' => CRM_Utils_Type::T_INT,
+          'description' => ts('ID of the relevant entity.'),
+          'required' => TRUE,
+          'where' => 'civicrm_translation.entity_id',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'add' => '5.39',
+        ],
+        'language' => [
+          'name' => 'language',
+          'type' => CRM_Utils_Type::T_STRING,
+          'title' => ts('Language'),
+          'description' => ts('Relevant language'),
+          'required' => TRUE,
+          'maxlength' => 5,
+          'size' => CRM_Utils_Type::SIX,
+          'where' => 'civicrm_translation.language',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'html' => [
+            'type' => 'Select',
+          ],
+          'pseudoconstant' => [
+            'optionGroupName' => 'languages',
+            'keyColumn' => 'name',
+            'optionEditPath' => 'civicrm/admin/options/languages',
+          ],
+          'add' => '5.39',
+        ],
+        'status_id' => [
+          'name' => 'status_id',
+          'type' => CRM_Utils_Type::T_INT,
+          'description' => ts('Specify whether the string is active, draft, etc'),
+          'required' => TRUE,
+          'where' => 'civicrm_translation.status_id',
+          'default' => '1',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'pseudoconstant' => [
+            'callback' => 'CRM_Core_BAO_Translation::getStatuses',
+          ],
+          'add' => '5.39',
+        ],
+        'string' => [
+          'name' => 'string',
+          'type' => CRM_Utils_Type::T_LONGTEXT,
+          'title' => ts('String'),
+          'description' => ts('Translated string'),
+          'required' => TRUE,
+          'where' => 'civicrm_translation.string',
+          'table_name' => 'civicrm_translation',
+          'entity' => 'Translation',
+          'bao' => 'CRM_Core_DAO_Translation',
+          'localizable' => 0,
+          'add' => '5.39',
+        ],
+      ];
+      CRM_Core_DAO_AllCoreTables::invoke(__CLASS__, 'fields_callback', Civi::$statics[__CLASS__]['fields']);
+    }
+    return Civi::$statics[__CLASS__]['fields'];
+  }
+
+  /**
+   * Return a mapping from field-name to the corresponding key (as used in fields()).
+   *
+   * @return array
+   *   Array(string $name => string $uniqueName).
+   */
+  public static function &fieldKeys() {
+    if (!isset(Civi::$statics[__CLASS__]['fieldKeys'])) {
+      Civi::$statics[__CLASS__]['fieldKeys'] = array_flip(CRM_Utils_Array::collect('name', self::fields()));
+    }
+    return Civi::$statics[__CLASS__]['fieldKeys'];
+  }
+
+  /**
+   * Returns the names of this table
+   *
+   * @return string
+   */
+  public static function getTableName() {
+    return self::$_tableName;
+  }
+
+  /**
+   * Returns if this table needs to be logged
+   *
+   * @return bool
+   */
+  public function getLog() {
+    return self::$_log;
+  }
+
+  /**
+   * Returns the list of fields that can be imported
+   *
+   * @param bool $prefix
+   *
+   * @return array
+   */
+  public static function &import($prefix = FALSE) {
+    $r = CRM_Core_DAO_AllCoreTables::getImports(__CLASS__, 'translation', $prefix, []);
+    return $r;
+  }
+
+  /**
+   * Returns the list of fields that can be exported
+   *
+   * @param bool $prefix
+   *
+   * @return array
+   */
+  public static function &export($prefix = FALSE) {
+    $r = CRM_Core_DAO_AllCoreTables::getExports(__CLASS__, 'translation', $prefix, []);
+    return $r;
+  }
+
+  /**
+   * Returns the list of indices
+   *
+   * @param bool $localize
+   *
+   * @return array
+   */
+  public static function indices($localize = TRUE) {
+    $indices = [
+      'index_entity_lang' => [
+        'name' => 'index_entity_lang',
+        'field' => [
+          0 => 'entity_id',
+          1 => 'entity_table',
+          2 => 'language',
+        ],
+        'localizable' => FALSE,
+        'sig' => 'civicrm_translation::0::entity_id::entity_table::language',
+      ],
+    ];
+    return ($localize && !empty($indices)) ? CRM_Core_DAO_AllCoreTables::multilingualize(__CLASS__, $indices) : $indices;
+  }
+
+}

CRM/Upgrade/Incremental/sql/5.39.alpha1.mysql.tpl

 {* file to handle db changes in 5.39.alpha1 during upgrade *}
+
+CREATE TABLE `civicrm_translation` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'Unique String ID',
+  `entity_table` varchar(64) NOT NULL COMMENT 'Table where referenced item is stored',
+  `entity_field` varchar(64) NOT NULL COMMENT 'Field where referenced item is stored',
+  `entity_id` int NOT NULL COMMENT 'ID of the relevant entity.',
+  `language` varchar(5) NOT NULL COMMENT 'Relevant language',
+  `status_id` tinyint NOT NULL DEFAULT 1 COMMENT 'Specify whether the string is active, draft, etc',
+  `string` longtext NOT NULL COMMENT 'Translated string',
+  PRIMARY KEY (`id`),
+  INDEX `index_entity_lang`(entity_id, entity_table, language)
+)
+ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;

CRM/Utils/Hook.php

@@ -1805,6 +1805,28 @@ abstract class CRM_Utils_Hook {
     );
   }
 
+  /**
+   * Define the list of fields supported in APIv4 data-translation.
+   *
+   * @param array $fields
+   *   List of data fields to translate, organized by table and column.
+   *   Omitted/unlisted fields are not translated. Any listed field may be translated.
+   *   Values should be TRUE-ish.
+   *   Ex: $fields['civicrm_event']['summary'] = TRUE
+   *   Ex: $fields['civicrm_event']['summary'] = 'yesplease';
+   *
+   * At time of writing, the `$fields` list x is prepopulated based on `<localizable>` fields in core's `xml/schema`.
+   * In the future, it may also be prepopulated with `<localizable>` fields in ext's `xml/schema`.
+   * For the interim, you may wish to fill-in `<localizable>` fields from ext's.
+   */
+  public static function translateFields(&$fields) {
+    return self::singleton()->invoke(['fields'], $fields, self::$_nullObject,
+      self::$_nullObject, self::$_nullObject, self::$_nullObject,
+      self::$_nullObject,
+      'civicrm_translateFields'
+    );
+  }
+
   /**
    * This hook allows changes to the spec of which tables to log.
    *

xml/schema/Core/Translation.xml

+<?xml version="1.0" encoding="iso-8859-1" ?>
+
+<table>
+  <add>5.39</add>
+  <base>CRM/Core</base>
+  <class>Translation</class>
+  <name>civicrm_translation</name>
+  <title>Translated String</title>
+  <titlePlural>Translated Strings</titlePlural>
+  <comment>Each string record is an alternate translation of some displayable string in the database.</comment>
+  <log>true</log>
+
+  <field>
+    <add>5.39</add>
+    <name>id</name>
+    <type>int unsigned</type>
+    <required>true</required>
+    <comment>Unique String ID</comment>
+  </field>
+  <primaryKey>
+    <name>id</name>
+    <autoincrement>true</autoincrement>
+  </primaryKey>
+
+  <field>
+    <add>5.39</add>
+    <name>entity_table</name>
+    <type>varchar</type>
+    <length>64</length>
+    <required>true</required>
+    <pseudoconstant>
+      <callback>CRM_Core_BAO_Translation::getEntityTables</callback>
+    </pseudoconstant>
+    <comment>Table where referenced item is stored</comment>
+  </field>
+
+  <field>
+    <add>5.39</add>
+    <name>entity_field</name>
+    <type>varchar</type>
+    <length>64</length>
+    <required>true</required>
+    <pseudoconstant>
+      <callback>CRM_Core_BAO_Translation::getEntityFields</callback>
+    </pseudoconstant>
+    <comment>Field where referenced item is stored</comment>
+  </field>
+
+  <field>
+    <add>5.39</add>
+    <name>entity_id</name>
+    <type>int</type>
+    <length>64</length>
+    <required>true</required>
+    <comment>ID of the relevant entity.</comment>
+  </field>
+
+  <field>
+    <add>5.39</add>
+    <name>language</name>
+    <type>varchar</type>
+    <length>5</length>
+    <required>true</required>
+    <comment>Relevant language</comment>
+    <html>
+      <type>Select</type>
+    </html>
+    <pseudoconstant>
+      <optionGroupName>languages</optionGroupName>
+      <keyColumn>name</keyColumn>
+      <optionEditPath>civicrm/admin/options/languages</optionEditPath>
+    </pseudoconstant>
+  </field>
+
+  <field>
+    <add>5.39</add>
+    <name>status_id</name>
+    <type>tinyint</type>
+    <length>3</length>
+    <default>1</default>
+    <required>true</required>
+    <pseudoconstant>
+      <callback>CRM_Core_BAO_Translation::getStatuses</callback>
+    </pseudoconstant>
+    <comment>Specify whether the string is active, draft, etc</comment>
+  </field>
+
+  <field>
+    <add>5.39</add>
+    <name>string</name>
+    <type>longtext</type>
+    <required>true</required>
+    <comment>Translated string</comment>
+  </field>
+
+  <dynamicForeignKey>
+    <add>5.39</add>
+    <idColumn>entity_id</idColumn>
+    <typeColumn>entity_table</typeColumn>
+  </dynamicForeignKey>
+
+  <index>
+    <add>5.39</add>
+    <!-- Expected queries:
+      "Admin UI: I'm editing a record. Show me all relevant translations."
+      "Public UI: I'm browsing a list of records. Show this page-worth of records in my preferred language."
+    -->
+    <name>index_entity_lang</name>
+    <!-- Prediction: In a large DB with many events/contribution-pages/groups/mailings/etc, entity ID will have best selectivity. -->
+    <!-- Prediction: Over diverse set of deployments, the selectivity of 'table' and 'language' will be similar. -->
+    <fieldName>entity_id</fieldName>
+    <fieldName>entity_table</fieldName>
+    <fieldName>language</fieldName>
+  </index>
+</table>

xml/schema/Core/files.xml

@@ -37,6 +37,7 @@
   <xi:include href="StateProvince.xml"        parse="xml" />
   <xi:include href="SystemLog.xml"            parse="xml" />
   <xi:include href="Tag.xml"                  parse="xml" />
+  <xi:include href="Translation.xml"          parse="xml" />
   <xi:include href="UFGroup.xml"              parse="xml" />
   <xi:include href="UFField.xml"              parse="xml" />
   <xi:include href="UFMatch.xml"              parse="xml" />