Add config validation against schema

Civi/Inlay/ArraySchema.php

+<?php
+/**
+ * 'Simple' array schema check.
+ *
+ * @author Rich Lott / Artful Robot
+ * Version: 1.0
+ *
+ */
+namespace Civi\Inlay;
+
+/**
+ * @class
+ *
+ * Schema is defined with an array. Each key is either a string or a regex used to match one or more keys in the input.
+ *
+ * e.g. new ArraySchema(['x' => ..., '/^[0-9]+$/' => ..., '//' => ...']) would allow for
+ * a key called 'x', numeric keys, and latterly, *any* key since // matches any string.
+ *
+ * Each value (...) in above example is also an array.
+ * The first positional item is 'MUST' or 'MAY'. If MUST, then this key (pattern) must match at least one item.
+ * If 'MAY' then if an input key matches this pattern, its value must match the schema, but if there is no input key matching,
+ * that's ok.
+ * If you do not include '//' as a key match, then all keys not matching other matches are considered invalid.
+ *
+ * The second positional value is a string of permitted gettype() values separated by |
+ * e.g. 'string|double|NULL' Note that gettype() returns 'boolean' for vars declared as bool. And 'double' for floats.
+ *
+ * The value MUST match one of these types. If it's NULL, no further checks are done.
+ *
+ * Beyond the positional (first and second) keys, are optional string keys:
+ *
+ * - oneOfStrictly and oneOf use in_array to check that the value matches.
+ * - gt, lt, gte, lte use > < >= <= to compare the value
+ * - regex provides a regex that must match.
+ * - schema provides a schema when the value is an array.
+ * - recurse provides a means to describe recursive schemas, such as this schema itself!
+ *   Its value is an array of keys within the schema. An empty array means recurse from the whole schema.
+ *
+ * Recurse example. Consider an array containing a representation of a directory, like this:
+ *
+ * [
+ *   ['name' => 'README.md'],
+ *   ['name' => 'css', 'children' => [
+ *     ['name' => 'web.css'],
+ *     ['name' => 'print.css'],
+ *   ]],
+ * ]
+ *
+ * This could be validated with:
+ *
+ * [
+ *   '/^\d+$/'    => ['MAY', 'array', 'schema' => [
+ *     'name'     => ['MUST', 'string'],
+ *     'children' => ['MAY', 'array', 'recurse' => []]
+ *   ]]
+ * ]
+ *
+ * If the input for the 'web.css' file used 'nom' instead of 'name', two errors would be generated:
+ * - Missing required key at 1»children»0 "name"
+ * - Unexpected keys at 1»children»0 ["nom"]
+ *
+ */
+class ArraySchema {
+
+  public $schema;
+
+  /**
+   * If true, AND if a type check includes integerString *after* integer,
+   * then cast the value to an int.
+   *
+   * @param bool
+   */
+  protected $coerceToInt = FALSE;
+
+  /**
+   * @param bool
+   */
+  protected $removeUnexpectedKeys = FALSE;
+
+  public static function getOwnSchema(): array {
+    return [
+      '//' => [ 'MUST', 'array',
+        'schema' => [
+          0 => ['MUST', 'string', 'oneOfStrictly' => ['MUST', 'MAY']],
+          1 => ['MUST', 'string|NULL', 'regex' => '/^(boolean|integer|integerString|numeric|double|string|array|NULL)([|](boolean|integer|integerString|numeric|double|string|array|NULL))*$/'],
+          'schema' => ['MAY', 'array', 'recurse' => []],
+          'regex' => ['MAY', 'string'],
+          'recurse' => ['MAY', 'array', 'schema' => [
+            '//' => ['MAY', 'string']
+          ]],
+          'oneOfStrictly' => ['MAY', 'array', 'schema' => ['/^[0-9]+$/' => ['MUST', 'integer|string|boolean|double|NULL']]],
+          'oneOf' => ['MAY', 'array', 'schema' => ['/^[0-9]+$/' => ['MUST', 'integer|string|integerString|numeric|boolean|double|NULL']]],
+          'gt' => ['MAY', 'string|integer|double'],
+          'lt' => ['MAY', 'string|integer|double'],
+          'gte' => ['MAY', 'string|integer|double'],
+          'lte' => ['MAY', 'string|integer|double'],
+          ]
+        ]
+      ];
+  }
+
+  public function __construct(array $schema, bool $validateSchema = TRUE) {
+    $this->schema = $schema;
+    if ($validateSchema) {
+      $a = new static(static::getOwnSchema(), FALSE);
+      $errors = $a->getErrors($this->schema);
+      if ($errors) {
+        throw new \RuntimeException("Attempted to construct ArraySchema with invalid schema:\n" . $this->formatErrorsAsString($errors));
+      }
+    }
+  }
+
+  /**
+   * Returns any validation errors as an array of tuples with 3 elements:
+   *
+   * - string error message
+   * - array list of input keys describing the path to the invalid item.
+   * - the value
+   */
+  public function getErrors(array $data) {
+    return $this->matches($data, [], $this->schema);
+  }
+
+  /**
+   * @return static
+   */
+  public function setRemoveUnexpectedKeys(bool $remove = TRUE) {
+    $this->removeUnexpectedKeys = $remove;
+    return $this;
+  }
+
+  /**
+   * @return static
+   */
+  public function setCoerceToInt(bool $y = TRUE) {
+    $this->coerceToInt = $y;
+    return $this;
+  }
+
+  protected function matches(array &$data, array $ancestry, array $schema): array {
+    $dataKeys = array_flip(array_keys($data));
+    $allErrors = [];
+    foreach ($schema as $keyMatch => $schema) {
+      $keyWasFound = 0;
+      if (substr($keyMatch, 0, 1) === '/') {
+        // Regexp
+        foreach($data as $key => &$value) {
+          if (preg_match($keyMatch, $key)) {
+            $keyWasFound++;
+            // Key is valid, what about value?
+            $valueErrors = $this->valueMatch($keyMatch, $schema, $key, $value, [...$ancestry, $key]);
+            if ($valueErrors) {
+              // print "errors " . json_encode($valueErrors, JSON_PRETTY_PRINT);
+              $allErrors = [...$allErrors, ...$valueErrors];
+            }
+            // As this key is now validated, remove it from $data
+            unset($dataKeys[$key]);
+          }
+        }
+        unset($value);
+      }
+      else {
+        // $keyMatch is a simple string.
+        if (array_key_exists($keyMatch, $data)) {
+          $keyWasFound++;
+          $valueErrors = $this->valueMatch($keyMatch, $schema, $keyMatch, $data[$keyMatch], [...$ancestry, $keyMatch]);
+          if ($valueErrors) {
+            // print "errors " . json_encode($valueErrors, JSON_PRETTY_PRINT);
+            $allErrors = [...$allErrors, ...$valueErrors];
+          }
+          // As this key is now validated, remove it from $data
+          unset($dataKeys[$keyMatch]);
+        }
+      }
+      if ($schema[0] === 'MUST' && $keyWasFound === 0) {
+        // print "MUST have $keyMatch but not found.\n";
+        $allErrors[] = ["Missing required key", $ancestry, $keyMatch];
+      }
+    }
+    if ($dataKeys) {
+      if ($this->removeUnexpectedKeys) {
+        foreach (array_keys($dataKeys) as $key) {
+          unset($data[$key]);
+        }
+      }
+      else {
+        $allErrors[] = ["Unexpected keys", $ancestry, array_keys($dataKeys)];
+      }
+    }
+    return $allErrors;
+  }
+
+  protected function valueMatch(string $keyMatch, array $schema, string $key, &$value, array $ancestry) {
+    $typeOK = FALSE;
+    $actualType = gettype($value);
+    $expectedTypes = $schema[1];
+    if ($expectedTypes) {
+      foreach (explode('|', $expectedTypes) as $type) {
+        if (in_array($actualType, ['boolean', 'integer', 'double', 'string', 'array'])
+          && $actualType === $type) {
+          $typeOK = TRUE;
+          break;
+        }
+        if ($actualType === 'NULL' && $type === 'NULL') {
+          // all ok, return early.
+          return [];
+        }
+        if ($type === 'integerString' && $actualType === 'string' && strval(intval($value)) === $value) {
+          $typeOK = TRUE;
+          if ($this->coerceToInt && preg_match('/integer[|].*integerString/', $expectedTypes)) {
+            $value = intval($value);
+          }
+          break;
+        }
+        if ($type === 'numeric' && is_numeric($value)) {
+          $typeOK = TRUE;
+          break;
+        }
+      }
+      if (!$typeOK) {
+        return [["Expected $expectedTypes but got $actualType", $ancestry, $value]];
+      }
+    }
+    $actualTypeIsArray = $actualType === 'array';
+    if ($actualTypeIsArray && is_array($schema['schema'] ?? NULL)) {
+      // Recursive.
+      return $this->matches($value, $ancestry, $schema['schema']);
+    }
+    if ($actualTypeIsArray && is_array($schema['recurse'] ?? NULL)) {
+      // Recursive schema.
+      $arraySchema = $this->schema;
+      foreach ($schema['recurse'] as $key) {
+        $arraySchema = $arraySchema[$key] ?? NULL;
+      }
+      if (!is_array($arraySchema)) {
+        throw new \RuntimeException("Invalid recurse expression in schema: " . json_encode($schema['recurse']));
+      }
+      return $this->matches($value, $ancestry, $arraySchema);
+    }
+
+   $schemaRequiresNotArray = array_keys(array_intersect_key($schema, array_flip(['oneOf', 'oneOfStrictly', 'regex', 'gt', 'gte', 'lt', 'lte'])));
+    if (count($schemaRequiresNotArray) && $actualTypeIsArray) {
+      return [["Got array which can't be compared to schema " . implode(', ', $schemaRequiresNotArray), $ancestry, $value]];
+    }
+
+    $enum = $schema['oneOf'] ?? $schema['oneOfStrictly'] ?? NULL;
+    $strict = is_array($schema['oneOfStrictly'] ?? NULL);
+    if (is_array($enum) && !in_array($value, $enum, $strict)) {
+      return [
+        [
+          "Got something that's not " . ($strict ? 'oneOfStrictly' : 'oneOf')
+          . ' ' . json_encode($schema[$strict ? 'oneOfStrictly' : 'oneOf']),
+          $ancestry, $value
+        ]
+      ];
+    }
+
+    if (!empty($schema['regex']) && !preg_match($schema['regex'], $value)) {
+      return [["Expected regex match $schema[regex]", $ancestry, $value]];
+    }
+
+    if (!empty($schema['gt']) && !($value > $schema['gt'])) {
+        return [["Expected gt $schema[gt]", $ancestry, $value]];
+    }
+    if (!empty($schema['gte']) && !($value >= $schema['gte'])) {
+        return [["Expected gte $schema[gte]", $ancestry, $value]];
+    }
+    if (!empty($schema['lt']) && !($value < $schema['lt'])) {
+        return [["Expected lt $schema[lt]", $ancestry, $value]];
+    }
+    if (!empty($schema['lte']) && !($value <= $schema['lte'])) {
+        return [["Expected lte $schema[lte]", $ancestry, $value]];
+    }
+    return [];
+  }
+
+  public function formatErrorsAsString(array $errors): string {
+    $messages = [];
+    foreach ($errors as $error) {
+      $message = array_shift($error);
+      $ancestry = array_shift($error);
+      $value = $error ? ' ' . json_encode(($error[0]), JSON_PRETTY_PRINT, JSON_UNESCAPED_SLASHES) : '';
+      $ancestry = $ancestry ? implode('»', $ancestry) : '(root)';
+      $messages[] = "{$message} at {$ancestry}{$value}";
+    }
+    return implode("\n", $messages);
+  }
+}

Civi/Inlay/Type.php

@@ -4,19 +4,19 @@ namespace Civi\Inlay;
 
 use \InvalidArgumentException;
 use Civi\Inlay\Config as InlayConfig;
+use Civi\Inlay\ArraySchema;
 use Civi\Api4\InlayConfigSet;
 
 /**
  * The base class for any type of Inlay.
  *
  */
-
-/**
- * Implementation abstract
- */
 abstract class Type {
   const INVALID_SUBCLASS = 1;
 
+  /** @const string Typically the version from the info.xml file, but only needs updating when your config changes. */
+  const CONFIG_VERSION = '';
+
   /** @var string human name for the inlay type. e.g. "Signup form"*/
   public static $typeName;
 
@@ -48,8 +48,6 @@ abstract class Type {
 
   /** @var array All field data except the config JSON blob */
   public $instanceData;
-  public function __construct() {
-  }
 
   public static function fromClass($class): Type {
     if (!(is_subclass_of($class, Type::class))) {
@@ -58,6 +56,7 @@ abstract class Type {
     $obj = new $class();
     return $obj;
   }
+
   /**
    * Instantiate the correct class from the data in $array.
    *
@@ -400,12 +399,62 @@ abstract class Type {
    * @return \Civi\Inlay\Type (this)
    */
   public function setConfig(array $config): Type {
-    $a = static::$defaultConfig;
+    // Check if migration needed
+    if (!empty(static::CONFIG_VERSION) && ($config['version'] ?? '') !== static::CONFIG_VERSION) {
+      $config = $this->migrateConfig($config);
+    }
+    // Reasonable fallback method:
     $this->config = array_intersect_key($config + static::$defaultConfig, static::$defaultConfig);
-
+    $this->validateConfig(TRUE);
     return $this;
   }
 
+  /**
+   * Called with an array of config when an Inlay\Type class has a CONFIG_VERSION set
+   * that differs from the 'version' in the $config array.
+   *
+   * Override this with suitable migrations. If significant you may wish to put those
+   * in other files.
+   *
+   * Note: this does not SAVE your migrated config; this will run each time old config is loaded.
+   * Your CRM_YourInlay_Upgrader code should do an API call to save migrated config. This is a
+   * precaution against automatically applying a migration in a persisted way. However, migrated
+   * content will get persisted if you edit the inlay's config and hit Save yourself, but it's
+   * assumed that you have then verified everything.
+   *
+   */
+  protected function migrateConfig(array $config): array {
+    $config['version'] = static::CONFIG_VERSION;
+    // ... your migrations here ...
+    return $config;
+  }
+
+  /**
+   * Check the config we have against a schema, if one exists,
+   * and return any errors.
+   */
+  public function validateConfig($coerce = TRUE): array {
+    $errors = [];
+    $schema = $this->getConfigSchema();
+    if (!empty($schema)) {
+      $validator = new ArraySchema($schema);
+      if ($coerce) {
+        $validator->setCoerceToInt()->setRemoveUnexpectedKeys();
+      }
+      $errors = $validator->getErrors($this->config);
+    }
+    return $errors;
+  }
+
+  /**
+   * Optionally you can override this with a schema definition for your config.
+   *
+   * @see Civi\Inlay\ArraySchema
+   */
+  public function getConfigSchema(): array {
+    return [];
+  }
+
   /**
    * Generates data to be served with the Javascript application code bundle.
    *

tests/phpunit/Civi/Inlay/ArraySchemaTest.php

+<?php
+use Civi\Inlay\ArraySchema;
+
+class ArraySchemaTest extends \PHPUnit\Framework\TestCase /*implements HeadlessInterface, HookInterface, TransactionalInterface*/ {
+
+  public function testOwnSchema() {
+    // This will test and throw exception if it fails.
+    new ArraySchema(ArraySchema::getOwnSchema());
+  }
+
+  /**
+   */
+  public function testExample() {
+    $this->runTests('files',
+      new ArraySchema([
+        '/^\d+$/' => ['MAY', 'array', 'schema' => [
+          'name' => ['MUST', 'string'],
+          'children' => ['MAY', 'array', 'recurse' => []]
+        ]]
+      ]),
+      [
+        [
+          [
+            ['name' => 'README.md'],
+            ['name' => 'css', 'children' => [
+              ['name' => 'web.css'],
+              ['name' => 'print.css'],
+            ]],
+          ],
+          0
+        ]
+      ]);
+  }
+
+  protected function runTests(string $setDescr, ArraySchema $a, array $casesAndErrorCounts) {
+    // print "\n## $setDescr ##\n\n";
+    foreach ($casesAndErrorCounts as $i => [$input, $errorCount]) {
+      $errors = $a->getErrors($input);
+      $this->assertCount($errorCount, $errors, "$setDescr #$i Failed with input " . json_encode($input, JSON_UNESCAPED_SLASHES));
+    }
+  }
+
+  public function testSimples() {
+
+    $this->runTests('simples',
+      new ArraySchema([
+        'x' => ['MUST', 'integer'],
+        'z' => ['MAY', 'string|NULL'],
+        'e' => ['MAY', 'string', 'oneOf' => ['aye', 'bee']],
+        'b' => ['MAY', 'integer', 'gt' => 2, 'lte' => '5'],
+        'n1' => ['MAY', 'integerString'],
+        'n2' => ['MAY', 'numeric'],
+      ]),
+      [
+        [ ['x' => 123],0 ],
+        [ ['x' => 'hello'], 1 ],
+        [ ['y' => 123], 2 ],
+        [ ['x' => []], 1 ],
+        [ ['x' => 123, 'z' => 123], 1 ],
+        [ ['x' => 123, 'z' => 'zed'], 0 ],
+        [ ['x' => 123, 'z' => NULL], 0 ],
+        [ ['x' => 123, 'e' => 'cee'], 1 ],
+        [ ['x' => 123, 'e' => 'bee'], 0 ],
+        [ ['x' => 123, 'b' => 1], 1 ],
+        [ ['x' => 123, 'b' => 3], 0 ],
+        [ ['x' => 123, 'b' => 5], 0 ],
+        [ ['x' => 123, 'b' => 6], 1 ],
+        [ ['x' => 123, 'n1' => '123'], 0],
+        [ ['x' => 123, 'n1' => '1.23'], 1],
+        [ ['x' => 123, 'n1' => 'not a number'], 1],
+        [ ['x' => 123, 'n1' => 123], 1],
+        [ ['x' => 123, 'n2' => 123], 0],
+        [ ['x' => 123, 'n2' => '123'], 0],
+        [ ['x' => 123, 'n2' => '123.23'], 0],
+        [ ['x' => 123, 'n2' => 'not a number'], 1],
+      ]);
+  }
+
+  public function testNested() {
+
+    $this->runTests('nested',
+      new ArraySchema([
+        'trunk' => ['MUST', 'array', 'schema' => [
+          'branch' => [
+            'MAY', 'array', 'schema' => [
+              'twig' => ['MUST', 'integer'],
+              '//' => ['MAY', NULL] // allow any keys in here.
+            ],
+          ]
+        ]]
+      ]),
+      [
+        [ ['x' => 123], 2 ],
+        [ ['trunk' => 123], 1 ], // 123 not array
+        [ ['trunk' => ['x' => 123]], 1 ], // x unexpected
+        [ ['trunk' => ['branch' => 123]], 1 ], // 123 not array
+        [ ['trunk' => ['branch' => ['twig' => 123, 'whatever' => 'else']]], 0 ],
+        [ ['trunk' => ['branch' => ['twig' => 'word']]], 1 ], // 'word' not integer
+      ]
+    );
+  }
+
+
+  // @todo test coersion and removing unrecognised keys
+}