Merge pull request #23854 from totten/master-mixin-wfmsg

Scan for classes based on the PHP interface (WorkflowMessageInterface, ExampleDataInterface)

CRM/Extension/Manager/Module.php

@@ -97,6 +97,15 @@ class CRM_Extension_Manager_Module extends CRM_Extension_Manager_Base {
     $this->callHook($info, 'enable');
   }
 
+  public function onPostReplace(CRM_Extension_Info $oldInfo, CRM_Extension_Info $newInfo) {
+    // Like everything, ClassScanner is probably affected by pre-existing/long-standing issue dev/core#3686.
+    // This may mitigate a couple edge-cases. But really #3686 needs a different+deeper fix.
+    \Civi\Core\ClassScanner::cache('structure')->flush();
+    \Civi\Core\ClassScanner::cache('index')->flush();
+
+    parent::onPostReplace($oldInfo, $newInfo);
+  }
+
   /**
    * @param CRM_Extension_Info $info
    */

CRM/Utils/Hook.php

@@ -1676,6 +1676,22 @@ abstract class CRM_Utils_Hook {
     );
   }
 
+  /**
+   * (EXPERIMENTAL) Scan extensions for a list of auto-registered interfaces.
+   *
+   * This hook is currently experimental. It is a means to implementing `mixin/scan-classes@1`.
+   * If there are no major difficulties circa 5.55, then it can be marked stable.
+   *
+   * @param string[] $classes
+   *   List of classes which may be of interest to the class-scanner.
+   */
+  public static function scanClasses(array &$classes) {
+    self::singleton()->invoke(['classes'], $classes, self::$_nullObject,
+      self::$_nullObject, self::$_nullObject, self::$_nullObject, self::$_nullObject,
+      'civicrm_scanClasses'
+    );
+  }
+
   /**
    * This hook is called when we are determining the contactID for a specific
    * email address

CRM/Utils/System.php

@@ -1487,13 +1487,7 @@ class CRM_Utils_System {
     // a bit aggressive, but livable for now
     CRM_Utils_Cache::singleton()->flush();
 
-    // Traditionally, systems running on memory-backed caches were quite
-    // zealous about destroying *all* memory-backed caches during a flush().
-    // These flushes simulate that legacy behavior. However, they should probably
-    // be removed at some point.
-    $localDrivers = ['CRM_Utils_Cache_ArrayCache', 'CRM_Utils_Cache_NoCache'];
-    if (Civi\Core\Container::isContainerBooted()
-      && !in_array(get_class(CRM_Utils_Cache::singleton()), $localDrivers)) {
+    if (Civi\Core\Container::isContainerBooted()) {
       Civi::cache('long')->flush();
       Civi::cache('settings')->flush();
       Civi::cache('js_strings')->flush();
@@ -1503,6 +1497,7 @@ class CRM_Utils_System {
       Civi::cache('customData')->flush();
       Civi::cache('contactTypes')->clear();
       Civi::cache('metadata')->clear();
+      \Civi\Core\ClassScanner::cache('index')->flush();
       CRM_Extension_System::singleton()->getCache()->flush();
       CRM_Cxn_CiviCxnHttp::singleton()->getCache()->flush();
     }

Civi/Api4/Action/ExampleData/Get.php

@@ -19,7 +19,7 @@ use Civi\Test\ExampleDataLoader;
 /**
  * Get a list of example data-sets.
  *
- * Examples are generated by scanning `*.ex.php` files. The scanner caches
+ * Examples are generated by scanning `ExampleDataInterface` files. The scanner caches
  * metadata fields (`name`, `title`, `tags`, `file`) to avoid extraneous scanning, but
  * substantive fields (`data`) are computed as-needed.
  *

Civi/Core/ClassScanner.php

+<?php
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+namespace Civi\Core;
+
+/**
+ * The ClassScanner is a helper for finding/loading classes based on their tagged interfaces.
+ *
+ * The implementation of scanning+caching are generally built on these assumptions:
+ *
+ * - Scanning the filesystem can be expensive. One scan should serve many consumers.
+ * - Consumers want to know about specific interfaces (`get(['interface' => 'CRM_Foo_BarInterface'])`.
+ *
+ * We reconcile these goals by performing a single scan and then storing separate cache-items for each
+ * known interface (eg `$cache->get(md5('CRM_Foo_BarInterface'))`).
+ */
+class ClassScanner {
+
+  /**
+   * We cache information about classes that support each interface. Which interfaces should we track?
+   */
+  const CIVI_INTERFACE_REGEX = ';^(CRM_|Civi\\\);';
+
+  /**
+   * We load PHP files to find classes. Which files should we load?
+   */
+  const CIVI_CLASS_FILE_REGEX = '/^([A-Z][A-Za-z0-9]*)\.php$/';
+
+  const TTL = 3 * 24 * 60 * 60;
+
+  /**
+   * @var array
+   */
+  private static $caches;
+
+  /**
+   * @param array $criteria
+   *   Ex: ['interface' => 'Civi\Core\HookInterface']
+   * @return string[]
+   *   List of matching classes.
+   */
+  public static function get(array $criteria): array {
+    if (!isset($criteria['interface'])) {
+      throw new \RuntimeException("Malformed request: ClassScanner::get() must specify an interface filter");
+    }
+
+    $cache = static::cache('index');
+    $interface = $criteria['interface'];
+    $interfaceId = md5($interface);
+
+    $knownInterfaces = $cache->get('knownInterfaces');
+    if ($knownInterfaces === NULL) {
+      $knownInterfaces = static::buildIndex($cache);
+      $cache->set('knownInterfaces', $knownInterfaces, static::TTL);
+    }
+    if (!in_array($interface, $knownInterfaces)) {
+      return [];
+    }
+
+    $classes = $cache->get($interfaceId);
+    if ($classes === NULL) {
+      // Some cache backends don't guarantee the completeness of the set.
+      //I suppose this one got purged early. We'll need to rebuild the whole set.
+      $knownInterfaces = static::buildIndex($cache);
+      $cache->set('knownInterfaces', $knownInterfaces, static::TTL);
+      $classes = $cache->get($interfaceId);
+    }
+
+    return static::filterLiveClasses($classes ?: [], $criteria);
+  }
+
+  /**
+   * Fill the 'index' cache with information about all available interfaces.
+   *
+   * Every extant interface will be stored as a separate cache-item.
+   *
+   * Example:
+   *   assert $cache->get(md5(HookInterface::class)) == ['CRM_Foo_Bar', 'Civi\Whiz\Bang']
+   *
+   * @return string[]
+   *   List of PHP interfaces that were detected
+   */
+  private static function buildIndex(\CRM_Utils_Cache_Interface $cache): array {
+    $allClasses = static::scanClasses();
+    $byInterface = [];
+    foreach ($allClasses as $class) {
+      foreach (static::getRelevantInterfaces($class) as $interface) {
+        $byInterface[$interface][] = $class;
+      }
+    }
+
+    $cache->flush();
+    foreach ($byInterface as $interface => $classes) {
+      $cache->set(md5($interface), $classes, static::TTL);
+    }
+
+    return array_keys($byInterface);
+  }
+
+  /**
+   * @return array
+   *   Ex: ['CRM_Foo_Bar', 'Civi\Whiz\Bang']
+   */
+  private static function scanClasses(): array {
+    $classes = static::scanCoreClasses();
+    if (\CRM_Utils_Constant::value('CIVICRM_UF') !== 'UnitTests') {
+      \CRM_Utils_Hook::scanClasses($classes);
+    }
+    return $classes;
+  }
+
+  /**
+   * @return array
+   *   Ex: ['CRM_Foo_Bar', 'Civi\Whiz\Bang']
+   */
+  private static function scanCoreClasses(): array {
+    $cache = static::cache('structure');
+    $cacheKey = 'ClassScanner_core';
+    $classes = $cache->get($cacheKey);
+    if ($classes !== NULL) {
+      return $classes;
+    }
+
+    $civicrmRoot = \Civi::paths()->getPath('[civicrm.root]/');
+
+    // TODO: Consider expanding this search.
+    $classes = [];
+    static::scanFolders($classes, $civicrmRoot, 'Civi/Test/ExampleData', '\\');
+    static::scanFolders($classes, $civicrmRoot, 'CRM/*/WorkflowMessage', '_');
+    static::scanFolders($classes, $civicrmRoot, 'Civi/*/WorkflowMessage', '\\');
+    static::scanFolders($classes, $civicrmRoot, 'Civi/WorkflowMessage', '\\');
+    if (\CRM_Utils_Constant::value('CIVICRM_UF') === 'UnitTests') {
+      static::scanFolders($classes, $civicrmRoot . 'tests/phpunit', 'CRM/*/WorkflowMessage', '_');
+      static::scanFolders($classes, $civicrmRoot . 'tests/phpunit', 'Civi/*/WorkflowMessage', '\\');
+    }
+
+    $cache->set($cacheKey, $classes, static::TTL);
+    return $classes;
+  }
+
+  private static function filterLiveClasses(array $classes, array $criteria): array {
+    return array_filter($classes, function($class) use ($criteria) {
+      if (!class_exists($class)) {
+        return FALSE;
+      }
+      $reflClass = new \ReflectionClass($class);
+      return !$reflClass->isAbstract() && ($reflClass)->implementsInterface($criteria['interface']);
+    });
+  }
+
+  private static function getRelevantInterfaces(string $class): array {
+    $rawInterfaceNames = (new \ReflectionClass($class))->getInterfaceNames();
+    return preg_grep(static::CIVI_INTERFACE_REGEX, $rawInterfaceNames);
+  }
+
+  /**
+   * Search some $classRoot folder for a list of classes.
+   *
+   * Return any classes that implement a Civi-related interface, such as ExampleDataInterface
+   * or HookInterface. (Specifically, interfaces matchinv CIVI_INTERFACE_REGEX.)
+   *
+   * @internal
+   *   Currently reserved for use within civicrm-core. Signature may change.
+   * @param string[] $classes
+   *   List of known/found classes.
+   * @param string $classRoot
+   *   The base folder in which to search.
+   *   Ex: The $civicrm_root or some extension's basedir.
+   * @param string $classDir
+   *   Folder to search (within the $classRoot).
+   *   May use wildcards.
+   *   Ex: "CRM" or "Civi"
+   * @param string $classDelim
+   *   Namespace separator, eg underscore or backslash.
+   */
+  public static function scanFolders(array &$classes, string $classRoot, string $classDir, string $classDelim): void {
+    $classRoot = \CRM_Utils_File::addTrailingSlash($classRoot, '/');
+
+    $baseDirs = (array) glob($classRoot . $classDir);
+    foreach ($baseDirs as $baseDir) {
+      foreach (\CRM_Utils_File::findFiles($baseDir, '*.php') as $absFile) {
+        if (!preg_match(static::CIVI_CLASS_FILE_REGEX, basename($absFile))) {
+          continue;
+        }
+        $absFile = str_replace(DIRECTORY_SEPARATOR, '/', $absFile);
+        $relFile = \CRM_Utils_File::relativize($absFile, $classRoot);
+        $class = str_replace('/', $classDelim, substr($relFile, 0, -4));
+        if (class_exists($class)) {
+          $interfaces = static::getRelevantInterfaces($class);
+          if ($interfaces) {
+            $classes[] = $class;
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * @param string $name
+   *   - The 'index' cache describes the list of live classes that match an interface. It persists for the
+   *     duration of the system-configuration (eg cleared by system-flush or enable/disable extension).
+   *   - The 'structure' cache describes the class-structure within each extension. It persists for the
+   *     duration of the current page-view and is essentially write-once. This minimizes extra scans during testing.
+   *     (It could almost use Civi::$statics, except we want it to survive throughout testing.)
+   *   - Note: Typical runtime usage should only hit the 'index' cache. The 'structure' cache should only
+   *     be relevant following a system-flush.
+   * @return \CRM_Utils_Cache_Interface
+   * @internal
+   */
+  public static function cache(string $name): \CRM_Utils_Cache_Interface {
+    // Class-scanner runs before container is available. Manage our own cache. (Similar to extension-cache.)
+    // However, unlike extension-cache, we do not want to prefetch all interface lists on all pageloads.
+
+    if (!isset(static::$caches[$name])) {
+      switch ($name) {
+        case 'index':
+          if (empty($_DB_DATAOBJECT['CONFIG'])) {
+            // Atypical example: You have a test with a @dataProvider that relies on ClassScanner. Runs before bot.
+            return new \CRM_Utils_Cache_ArrayCache([]);
+          }
+          static::$caches[$name] = \CRM_Utils_Cache::create([
+            'name' => 'classes',
+            'type' => ['*memory*', 'SqlGroup', 'ArrayCache'],
+            'fastArray' => TRUE,
+          ]);
+
+        case 'structure':
+          static::$caches[$name] = new \CRM_Utils_Cache_ArrayCache([]);
+          break;
+
+      }
+    }
+
+    return static::$caches[$name];
+  }
+
+}

Civi/Test/ExampleDataLoader.php

@@ -11,6 +11,8 @@
 
 namespace Civi\Test;
 
+use Civi\Core\ClassScanner;
+
 class ExampleDataLoader {
 
   /**
@@ -64,7 +66,7 @@ class ExampleDataLoader {
       return NULL;
     }
 
-    $obj = $this->createObj($example['file'], $example['class']);
+    $obj = $this->createObj($example['class']);
     $obj->build($example);
     return $example;
   }
@@ -77,22 +79,15 @@ class ExampleDataLoader {
    * @throws \ReflectionException
    */
   protected function findMetas(): array {
-    $classes = array_merge(
-      // This scope of search is decidedly narrow - it should probably be expanded.
-      $this->scanExampleClasses(\Civi::paths()->getPath('[civicrm.root]/'), 'Civi/Test/ExampleData', '\\'),
-      $this->scanExampleClasses(\Civi::paths()->getPath('[civicrm.root]/'), 'CRM/*/WorkflowMessage', '_'),
-      $this->scanExampleClasses(\Civi::paths()->getPath('[civicrm.root]/'), 'Civi/*/WorkflowMessage', '\\'),
-      $this->scanExampleClasses(\Civi::paths()->getPath('[civicrm.root]/'), 'Civi/WorkflowMessage', '\\'),
-      $this->scanExampleClasses(\Civi::paths()->getPath('[civicrm.root]/tests/phpunit/'), 'CRM/*/WorkflowMessage', '_'),
-      $this->scanExampleClasses(\Civi::paths()->getPath('[civicrm.root]/tests/phpunit/'), 'Civi/*/WorkflowMessage', '\\')
-    );
+    $classes = ClassScanner::get(['interface' => ExampleDataInterface::class]);
 
     $all = [];
-    foreach ($classes as $file => $class) {
-      $obj = $this->createObj($file, $class);
+    foreach ($classes as $class) {
+      $reflClass = new \ReflectionClass($class);
+      $obj = $this->createObj($class);
       $offset = 0;
       foreach ($obj->getExamples() as $example) {
-        $example['file'] = $file;
+        $example['file'] = \CRM_Utils_File::relativize($reflClass->getFileName(), \Civi::paths()->getPath('[civicrm.root]/'));
         $example['class'] = $class;
         if (!isset($example['name'])) {
           $example['name'] = $example['class'] . '#' . $offset;
@@ -105,43 +100,9 @@ class ExampleDataLoader {
     return $all;
   }
 
-  /**
-   * @param $classRoot
-   *   Ex: Civi root dir.
-   * @param $classDir
-   *   Folder to search (within the parent).
-   * @param $classDelim
-   *   Namespace separator, eg underscore or backslash.
-   * @return array
-   *   Array(string $includeFile => string $className).
-   */
-  private function scanExampleClasses($classRoot, $classDir, $classDelim): array {
-    $civiRoot = \Civi::paths()->getPath('[civicrm.root]/');
-    $classRoot = \CRM_Utils_File::addTrailingSlash($classRoot, '/');
-    // Prefer include-paths relative to civiRoot - eg make tests/phpunit/* loadable at runtime.
-    $includeRoot = \CRM_Utils_File::isChildPath($civiRoot, $classRoot) ? $civiRoot : $classRoot;
-
-    $r = [];
-    $exDirs = (array) glob($classRoot . $classDir);
-    foreach ($exDirs as $exDir) {
-      foreach (\CRM_Utils_File::findFiles($exDir, '*.ex.php') as $file) {
-        $file = str_replace(DIRECTORY_SEPARATOR, '/', $file);
-        $includeFile = \CRM_Utils_File::relativize($file, $includeRoot);
-        $classFile = \CRM_Utils_File::relativize($file, $classRoot);
-        $class = str_replace('/', $classDelim, preg_replace('/\.ex\.php$/', '',
-          $classFile));
-        $r[$includeFile] = $class;
-      }
-    }
-    return $r;
-  }
-
-  private function createObj(?string $file, ?string $class): ExampleDataInterface {
-    if ($file) {
-      include_once $file;
-    }
+  private function createObj(?string $class): ExampleDataInterface {
     if (!class_exists($class)) {
-      throw new \CRM_Core_Exception("Failed to read example (class '{$class}' in file '{$file}')");
+      throw new \CRM_Core_Exception("Failed to read example (class '{$class}')");
     }
 
     return new $class();

Civi/WorkflowMessage/WorkflowMessage.php

@@ -13,6 +13,7 @@
 namespace Civi\WorkflowMessage;
 
 use Civi\Api4\Utils\ReflectionUtils;
+use Civi\Core\ClassScanner;
 use Civi\WorkflowMessage\Exception\WorkflowMessageException;
 
 /**
@@ -146,18 +147,9 @@ class WorkflowMessage {
     $map = $cache->get($cacheKey);
     if ($map === NULL) {
       $map = [];
-      $map['generic'] = GenericWorkflowMessage::class;
-      $baseDirs = explode(PATH_SEPARATOR, get_include_path());
-      foreach ($baseDirs as $baseDir) {
-        $baseDir = \CRM_Utils_File::addTrailingSlash($baseDir);
-        $glob = (array) glob($baseDir . 'CRM/*/WorkflowMessage/*.php');
-        $glob = preg_grep('/\.ex\.php$/', $glob, PREG_GREP_INVERT);
-        foreach ($glob as $file) {
-          $class = strtr(preg_replace('/\.php$/', '', \CRM_Utils_File::relativize($file, $baseDir)), ['/' => '_', '\\' => '_']);
-          if (class_exists($class) && (new \ReflectionClass($class))->implementsInterface(WorkflowMessageInterface::class)) {
-            $map[$class::WORKFLOW] = $class;
-          }
-        }
+      foreach (ClassScanner::get(['interface' => WorkflowMessageInterface::class]) as $wfClass) {
+        $wfName = ($wfClass === GenericWorkflowMessage::class) ? 'generic' : $wfClass::WORKFLOW;
+        $map[$wfName] = $wfClass;
       }
       $cache->set($cacheKey, $map);
     }

mixin/scan-classes@1/example/CRM/Shimmy/ShimmyMessage.php

+<?php
+
+class CRM_Shimmy_ShimmyMessage extends Civi\WorkflowMessage\GenericWorkflowMessage {
+
+  public const WORKFLOW = 'shimmy_message_example';
+
+  /**
+   * @var string
+   * @scope tplParams
+   */
+  protected $foobar;
+
+}

mixin/scan-classes@1/example/tests/mixin/ScanClassesTest.php

+<?php
+
+namespace Civi\Shimmy\Mixins;
+
+/**
+ * Assert that the 'scan-classes' mixin is working properly.
+ *
+ * This class defines the assertions to run when installing or uninstalling the extension.
+ * It is called as part of E2E_Shimmy_LifecycleTest.
+ *
+ * @see E2E_Shimmy_LifecycleTest
+ */
+class ScanClassesTest extends \PHPUnit\Framework\Assert {
+
+  public function testPreConditions($cv) {
+    $this->assertFileExists(static::getPath('/CRM/Shimmy/ShimmyMessage.php'), 'The shimmy extension must have example PHP files.');
+  }
+
+  public function testInstalled($cv) {
+    // Assert that WorkflowMessageInterface's are registered.
+    $items = $cv->api4('WorkflowMessage', 'get', ['where' => [['name', '=', 'shimmy_message_example']]]);
+    $this->assertEquals('CRM_Shimmy_ShimmyMessage', $items[0]['class']);
+  }
+
+  public function testDisabled($cv) {
+    // Assert that WorkflowMessageInterface's are removed.
+    $items = $cv->api4('WorkflowMessage', 'get', ['where' => [['name', '=', 'shimmy_message_example']]]);
+    $this->assertEmpty($items);
+  }
+
+  public function testUninstalled($cv) {
+    // Assert that WorkflowMessageInterface's are removed.
+    $items = $cv->api4('WorkflowMessage', 'get', ['where' => [['name', '=', 'shimmy_message_example']]]);
+    $this->assertEmpty($items);
+  }
+
+  protected static function getPath($suffix = ''): string {
+    return dirname(__DIR__, 2) . $suffix;
+  }
+
+}

mixin/scan-classes@1/mixin.php

+<?php
+
+/**
+ * Scan for files which implement common Civi-PHP interfaces.
+ *
+ * Specifically, this listens to `hook_scanClasses` and reports any classes with Civi-related
+ * interfaces (eg `CRM_Foo_BarInterface` or `Civi\Foo\BarInterface`). For example:
+ *
+ *   - \Civi\Core\HookInterface
+ *   - \Civi\Test\ExampleDataInterface
+ *   - \Civi\WorkflowMessage\WorkflowMessageInterface
+ *
+ * If you are adding this to an existing extension, take care that you meet these assumptions:
+ *
+ *   - Classes live in 'CRM_' ('./CRM/**.php') or 'Civi\' ('./Civi/**.php').
+ *   - Class files only begin with uppercase letters.
+ *   - Class files only contain alphanumerics.
+ *   - Class files never have multiple dots in the name. ("CRM/Foo.php" is a class; "CRM/Foo.bar.php" is not).
+ *   - The ONLY files which match these patterns are STRICTLY class files.
+ *   - The ONLY classes which match these patterns are SAFE/INTENDED for use with `hook_scanClasses`.
+ *
+ * To minimize unintended activations, this only loads Civi interfaces. It skips other interfaces.
+ *
+ * @mixinName scan-classes
+ * @mixinVersion 1.0.0
+ * @since 5.52
+ *
+ * @param CRM_Extension_MixInfo $mixInfo
+ *   On newer deployments, this will be an instance of MixInfo. On older deployments, Civix may polyfill with a work-a-like.
+ * @param \CRM_Extension_BootCache $bootCache
+ *   On newer deployments, this will be an instance of MixInfo. On older deployments, Civix may polyfill with a work-a-like.
+ */
+
+/**
+ * @param \CRM_Extension_MixInfo $mixInfo
+ * @param \CRM_Extension_BootCache $bootCache
+ */
+return function ($mixInfo, $bootCache) {
+  /**
+   * @param \Civi\Core\Event\GenericHookEvent $event
+   */
+  Civi::dispatcher()->addListener('hook_civicrm_scanClasses', function ($event) use ($mixInfo) {
+    if (!$mixInfo->isActive()) {
+      return;
+    }
+
+    $cache = \Civi\Core\ClassScanner::cache('structure');
+    $cacheKey = $mixInfo->longName;
+    $all = $cache->get($cacheKey);
+    if ($all === NULL) {
+      $baseDir = CRM_Utils_File::addTrailingSlash($mixInfo->getPath());
+      $all = [];
+
+      \Civi\Core\ClassScanner::scanFolders($all, $baseDir, 'CRM', '_');
+      \Civi\Core\ClassScanner::scanFolders($all, $baseDir, 'Civi', '\\');
+      if (defined('CIVICRM_TEST')) {
+        \Civi\Core\ClassScanner::scanFolders($all, "$baseDir/tests/phpunit", 'CRM', '_');
+        \Civi\Core\ClassScanner::scanFolders($all, "$baseDir/tests/phpunit", 'Civi', '\\');
+      }
+      $cache->set($cacheKey, $all, \Civi\Core\ClassScanner::TTL);
+    }
+
+    $event->classes = array_merge($event->classes, $all);
+  });
+
+};