Commit 3809a936 authored by Mathieu Lutfy's avatar Mathieu Lutfy Committed by Aegir user

extensions/extensions-directory#14 Convert code from civicrm_org_stats to civi...

extensions/extensions-directory#14 Convert code from civicrm_org_stats to civi extension org.civicrm.civiorgstats
parent 764d9fb2
<?php
/**
* @file
* Code for the CiviCRM Extensions Stats module.
*/
class CRM_Civicrmorgstats_Utils {
protected $stats;
protected $client;
public function __construct() {
$this->stats = [
'messages' => [],
'updated' => 0,
'unchanged' => 0,
'errors' => 0,
];
$this->client = new GuzzleHttp\Client();
}
/**
* Returns the stats about the update job itself.
*/
public function getUpdateStats() {
return $this->stats;
}
protected function updateRecord($node, $usage) {
if (!empty($node->cu_node)) {
db_query("UPDATE field_data_field_extension_current_usage
SET field_extension_current_usage_value = :usage_value
WHERE entity_id = {$node->nid} AND entity_type = 'node'", array(':usage_value' => $usage));
db_query("UPDATE field_revision_field_extension_current_usage
SET field_extension_current_usage_value = :usage_value
WHERE entity_id = {$node->nid} AND entity_type = 'node' and revision_id = {$node->revision_id}", array(':usage_value' => $usage));
}
else {
db_query("INSERT INTO field_data_field_extension_current_usage (entity_type, bundle, entity_id, revision_id, `language`, delta, field_extension_current_usage_value)
VALUES ('node', 'extension', {$node->nid}, {$node->revision_id}, '{$node->language}', 0, :usage_value)", array(':usage_value' => $usage));
db_query("INSERT INTO field_revision_field_extension_current_usage (entity_type, bundle, entity_id, revision_id, `language`, delta, field_extension_current_usage_value)
VALUES ('node', 'extension', {$node->nid}, {$node->revision_id}, '{$node->language}', 0, :usage_value)", array(':usage_value' => $usage));
}
}
/**
* Clear drupal's caches related to node field data
*/
protected function clearCache() {
cache_clear_all('*', 'cache_field', TRUE);
cache_clear_all('*', 'cache_page', TRUE);
}
/**
*
*/
protected function logError($name, $error) {
$this->stats['errors']++;
$message = E::ts("%1: failed to download Extension stats. Http response: %2", [
1 => $name,
2 => $error,
]);
Civi::log()->warning('Civicrmorgstats: ' . $message);
$this->stats['errors']++;
$this->stats['messages'][] = $message;
}
}
<?php
/**
* @file
* Code for the CiviCRM Extensions Stats module.
*/
require_once("civicrm_org_stats.inc");
class civicrm_org_stats_drupal extends civicrm_org_stats {
const URL = 'https://drupal.org/project/';
class CRM_Civicrmorgstats_Utils_Drupal extends CRM_Civicrmorgstats_Utils {
const URL = 'https://www.drupal.org/project/';
const DRUPAL = 128;
public function update() {
......@@ -20,43 +20,49 @@ class civicrm_org_stats_drupal extends civicrm_org_stats {
AND fq.field_extension_fq_name_value NOT LIKE '% %'
AND cms.field_extension_cms_tid = " . self::DRUPAL
);
foreach ($nodes as $node) {
// Drupal will return a 403 if we request too often
sleep(2);
$usage = $this->getStats(str_replace('-', '_', $node->name));
// No need to update
if ($usage === FALSE || (!empty($node->cu_node) && $node->usage_value == $usage)) {
if ($node->usage_value == $usage) {
// No need to update
$this->stats['unchanged']++;
continue;
}
$clear_cache = TRUE;
// Update existing field
if (!empty($node->cu_node)) {
$this->updateRecord($node, $usage);
}
// Insert if necessary
else {
$this->insertRecord($node, $usage);
}
}
if (isset($clear_cache)) {
$this->clearCache();
$this->stats['updated']++;
$this->updateRecord($node, $usage);
}
$this->clearCache();
}
/**
* Search project page for usage data.
* @param $name
* @return bool|int
* @return int
*/
private function getStats($name) {
$html = $this->fetch($name);
if ($html === FALSE) {
return FALSE;
$this->logError($name, "fixme http error");
return 0;
}
$html = stristr($html, 'project information</h3>');
$matches = [];
preg_match('#<strong>([0-9,]+)</strong>#', $html, $matches);
return !empty($matches[1]) ? str_replace(',', '', $matches[1]) : FALSE;
}
if (preg_match('#<strong>([0-9,]+)</strong>#', $html, $matches)) {
return str_replace(',', '', $matches[1]);
}
$this->logError($name, "Stats not found in the HTML");
return 0;
}
/**
* Scrape a drupal.org project page for usage stats
......@@ -64,10 +70,26 @@ class civicrm_org_stats_drupal extends civicrm_org_stats {
* @return bool|string
*/
private function fetch($name) {
$returnedRawValues = drupal_http_request(self::URL . $name);
if (empty($returnedRawValues->error) && !empty($returnedRawValues->data)) {
return $returnedRawValues->data;
try {
$response = $this->client->request('GET', self::URL . $name, [
'headers' => [
'User-Agent' => 'User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/68.0',
'Referer' => 'https://civicrm.org/extensions',
],
]);
return (string) $response->getBody()->getContents();
}
return FALSE;
catch (ClientException $e) {
$this->logError($name, $e->getResponse()->getStatusCode());
}
catch (Exception $e) {
// FIXME: temporary while debugging 403 errors from Drupal.org
die(print_r($e->getResponse()->getBody()->getContents(), 1));
}
return false;
}
}
<?php
use CRM_Civicrmorgstats_ExtensionUtil as E;
use GuzzleHttp\Exception\ClientException;
/**
* @file
* Code for the CiviCRM Extensions Stats module.
*/
class CRM_Civicrmorgstats_Utils_Extensions extends CRM_Civicrmorgstats_Utils {
const EXT_BASE_URL = "https://stats.civicrm.org/json/ext/";
const TID_NATIVE = 127;
/**
* Main update function
*/
public function update() {
$nodes = db_query("
SELECT fq.entity_id as nid, fq.field_extension_fq_name_value as name, fq.language, fq.revision_id,
cu.entity_id as cu_node, cu.field_extension_current_usage_value as usage_value
FROM field_data_field_extension_fq_name fq
INNER JOIN field_data_field_extension_cms cms ON cms.entity_id = fq.entity_id
LEFT JOIN field_data_field_extension_current_usage cu ON cu.entity_id = fq.entity_id
WHERE fq.deleted = 0 AND cms.field_extension_cms_tid = " . self::TID_NATIVE);
foreach ($nodes as $node) {
$usage = $this->getStats($node->name);
if ($node->usage_value == $usage) {
// No need to update
$this->stats['unchanged']++;
continue;
}
$this->stats['updated']++;
$this->updateRecord($node, $usage);
}
$this->clearCache();
}
/**
* Search project stats for usage data.
* @param $name
* @return int
*/
private function getStats($name) {
try {
$url = self::EXT_BASE_URL . $name . '.json';
$response = $this->client->request('GET', $url, [
'headers' => [
'User-Agent' => 'CiviCRM.org Stats Update',
],
]);
if ($response->getBody()) {
$body = $response->getBody();
$json = json_decode($body, true);
$usage = CRM_Utils_Array::value('num_sites', $json[0], 0);
return $usage;
}
}
catch (ClientException $e) {
$this->logError($name, $e->getResponse()->getStatusCode());
}
return 0;
}
}
This diff is collapsed.
# CiviCRM.org Stats
Provides Scheduled Jobs for updating Extension statistics. This is a port of
what used to be a Drupal7 module (civicrm_org_stats).
To report issues with this extension, please use:
https://lab.civicrm.org/extensions/extensions-directory/issues/
The extension is licensed under [AGPL-3.0](LICENSE.txt).
## Requirements
* PHP v7.2+
* CiviCRM 5.latest
## Installation
Install as a regular extension.
## Usage
Once enabled, two jobs will automatically be enabled (Civicrmorgstatsdrupal and Civicrmorgstatsextensions).
## Known issues
There is still some Drupal7-specific code:
* SQL queries assume a specific Entity table structure
* Usage of `db_query`
* Usage of `cache_clear_all`
<?php
/**
* @file
* This file declares a managed database record of type "Job".
*/
return [
0 => [
'name' => 'Cron:Job.CivicrmorgstatsDrupal',
'entity' => 'Job',
'params' => [
'version' => 3,
'name' => 'CiviCRM.org Stats for Drupal',
'description' => 'Updates CiviCRM.org Stats for Drupal',
'run_frequency' => 'Weekly',
'api_entity' => 'Job',
'api_action' => 'Civicrmorgstatsdrupal',
'parameters' => '',
],
'update' => 'never',
],
];
<?php
function civicrm_api3_job_civicrmorgstatsdrupal($params) {
$helper = new CRM_Civicrmorgstats_Utils_Drupal();
$stats = $helper->update();
$output = E::ts('Updated: %1, Unchanged: %2, Errors: %3.', [
1 => $stats['updated'],
2 => $stats['unchanged'],
3 => $stats['errors'],
]);
$output .= '<br><br>' . implode('<br>', $stats['messages']);
return civicrm_api3_create_success($output);
}
<?php
/**
* @file
* This file declares a managed database record of type "Job".
*/
return [
0 => [
'name' => 'Cron:Job.Civicrmorgstatsextensions',
'entity' => 'Job',
'params' => [
'version' => 3,
'name' => 'CiviCRM.org Stats for Extensions',
'description' => 'Updates CiviCRM.org Stats for Extensions',
'run_frequency' => 'Weekly',
'api_entity' => 'Job',
'api_action' => 'Civicrmorgstatsextensions',
'parameters' => '',
],
'update' => 'never',
],
];
<?php
use CRM_Civicrmorgstats_ExtensionUtil as E;
function civicrm_api3_job_civicrmorgstatsextensions($params) {
$helper = new CRM_Civicrmorgstats_Utils_Extensions();
$stats = $helper->update();
$output = E::ts('Updated: %1, Unchanged: %2, Errors: %3.', [
1 => $stats['updated'],
2 => $stats['unchanged'],
3 => $stats['errors'],
]);
$output .= '<br><br>' . implode('<br>', $stats['messages']);
return civicrm_api3_create_success($output);
}
This diff is collapsed.
<?php
require_once 'civicrmorgstats.civix.php';
use CRM_Civicrmorgstats_ExtensionUtil as E;
/**
* Implements hook_civicrm_config().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_config/
*/
function civicrmorgstats_civicrm_config(&$config) {
_civicrmorgstats_civix_civicrm_config($config);
}
/**
* Implements hook_civicrm_xmlMenu().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_xmlMenu
*/
function civicrmorgstats_civicrm_xmlMenu(&$files) {
_civicrmorgstats_civix_civicrm_xmlMenu($files);
}
/**
* Implements hook_civicrm_install().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_install
*/
function civicrmorgstats_civicrm_install() {
_civicrmorgstats_civix_civicrm_install();
}
/**
* Implements hook_civicrm_postInstall().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_postInstall
*/
function civicrmorgstats_civicrm_postInstall() {
_civicrmorgstats_civix_civicrm_postInstall();
}
/**
* Implements hook_civicrm_uninstall().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_uninstall
*/
function civicrmorgstats_civicrm_uninstall() {
_civicrmorgstats_civix_civicrm_uninstall();
}
/**
* Implements hook_civicrm_enable().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_enable
*/
function civicrmorgstats_civicrm_enable() {
_civicrmorgstats_civix_civicrm_enable();
}
/**
* Implements hook_civicrm_disable().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_disable
*/
function civicrmorgstats_civicrm_disable() {
_civicrmorgstats_civix_civicrm_disable();
}
/**
* Implements hook_civicrm_upgrade().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_upgrade
*/
function civicrmorgstats_civicrm_upgrade($op, CRM_Queue_Queue $queue = NULL) {
return _civicrmorgstats_civix_civicrm_upgrade($op, $queue);
}
/**
* Implements hook_civicrm_managed().
*
* Generate a list of entities to create/deactivate/delete when this module
* is installed, disabled, uninstalled.
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_managed
*/
function civicrmorgstats_civicrm_managed(&$entities) {
_civicrmorgstats_civix_civicrm_managed($entities);
}
/**
* Implements hook_civicrm_angularModules().
*
* Generate a list of Angular modules.
*
* Note: This hook only runs in CiviCRM 4.5+. It may
* use features only available in v4.6+.
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_angularModules
*/
function civicrmorgstats_civicrm_angularModules(&$angularModules) {
_civicrmorgstats_civix_civicrm_angularModules($angularModules);
}
/**
* Implements hook_civicrm_alterSettingsFolders().
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_alterSettingsFolders
*/
function civicrmorgstats_civicrm_alterSettingsFolders(&$metaDataFolders = NULL) {
_civicrmorgstats_civix_civicrm_alterSettingsFolders($metaDataFolders);
}
/**
* Implements hook_civicrm_entityTypes().
*
* Declare entity types provided by this module.
*
* @link https://docs.civicrm.org/dev/en/latest/hooks/hook_civicrm_entityTypes
*/
function civicrmorgstats_civicrm_entityTypes(&$entityTypes) {
_civicrmorgstats_civix_civicrm_entityTypes($entityTypes);
}
/**
* Implements hook_civicrm_thems().
*/
function civicrmorgstats_civicrm_themes(&$themes) {
_civicrmorgstats_civix_civicrm_themes($themes);
}
<?xml version="1.0"?>
<extension key="org.civicrm.civicrmorgstats" type="module">
<file>civicrmorgstats</file>
<name>CiviCRM.org Stats</name>
<description>Calculates various statistics for civicrm.org</description>
<license>AGPL-3.0</license>
<maintainer>
<author>Various Authors</author>
<email>info@civicrm.org</email>
</maintainer>
<urls>
<url desc="Main Extension Page">http://FIXME</url>
<url desc="Documentation">http://FIXME</url>
<url desc="Support">http://FIXME</url>
<url desc="Licensing">http://www.gnu.org/licenses/agpl-3.0.html</url>
</urls>
<releaseDate>2019-09-02</releaseDate>
<version>1.0</version>
<develStage>alpha</develStage>
<compatibility>
<ver>5.0</ver>
</compatibility>
<comments></comments>
<civix>
<namespace>CRM/Civicrmorgstats</namespace>
</civix>
</extension>
civicrm_org_stats
=================
Provides a few Drupal tokens for CiviCRM stats.
Extensions stats are not updated via org.civicrm.civiorgstats
<?php
/**
* @file
* Code for the CiviCRM Extensions Stats module.
*/
require_once("civicrm_org_stats.inc");
class civicrm_org_stats_extensions extends civicrm_org_stats {
private $stats = array();
const URL = "https://stats.civicrm.org/json/extensions-detail.json";
public function fetch() {
$returnedRawValues = drupal_http_request(self::URL);
if (!empty($returnedRawValues->data)) {
$decodedValues = json_decode($returnedRawValues->data, TRUE);
}
if (!empty($decodedValues)) {
foreach ($decodedValues as $val) {
if (!empty($val['name'])) {
$this->stats[$val['name']] = (int) $val['num_sites'];
}
}
return TRUE;
}
return FALSE;
}
public function update() {
$nodes = db_query("
SELECT fq.entity_id as nid, fq.field_extension_fq_name_value as name, fq.language, fq.revision_id, cu.entity_id as cu_node, cu.field_extension_current_usage_value as usage_value
FROM field_data_field_extension_fq_name fq
LEFT JOIN field_data_field_extension_current_usage cu ON cu.entity_id = fq.entity_id
WHERE fq.deleted = 0
AND fq.field_extension_fq_name_value IN (:keys)",
array(':keys' => array_keys($this->stats))
);
foreach ($nodes as $node) {
$usage = $this->stats[$node->name];
// No need to update
if (!empty($node->cu_node) && $node->usage_value == $usage) {
continue;
}
$clear_cache = TRUE;
// Update existing field
if (!empty($node->cu_node)) {
$this->updateRecord($node, $usage);
}
// Insert if necessary
else {
$this->insertRecord($node, $usage);
}
}
if (isset($clear_cache)) {
$this->clearCache();
}
}
}
<?php
/**
* @file
* Code for the CiviCRM Extensions Stats module.
*/
class civicrm_org_stats {
protected function updateRecord($node, $usage) {
db_query("UPDATE field_data_field_extension_current_usage SET field_extension_current_usage_value = :usage_value WHERE entity_id = {$node->nid} AND entity_type = 'node'", array(':usage_value' => $usage));
db_query("UPDATE field_revision_field_extension_current_usage SET field_extension_current_usage_value = :usage_value WHERE entity_id = {$node->nid} AND entity_type = 'node' and revision_id = {$node->revision_id}", array(':usage_value' => $usage));
}
protected function insertRecord($node, $usage) {
db_query("INSERT INTO field_data_field_extension_current_usage (entity_type, bundle, entity_id, revision_id, `language`, delta, field_extension_current_usage_value)
VALUES ('node', 'extension', {$node->nid}, {$node->revision_id}, '{$node->language}', 0, :usage_value)", array(':usage_value' => $usage));
db_query("INSERT INTO field_revision_field_extension_current_usage (entity_type, bundle, entity_id, revision_id, `language`, delta, field_extension_current_usage_value)
VALUES ('node', 'extension', {$node->nid}, {$node->revision_id}, '{$node->language}', 0, :usage_value)", array(':usage_value' => $usage));
}
/**
* Clear drupal's caches related to node field data
*/
protected function clearCache() {
cache_clear_all('*', 'cache_field', TRUE);
cache_clear_all('*', 'cache_page', TRUE);
}
}
......@@ -5,34 +5,6 @@
* CiviCRM Stats module
*/
/**
* Implements hook_cron()
* Update extension stats weekly.
*/
function civicrm_org_stats_cron()
{
$now = time();
$last_run = variable_get('civicrm_org_stats_last_run', 0);
if ($last_run == 'drupal') {
require_once 'civicrm_org_stats.drupal.inc';
$stats = new civicrm_org_stats_drupal();
$stats->update();
// See you next week :)
variable_set('civicrm_org_stats_last_run', $now);
}
// Has it been at least 6 days since last run?
elseif ($last_run < ($now - (60 * 60 * 24 * 6))) {
require_once 'civicrm_org_stats.extensions.inc';