Commit e4827476 authored by cividesk's avatar cividesk

Initial commit: Github, pingback, Sourceforge and JIRA working.

Generate working, and Klipfolio reading from generated JSON.
Code in production on sushi. Tested and working.
parents
/.idea
/vendor
/json
\ No newline at end of file
# CiviCRM Statistics gathering
This package's purpose is to:
1. gather project statistics from various sources
2. summarize these statistics in json files
3. use these json files in project dashboards
Goals 1 and 2 are realized by this code. Goal 3 is realized by leveraging the published json files from 2
in external dashboard services such as Klipfolio or any other service capable of ingesting json.
# 1. Gather project statistics
This is done by the 'getdata.php' scripts in each subdirectory. Thea reason these scripts are independant is that we
might need different refresh schedules for each data source.
# 2. Summarize the stats in json files
This is done by the 'generate.php' script in the project root. This file will consume the 'generate.inc.php' files in
each subdirectory to gather the json file name and query needed to be run to generate it.
# Other considerations
You should run composer to download the project dependencies.
The project file 'config.php' is not published as it contains sensitive data.
Each subdirectory might also contain:
- a 'db_create.sql' for creating the tables it needs in the database
- a 'refs' folder with links and/or documentation that helped in the development
\ No newline at end of file
<?php
/*
* Maintains up to date the common tables used for reporting
*/
require_once(dirname(__DIR__) . '/config.php');
// Initialize database and prepared statements
$dbh = new PDO('mysql:dbname='.DBNAME.';host='.DBHOST, DBUSER, DBPASS);
$stm_m = $dbh->prepare("
INSERT INTO common_month (month)
VALUES (:month)
");
/*
* The common_month table has all months from the beginning of the project till now
*/
// Get the latest month already in the database
$result = $dbh->query("SELECT MAX(month) FROM common_month;")->fetch();
if (empty($result[0])) $result[0] = DATESTART;
$current = $result[0];
// And loop until today, creating each month as we go
while (strtotime($current) < time()) {
$current = date('Y-m-01 00:00:00', strtotime("$current +1 month"));
$stm_m->bindParam(':month', $current);
$stm_m->execute();
}
{
"minimum-stability": "dev",
"prefer-stable": true,
"require": {
"chobie/jira-api-restclient": "2.0.*"
}
}
<?php
const DBHOST = 'localhost';
const DBNAME = 'civicrm_stats';
const DBUSER = 'root';
const DBPASS = '';
const DBPING = 'civicrm_raw_stats';
const DATESTART = '2004-12-07 06:13:00'; // Date of the start of the project (from JIRA)
\ No newline at end of file
-- phpMyAdmin SQL Dump
-- version 4.0.8
-- http://www.phpmyadmin.net
--
-- Host: localhost
-- Generation Time: Sep 14, 2014 at 07:46 PM
-- Server version: 5.1.71
-- PHP Version: 5.3.3
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET time_zone = "+00:00";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
--
-- Table structure for table `common_month`
--
DROP TABLE IF EXISTS `common_month`;
CREATE TABLE IF NOT EXISTS `common_month` (
`month` datetime NOT NULL,
PRIMARY KEY (`month`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-- --------------------------------------------------------
--
-- Table structure for table `github_commit`
--
DROP TABLE IF EXISTS `github_commit`;
CREATE TABLE IF NOT EXISTS `github_commit` (
`repository` varchar(100) CHARACTER SET ascii NOT NULL,
`hash` char(40) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
`author_login` varchar(100) CHARACTER SET ascii NOT NULL,
`author_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`committer_login` varchar(100) CHARACTER SET ascii NOT NULL,
`committer_date` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`message` varchar(1000) COLLATE utf8_unicode_ci DEFAULT NULL,
UNIQUE KEY `hash` (`hash`),
KEY `repository` (`repository`),
KEY `author_login` (`author_login`),
KEY `author_date` (`author_date`),
KEY `committer_login` (`committer_login`),
KEY `committer_date` (`committer_date`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-- --------------------------------------------------------
--
-- Table structure for table `github_user`
--
DROP TABLE IF EXISTS `github_user`;
CREATE TABLE IF NOT EXISTS `github_user` (
`id` int(11) NOT NULL,
`login` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`name` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`company` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`email` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`location` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`avatar_url` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
PRIMARY KEY (`login`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-- --------------------------------------------------------
--
-- Table structure for table `jira_issue`
--
DROP TABLE IF EXISTS `jira_issue`;
CREATE TABLE IF NOT EXISTS `jira_issue` (
`jira_id` int(11) NOT NULL COMMENT 'Needed only for proper display in PHPMyAdmin',
`project` varchar(8) COLLATE utf8_unicode_ci NOT NULL,
`issue` varchar(16) COLLATE utf8_unicode_ci NOT NULL,
`summary` varchar(128) COLLATE utf8_unicode_ci DEFAULT NULL,
`type` varchar(16) COLLATE utf8_unicode_ci DEFAULT NULL,
`priority` varchar(16) COLLATE utf8_unicode_ci DEFAULT NULL,
`reporter` varchar(32) COLLATE utf8_unicode_ci DEFAULT NULL,
`assignee` varchar(32) COLLATE utf8_unicode_ci DEFAULT NULL,
`status` varchar(32) COLLATE utf8_unicode_ci NOT NULL,
`resolution` varchar(32) COLLATE utf8_unicode_ci DEFAULT NULL,
`created` datetime NOT NULL,
`updated` datetime DEFAULT NULL,
`resolved` datetime DEFAULT NULL,
PRIMARY KEY (`jira_id`),
UNIQUE KEY `issue` (`issue`),
KEY `created` (`created`),
KEY `resolved` (`resolved`),
KEY `reporter` (`reporter`),
KEY `assignee` (`assignee`),
KEY `status` (`status`),
KEY `resolution` (`resolution`),
KEY `type` (`type`),
KEY `priority` (`priority`),
KEY `project` (`project`),
KEY `updated` (`updated`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-- --------------------------------------------------------
--
-- Table structure for table `jira_version`
--
DROP TABLE IF EXISTS `jira_version`;
CREATE TABLE IF NOT EXISTS `jira_version` (
`issue` varchar(16) COLLATE utf8_unicode_ci NOT NULL,
`type` enum('Affects','Fix') COLLATE utf8_unicode_ci NOT NULL,
`version` varchar(16) COLLATE utf8_unicode_ci NOT NULL,
KEY `issue` (`issue`),
KEY `version` (`version`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-- --------------------------------------------------------
--
-- Table structure for table `pingback_extension`
--
DROP TABLE IF EXISTS `pingback_extension`;
CREATE TABLE IF NOT EXISTS `pingback_extension` (
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`num_sites` int(11) unsigned DEFAULT NULL,
PRIMARY KEY (`name`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-- --------------------------------------------------------
--
-- Table structure for table `pingback_site`
--
DROP TABLE IF EXISTS `pingback_site`;
CREATE TABLE IF NOT EXISTS `pingback_site` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`hash` varchar(32) COLLATE ascii_bin DEFAULT NULL,
`version` text COLLATE ascii_bin,
`lang` text COLLATE ascii_bin,
`uf` text COLLATE ascii_bin,
`ufv` text COLLATE ascii_bin,
`MySQL` text COLLATE ascii_bin,
`PHP` text COLLATE ascii_bin,
`first_ping_id` bigint(20) unsigned NOT NULL,
`first_timestamp` timestamp NULL DEFAULT NULL,
`last_ping_id` bigint(20) unsigned NOT NULL,
`last_timestamp` timestamp NULL DEFAULT NULL,
`num_pings` int(11) unsigned DEFAULT '1',
`is_active` int(1) unsigned DEFAULT NULL,
`Contact` int(11) unsigned DEFAULT NULL,
`Contribution` int(11) unsigned DEFAULT NULL,
`Participant` int(11) unsigned DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `hash` (`hash`),
KEY `first_timestamp` (`first_timestamp`),
KEY `last_ping_id` (`last_ping_id`),
KEY `last_timestamp` (`last_timestamp`),
KEY `is_active` (`is_active`)
) ENGINE=MyISAM DEFAULT CHARSET=ascii COLLATE=ascii_bin AUTO_INCREMENT=120191 ;
-- --------------------------------------------------------
--
-- Table structure for table `sourceforge_download`
--
DROP TABLE IF EXISTS `sourceforge_download`;
CREATE TABLE IF NOT EXISTS `sourceforge_download` (
`type` char(1) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
`label` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`value` int(10) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
<?php
require_once('config.php');
$directory = __DIR__ . '/json';
// Initialize database
$dbh = new PDO('mysql:dbname='.DBNAME.';host='.DBHOST, DBUSER, DBPASS);
// Initialize the queries array from each subdirectories' include file
$queries = array();
$files = glob_recursive('generate.inc.php');
foreach ($files as $file) {
require_once($file);
}
// Create target directory if non existent
if (!file_exists($directory)) {
mkdir($directory);
}
// Run the queries and generate target file
foreach ($queries as $query) {
echo "Creating $query[file] ...";
$count = 0;
$out = '[';
foreach($dbh->query($query['query'], PDO::FETCH_ASSOC) as $row) {
$out .= json_encode($row) . ',';
$count ++;
}
$out[strlen($out)-1] = ']';
file_put_contents($directory . '/' . $query['file'], $out);
echo " ($count records)" . PHP_EOL;
}
function glob_recursive($pattern, $flags = 0) {
$files = glob($pattern, $flags);
foreach (glob(dirname($pattern).'/*', GLOB_ONLYDIR|GLOB_NOSORT) as $dir)
$files = array_merge($files, glob_recursive($dir.'/'.basename($pattern), $flags));
return $files;
}
\ No newline at end of file
<?php
$queries[] = array(
'file' => "commits-by-month.json",
'query' => "
SELECT `date`, commits, (@runtot := @runtot + q1.commits) AS runtot
FROM (
SELECT DATE_FORMAT(committer_date, '%Y-%m') AS `date`, COUNT(*) AS commits
FROM github_commit
GROUP BY `date`
ORDER BY `date` ASC
) q1
JOIN (SELECT @runtot := 0 AS var1) init;
",
);
$queries[] = array(
'file' => "commits-by-author.json",
'query' => "
SELECT u.login AS login, u.name, u.company, u.location, COUNT(*) AS commits
FROM github_commit c
LEFT JOIN github_user u ON u.login = c.author_login
WHERE c.author_login > ''
GROUP BY login
ORDER BY commits DESC
",
);
\ No newline at end of file
<?php
require_once(dirname(__DIR__) . '/config.php');
require_once(__DIR__ . '/github-php-client-master/client/GitHubClient.php');
$repos = array(
'civicrm/civicrm-core',
'civicrm/civicrm-packages',
'civicrm/civicrm-drupal',
'civicrm/civicrm-wordpress',
'civicrm/civicrm-joomla',
);
// Initialize database and prepared statements
$dbh = new PDO('mysql:dbname='.DBNAME.';host='.DBHOST, DBUSER, DBPASS);
$stm_c = $dbh->prepare("
INSERT INTO github_commit (repository, hash, author_login, author_date, committer_login, committer_date, message)
VALUES (:repository, :hash, :author_login, :author_date, :committer_login, :committer_date, :message);
");
$stm_u = $dbh->prepare("
INSERT INTO github_user (id, login, name, company, email, location, avatar_url)
VALUES (:id, :login, :name, :company, :email, :location, :avatar_url)
ON DUPLICATE KEY UPDATE
id=:id, login=:login, name=:name, company=:company, email=:email, location=:location, avatar_url=:avatar_url;
");
// Initialize GitHub
$client = new GitHubClient();
$client->setCredentials('nganivet', '6CS9kJ4drP');
// Loop over each repository
$users = array();
foreach($repos as $repo) {
echo "Repository $repo ...";
$new_commits = 0;
$stm_c->bindParam(':repository', $repo);
// Get last commit date in database
$result = $dbh->query("SELECT MAX(author_date) FROM github_commit WHERE repository='$repo';")->fetch();
// Get commits from GitHub
$parts = explode('/', $repo);
$client->setPage(); // reinitialize the results pager
$commits = $client->repos->commits->listCommitsOnRepository($parts[0], $parts[1], null, null, null, $result[0]);
// Loop over each commit (paged)
while (sizeof($commits)) {
foreach($commits as $commit) {
/* @var $commit GitHubCommit */
// Write commit to database
$stm_c->bindParam(':hash', $commit->getSha());
$author_login = ($commit->getAuthor() ? $commit->getAuthor()->getLogin() : '');
$stm_c->bindParam(':author_login', $author_login);
$stm_c->bindParam(':author_date', $commit->getCommit()->getAuthor()->getDate());
$committer_login = ($commit->getCommitter() ? $commit->getCommitter()->getLogin() : '');
$stm_c->bindParam(':committer_login', $committer_login);
$stm_c->bindParam(':committer_date', $commit->getCommit()->getCommitter()->getDate());
$stm_c->bindParam(':message', $commit->getCommit()->getMessage());
$stm_c->execute();
$new_commits++;
}
$commits = $client->getNextPage();
}
echo " $new_commits new commit(s)" . PHP_EOL;
}
// Now update all user records
echo "Updating users ...";
echo $updated = 0;
$query = "SELECT DISTINCT author_login FROM github_commit WHERE author_login > ''";
foreach ($dbh->query($query) as $row) {
$client->setPage(); // reinitialize the results pager
$user = $client->users->getSingleUser($row[0]);
/* @var $user GitHubFullUser */
if ($user) {
$stm_u->bindParam(':id', $user->getId());
$stm_u->bindParam(':login', $user->getLogin());
$stm_u->bindParam(':name', $user->getName());
$stm_u->bindParam(':company', $user->getCompany());
$stm_u->bindParam(':email', $user->getEmail());
$stm_u->bindParam(':location', $user->getLocation());
$stm_u->bindParam(':avatar_url', $user->getAvatarUrl());
$stm_u->execute();
$updated ++;
} else {
echo "Unknown user - login: " . $row[0] . "\n";
}
}
echo " $updated updates" . PHP_EOL;
\ No newline at end of file
# GitHub API PHP Client
See [full API reference](https://github.com/tan-tan-kanarek/github-php-client/blob/master/client.md "full API reference")
## Authenticating
```php
<?php
require_once(__DIR__ . '/client/GitHubClient.php');
$client = new GitHubClient();
$client->setCredentials($username, $password);
```
## Listing commits
```php
<?php
require_once(__DIR__ . '/client/GitHubClient.php');
$owner = 'tan-tan-kanarek';
$repo = 'github-php-client';
$client = new GitHubClient();
$client->setPage();
$client->setPageSize(2);
$commits = $client->repos->commits->listCommitsOnRepository($owner, $repo);
echo "Count: " . count($commits) . "\n";
foreach($commits as $commit)
{
/* @var $commit GitHubCommit */
echo get_class($commit) . " - Sha: " . $commit->getSha() . "\n";
}
$commits = $client->getNextPage();
echo "Count: " . count($commits) . "\n";
foreach($commits as $commit)
{
/* @var $commit GitHubCommit */
echo get_class($commit) . " - Sha: " . $commit->getSha() . "\n";
}
```
## Listing issues
```php
<?php
require_once(__DIR__ . '/client/GitHubClient.php');
$owner = 'tan-tan-kanarek';
$repo = 'github-php-client';
$client = new GitHubClient();
$client->setPage();
$client->setPageSize(2);
$issues = $client->issues->listIssues($owner, $repo);
foreach ($issues as $issue)
{
/* @var $issue GitHubIssue */
echo get_class($issue) . "[" . $issue->getNumber() . "]: " . $issue->getTitle() . "\n";
}
```
## Creating a release
```php
<?php
require_once(__DIR__ . '/client/GitHubClient.php');
$owner = 'tan-tan-kanarek';
$repo = 'github-php-client';
$username = 'tan-tan-kanarek';
$password = 'myPassword';
$tag_name = 'myTag';
$target_commitish = 'master';
$name = 'myReleaseName';
$body = 'My release description';
$draft = false;
$prerelease = true;
$client = new GitHubClient();
$client->setDebug(true);
$client->setCredentials($username, $password);
$release = $client->repos->releases->create($owner, $repo, $tag_name, $target_commitish, $name, $body, $draft, $prerelease);
/* @var $release GitHubReposRelease */
$releaseId = $release->getId();
$filePath = 'C:\myPath\bin\myFile.jar';
$contentType = 'application/java-archive';
$name = 'MyFile-1.0.0.jar';
$client->repos->releases->assets->upload($owner, $repo, $releaseId, $name, $contentType, $filePath);
```
[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/ivanfemia/github-php-client/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
This diff is collapsed.
<?php
require_once(__DIR__ . '/GitHubClientBase.php');
require_once(__DIR__ . '/services/GitHubActivity.php');
require_once(__DIR__ . '/services/GitHubChangelog.php');
require_once(__DIR__ . '/services/GitHubGists.php');
require_once(__DIR__ . '/services/GitHubGit.php');
require_once(__DIR__ . '/services/GitHubGitignore.php');
require_once(__DIR__ . '/services/GitHubIssues.php');
require_once(__DIR__ . '/services/GitHubLibraries.php');
require_once(__DIR__ . '/services/GitHubMarkdown.php');
require_once(__DIR__ . '/services/GitHubMedia.php');
require_once(__DIR__ . '/services/GitHubMeta.php');
require_once(__DIR__ . '/services/GitHubOauth.php');
require_once(__DIR__ . '/services/GitHubOrgs.php');
require_once(__DIR__ . '/services/GitHubPulls.php');
require_once(__DIR__ . '/services/GitHubRepos.php');
require_once(__DIR__ . '/services/GitHubSearch.php');
require_once(__DIR__ . '/services/GitHubUsers.php');
class GitHubClient extends GitHubClientBase
{
/**
* @var GitHubActivity
*/
public $activity;
/**
* @var GitHubChangelog
*/
public $changelog;
/**
* @var GitHubGists
*/
public $gists;
/**
* @var GitHubGit
*/
public $git;
/**
* @var GitHubGitignore
*/
public $gitignore;
/**
* @var GitHubIssues
*/
public $issues;
/**
* @var GitHubLibraries
*/
public $libraries;
/**
* @var GitHubMarkdown
*/
public $markdown;
/**
* @var GitHubMedia
*/
public $media;
/**
* @var GitHubMeta
*/
public $meta;
/**
* @var GitHubOauth
*/
public $oauth;
/**
* @var GitHubOrgs
*/
public $orgs;
/**
* @var GitHubPulls
*/
public $pulls;
/**
* @var GitHubRepos
*/
public $repos;
/**
* @var GitHubSearch
*/
public $search;
/**
* @var GitHubUsers
*/
public $users;
/**
* Initialize sub services
*/
public function __construct()
{
$this->activity = new GitHubActivity($this);
$this->changelog = new GitHubChangelog($this);
$this->gists = new GitHubGists($this);
$this->git = new GitHubGit($this);
$this->gitignore = new GitHubGitignore($this);
$this->issues = new GitHubIssues($this);
$this->libraries = new GitHubLibraries($this);
$this->markdown = new GitHubMarkdown($this);
$this->media = new GitHubMedia($this);
$this->meta = new GitHubMeta($this);
$this->oauth = new GitHubOauth($this);
$this->orgs = new GitHubOrgs($this);
$this->pulls = new GitHubPulls($this);
$this->repos = new GitHubRepos($this);
$this->search = new GitHubSearch($this);
$this->users = new GitHubUsers($this);
}