civiextensions-update-transifex.php 7.94 KB
Newer Older
1
#!/usr/bin/env php
2 3 4 5 6
<?php

/**
 * For all civicrm native extensions (cms-agnostic) published in
 * the civicrm.org/extensions directory,
7
 *
8 9 10
 * - clone the repo and extract the strings from the latest version,
 * - the extracted .pot files are stored in the civicrm-l10n-extensions
 *   directory,
11 12 13 14 15
 * - check if the tag was already processed,
 * - push the source strings to Transifex (project/civicrm_extensions),
 * - pull translated strings from Transifex
 * - add the new extensions to git.
 * - TODO: commit new translated strings for all extensions (using commit-to-git.sh).
16 17 18 19 20 21 22 23
 *
 * Depends on packages: php5-cli, moreutils, gettext.
 * Depends on repositories: l10n (until those scripts are moved to civix).
 *
 * For more information, view the README.md and:
 * https://issues.civicrm.org/jira/browse/INFRA-104
 */

24 25 26 27
if (empty($_SERVER['HOME'])) {
  echo "ERROR: \$_SERVER['HOME'] was not set. I'm lost.\n";
  exit(1);
}
28

29
// civicrm.org URL that lists us the git repos for extensions
30
define('CIVIEXTENSIONS_REPO_LIST', 'https://civicrm.org/extdir/git-urls.json');
31

32 33 34
// cache directory on this server where we clone git repos
// no need for a trailing slash
define('CIVIEXTENSIONS_CACHE_DIR', $_SERVER['HOME'] . "/repositories/cache");
35

36 37 38 39 40 41 42 43 44 45 46 47 48
// data directory where we keep track of processed tags
// to avoid extracting strings on known versions over and over.
define('CIVIEXTENSIONS_METADATA_DIR', $_SERVER['HOME'] . '/repositories/.civiextensions-metadata');

// repository with the .pot files generated
// that we will commit to git to have a history similar to l10n/pot/
// i.e.: po/[shortname]/xx_XX/[shortname].po
define('CIVIEXTENSIONS_L10N_REPO_DIR', $_SERVER['HOME'] . '/repositories/civicrm-l10n-extensions/');

function main() {
  // cache directory on this server where we clone git repos
  // no need for a trailing slash
  $download_dir = CIVIEXTENSIONS_CACHE_DIR;
49 50 51 52 53

  if (! file_exists($download_dir)) {
    mkdir($download_dir, 0755, TRUE);
  }

54 55 56 57 58 59 60 61 62 63 64 65 66
  // data directory where we keep track of processed tags
  // to avoid extracting strings on known versions over and over.
  if (! file_exists(CIVIEXTENSIONS_METADATA_DIR)) {
    mkdir(CIVIEXTENSIONS_METADATA_DIR, 0755, TRUE);
  }

  // repository with the .pot files generated
  // that we will commit to git to have a history similar to l10n/pot/
  // i.e.: po/[shortname]/xx_XX/[shortname].po
  $l10n_repo_dir = CIVIEXTENSIONS_L10N_REPO_DIR;

  if (! file_exists($l10n_repo_dir)) {
    echo "$l10n_repo_dir: directory does not exist. You need to clone the civicrm-l10n-extensions repo first in $l10n_repo_dir.\n";
67 68 69 70
    exit(1);
  }

  // Fetch the list of repos and clone locally
71
  $gitrepos_raw = file_get_contents(CIVIEXTENSIONS_REPO_LIST);
72 73 74 75 76 77 78 79

  if (! $gitrepos_raw) {
    echo "ERROR: could not read the list of repositories for extensions.";
    exit(1);
  }

  $gitrepos = json_decode($gitrepos_raw);

80 81 82 83 84 85
  foreach ($gitrepos as $gitinfo) {
    // Only extract extensions that are ready for automatic distribution
    if ($gitinfo->ready != 'ready') {
      continue;
    }

86
    // Risky security
87 88
    if (! preg_match('/^[-_0-9-a-zA-Z\.:\/]+$/', $gitinfo->git_url)) {
      echo "Skipping suspicious repo URL: {$gitinfo->git_url}\n";
89 90 91
      continue;
    }

92
    $reponame = basename($gitinfo->git_url);
93
    $reponame = preg_replace('/\.git$/', '', $reponame);
94

95
    $extdir = escapeshellarg("$download_dir/$reponame");
96 97 98

    echo "Repo: $extdir\n";

99 100 101
    if (file_exists("$download_dir/$reponame")) {
      echo "* Updating with git fetch --all..\n";
      system("cd $extdir; git fetch -q --all");
102 103 104
    }
    else {
      // Runs: git clone git://example.org/foo.git ~/repositories/foo-hash
105
      echo "* New repo (or not in cache), cloning it...\n";
106
      system('git clone ' . escapeshellarg($gitinfo->git_url) . ' ' . $extdir);
107 108 109 110 111 112 113 114 115 116
    }

    // Get the short name from the info.xml
    // TODO: fix civihr use-case (packages).
    // NB: do not use $extdir, since it is a shell-escaped variable.
    $shortname = civiextensions_getname_from_xml("$download_dir/$reponame");

    if (! $shortname) {
      print "Info: could not get short name for $reponame, skipping.\n";
      continue;
117 118 119 120 121 122 123 124
    }

    // NB: we check for tags having the format: v1.0 1.0 1.0.0
    // meaning that we ignore anything such as 1.0-beta1
    // since the alphabetical sort would not make sense, and we
    // cannot predict what funky tags people may use.
    $tag = system("cd $extdir; git tag | grep -E '^v?[-\.0-9]+$' | sort -n | tail -1");

125 126 127 128 129 130 131 132 133
    // Do not process the extension unless it is a new tag
    $last_processed_tag = civiextensions_get_last_processed_tag($shortname);

    if ($tag <= $last_processed_tag) {
      echo "* Skipping: tag already processed: $tag\n";
      continue;
    }

    echo "* Processing tag: $tag ...\n";
134 135
    system("cd $extdir; git checkout tags/" . $tag);

136 137 138 139 140 141 142 143 144
    // If the directory does not exists, we assume that it means
    // that we also need to create the resource in Transifex.
    // If it's a new resource, it needs to be added after pot generation.
    $add_to_transifex = FALSE;

    if (! file_exists("$l10n_repo_dir/po/$shortname")) {
      $add_to_transifex = TRUE;
    }

145
    // Extract the ts() strings
146
    system("cd $extdir; env POTDIR=$l10n_repo_dir/po CIVI_KEEP_IT_QUIET=1 ~/l10n/civicrm-l10n-core/bin/create-pot-files-extensions.sh");
147 148 149 150 151 152 153 154

    // Send strings to Transifex, add new resource if necessary
    // NB: we do not pull the strings at this point, it will be done
    // by another script that pulls in translations every day.
    if ($add_to_transifex) {
      print "Adding Transifex resource: $shortname\n";
      system("cd $l10n_repo_dir; tx set --auto-local -r civicrm_extensions.$shortname 'po/$shortname/<lang>/$shortname.po' --source-lang en --source-file po/$shortname/pot/$shortname.pot --execute");
      system("cd $l10n_repo_dir; git add .tx/config; git commit -m 'Adding new extension: $shortname'");
155 156
    }

157 158 159 160 161 162 163 164
    system("cd $l10n_repo_dir; tx push -s -r civicrm_extensions.$shortname");

    // Commit to civicrm-l10n-extensions repo and push
    system("cd $l10n_repo_dir; git add $l10n_repo_dir/po/$shortname; git commit -m 'Automatic commit for version $tag of extension $shortname'");
    system("cd $l10n_repo_dir; git push origin master");

    civiextensions_set_last_processed_tag($shortname, $tag);
    echo "* Done, yay!\n\n";
165 166 167
  }
}

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
/**
 * Returns the "file" attribute from the info.xml of
 * an extension. Ex: for ca.bidon.i18nexample, it will
 * return "i18nexample".
 *
 * We cannot assume that it is the last bit of the name,
 * because many extensions have a repo name that does
 * not match their namespace (ex: civivolunteer).
 *
 * @param String $extdir Path to the extension.
 * @returns Mixed File attribute, false if none found.
 */
function civiextensions_getname_from_xml($extdir) {
  if (! file_exists($extdir . '/info.xml')) {
    print "Could not find info.xml\n";
    return FALSE;
  }

  $info = simplexml_load_file($extdir . '/info.xml');

  if (! $info) {
    print "Could not parse info.xml\n";
    return FALSE;
  }

  if (! preg_match('/^[A-Za-z0-9][-_A-Za-z0-9]*$/', $info->file)) {
    print "File attribute from info.xml is suspicious\n";
    return FALSE;
  }

  return $info->file;
}

/**
 * Fetches the last processed tag that is stored in the
 * ~/repositories/.civiextensions-tags/$shorname.tag
 */
function civiextensions_get_last_processed_tag($shortname) {
  $data_dir = CIVIEXTENSIONS_METADATA_DIR;
  $filename = "$data_dir/{$shortname}.tag";

  if (! file_exists($filename)) {
    return FALSE;
  }

  $tag = file_get_contents($filename);
  return $tag;
}

/**
 * Sets the last processed tag, so that we do not extract
 * the same strings all the time, unless the code has changed.
 */
function civiextensions_set_last_processed_tag($shortname, $tag) {
  $data_dir = CIVIEXTENSIONS_METADATA_DIR;
  $filename = "$data_dir/{$shortname}.tag";

  if (! $handle = fopen($filename, 'w')) {
    echo "Cannot open file ($filename)";
    return;
  }

  if (fwrite($handle, $tag) === FALSE) {
    echo "Cannot write to file ($filename)";
    return;
  }

  fclose($handle);
}

238
main();