Completely task-ify the localization scanning code and restore the

incoming_translation key cache.  The only part of this that is not
broken down is the l10n_client::fetch_updates() part.

3 files changed, 117 insertions, 52 deletions

diff --git a/core/controllers/admin_maintenance.php b/core/controllers/admin_maintenance.php
index 794f2253..478009fb 100644
--- a/core/controllers/admin_maintenance.php
+++ b/core/controllers/admin_maintenance.php
@@ -157,12 +157,18 @@ class Admin_Maintenance_Controller extends Admin_Controller {
         break;
       }
       print json_encode(array("result" => "success",
-                              "task" => $task->as_array(),
+                              "task" => array(
+                                "percent_complete" => $task->percent_complete,
+                                "status" => $task->status,
+                                "done" => $task->done),
                               "location" => url::site("admin/maintenance")));
 
     } else {
       print json_encode(array("result" => "in_progress",
-                              "task" => $task->as_array()));
+                              "task" => array(
+                                "percent_complete" => $task->percent_complete,
+                                "status" => $task->status,
+                                "done" => $task->done)));
     }
   }
 }
diff --git a/core/helpers/core_task.php b/core/helpers/core_task.php
index b9d6253a..416950e7 100644
--- a/core/helpers/core_task.php
+++ b/core/helpers/core_task.php
@@ -20,7 +20,8 @@
 class core_task_Core {
   static function available_tasks() {
     $dirty_count = graphics::find_dirty_images_query()->count();
-    return array(Task_Definition::factory()
+    $tasks = array();
+    $tasks[] = Task_Definition::factory()
                  ->callback("core_task::rebuild_dirty_images")
                  ->name(t("Rebuild Images"))
                  ->description($dirty_count ?
@@ -28,12 +29,15 @@ class core_task_Core {
                                   "You have %count out of date photos",
                                   $dirty_count)
                                : t("All your photos are up to date"))
-                 ->severity($dirty_count ? log::WARNING : log::SUCCESS),
-                 Task_Definition::factory()
+      ->severity($dirty_count ? log::WARNING : log::SUCCESS);
+
+    $tasks[] = Task_Definition::factory()
                  ->callback("core_task::update_l10n")
                  ->name(t("Update translations"))
                  ->description(t("Download new and updated translated strings"))
-                 ->severity(log::SUCCESS));
+      ->severity(log::SUCCESS);
+
+    return $tasks;
   }
 
   /**
@@ -79,12 +83,84 @@ class core_task_Core {
     }
   }
 
-  static function update_l10n($task) {
-    l10n_scanner::update_index();
-    l10n_client::fetch_updates();
-    $task->done = true;
-    $task->state = "success";
-    $task->percent_complete = 100;
-    $task->status = t("Translations installed/updated");
+  static function update_l10n(&$task) {
+    $start = microtime(true);
+    $dirs = $task->get("dirs");
+    $files = $task->get("files");
+    $cache = $task->get("cache", array());
+    $i = 0;
+
+    switch ($task->get("mode", "init")) {
+    case "init":  // 0%
+      $dirs = array("core", "modules", "themes", "installer");
+      $files = array();
+      $task->set("mode", "find_files");
+      $task->status = t("Finding files");
+      break;
+
+    case "find_files":  // 0% - 10%
+      while (($dir = array_pop($dirs)) && microtime(true) - $start < 0.5) {
+        if (basename($dir) == "tests") {
+          continue;
+        }
+
+        foreach (glob(DOCROOT . "$dir/*") as $path) {
+          $relative_path = str_replace(DOCROOT, "", $path);
+          if (is_dir($path)) {
+            $dirs[] = $relative_path;
+          } else {
+            $files[] = $relative_path;
+          }
+        }
+      }
+
+      $task->status = t2("Finding files: found 1 file",
+                         "Finding files: found %count files", count($files));
+
+      if (!$dirs) {
+        $task->set("mode", "scan_files");
+        $task->set("total_files", count($files));
+        $task->status = t("Scanning files");
+        $task->percent_complete = 10;
+      }
+      break;
+
+    case "scan_files": // 10% - 90%
+      while (($file = array_pop($files)) && microtime(true) - $start < 0.5) {
+        $file = DOCROOT . $file;
+        switch (pathinfo($file, PATHINFO_EXTENSION)) {
+        case "php":
+          l10n_scanner::scan_php_file($file, $cache);
+          break;
+
+        case "info":
+          l10n_scanner::scan_info_file($file, $cache);
+          break;
+        }
+      }
+
+      $total_files = $task->get("total_files");
+      $task->status = t2("Scanning files: scanned 1 file",
+                         "Scanning files: scanned %count files", $total_files - count($files));
+
+      $task->percent_complete = 10 + 80 * ($total_files - count($files)) / $total_files;
+      if (empty($files)) {
+        $task->set("mode", "fetch_updates");
+        $task->status = t("Fetching updates");
+        $task->percent_complete = 90;
+      }
+      break;
+
+    case "fetch_updates":  // 90% - 100%
+      l10n_client::fetch_updates();
+      $task->done = true;
+      $task->state = "success";
+      $task->status = t("Translations installed/updated");
+      $task->percent_complete = 100;
+    }
+
+    $task->set("files", $files);
+    $task->set("dirs", $dirs);
+    $task->set("cache", $cache);
   }
 }
\ No newline at end of file
diff --git a/core/helpers/l10n_scanner.php b/core/helpers/l10n_scanner.php
index af2c04cf..fe659d35 100644
--- a/core/helpers/l10n_scanner.php
+++ b/core/helpers/l10n_scanner.php
@@ -24,41 +24,24 @@
 class l10n_scanner_Core {
   // Based on Drupal's potx module, originally written by:
   // G‡bor Hojtsy http://drupal.org/user/4166
+  public static $cache;
 
-  // @todo Report progress via callback
-  static function update_index() {
-    $stack = array(DOCROOT . "core",
-                   DOCROOT . "modules",
-                   DOCROOT . "themes",
-                   DOCROOT . "installer");
-
-    while ($stack) {
-      $path = array_pop($stack);
-      if (basename($path) == "tests") {
-        continue;
-      }
-
-      if (is_dir($path)) {
-        $stack = array_merge($stack, glob("$path/*"));
-      } else {
-        switch (pathinfo($path, PATHINFO_EXTENSION)) {
-        case "php":
-          l10n_scanner::_scan_php_file($path);
-          break;
-
-        case "info":
-          l10n_scanner::_scan_info_file($path);
-          break;
-        }
+  static function process_message($message, &$cache) {
+    if (empty($cache)) {
+      foreach (Database::instance()
+               ->select("key")
+               ->from("incoming_translations")
+               ->where("locale", "root")
+               ->get() as $row) {
+        $cache[$row->key] = true;
       }
-      flush();
     }
-  }
 
-  static function process_message($message) {
-    // @todo this is O(N) queries over the number of messages.  Precache all message keys
-    // in the task context and then do lookups over that to get it down to O(1).
     $key = I18n::get_message_key($message);
+    if (array_key_exists($key, $cache)) {
+      return $cache[$key];
+    }
+
     $entry = ORM::factory("incoming_translation", array("key" => $key));
     if (!$entry->loaded) {
       $entry->key = $key;
@@ -68,7 +51,7 @@ class l10n_scanner_Core {
     }
   }
 
-  private static function _scan_php_file($file) {
+  static function scan_php_file($file, &$cache) {
     $code = file_get_contents($file);
     $raw_tokens = token_get_all($code);
     unset($code);
@@ -91,24 +74,24 @@ class l10n_scanner_Core {
     unset($raw_tokens);
 
     if (!empty($func_token_list["t"])) {
-      l10n_scanner::_parse_t_calls($tokens, $func_token_list["t"]);
+      l10n_scanner::_parse_t_calls($tokens, $func_token_list["t"], $cache);
     }
     if (!empty($func_token_list["t2"])) {
-      l10n_scanner::_parse_plural_calls($tokens, $func_token_list["t2"]);
+      l10n_scanner::_parse_plural_calls($tokens, $func_token_list["t2"], $cache);
     }
   }
 
-  private static function _scan_info_file($file) {
+  static function scan_info_file($file, &$cache) {
     $code = file_get_contents($file);
     if (preg_match("#name\s*?=\s*(.*?)\ndescription\s*?=\s*(.*)\n#", $code, $matches)) {
       unset($matches[0]);
       foreach ($matches as $string) {
-        l10n_scanner::process_message($string);
+        l10n_scanner::process_message($string, $cache);
       }
     }
   }
 
-  private static function _parse_t_calls(&$tokens, &$call_list) {
+  private static function _parse_t_calls(&$tokens, &$call_list, &$cache) {
     foreach ($call_list as $index) {
       $function_name = $tokens[$index++];
       $parens = $tokens[$index++];
@@ -119,7 +102,7 @@ class l10n_scanner_Core {
         if (in_array($next_token, array(")", ","))
             && (is_array($first_param) && ($first_param[0] == T_CONSTANT_ENCAPSED_STRING))) {
           $message = self::_escape_quoted_string($first_param[1]);
-          l10n_scanner::process_message($message);
+          l10n_scanner::process_message($message, $cache);
         } else {
           // t() found, but inside is something which is not a string literal.
           // @todo Call status callback with error filename/line.
@@ -128,7 +111,7 @@ class l10n_scanner_Core {
     }
   }
 
-  private static function _parse_plural_calls(&$tokens, &$call_list) {
+  private static function _parse_plural_calls(&$tokens, &$call_list, &$cache) {
     foreach ($call_list as $index) {
       $function_name = $tokens[$index++];
       $parens = $tokens[$index++];
@@ -143,7 +126,7 @@ class l10n_scanner_Core {
             && is_array($second_param) && $second_param[0] == T_CONSTANT_ENCAPSED_STRING) {
           $singular = self::_escape_quoted_string($first_param[1]);
           $plural = self::_escape_quoted_string($first_param[1]);
-          l10n_scanner::process_message(array("one" => $singular, "other" => $plural));
+          l10n_scanner::process_message(array("one" => $singular, "other" => $plural), $cache);
         } else {
           // t2() found, but inside is something which is not a string literal.
           // @todo Call status callback with error filename/line.