Change "fetch translations" task to fetch them in batches.

The request limit was in place already, but the client didn't respect it before, leading to unhappy users in case they had lots of 3rd party modules installed, or more than 2-3 locales enabled. This is all taken care of now.

2 files changed, 56 insertions, 22 deletions

diff --git a/modules/gallery/helpers/gallery_task.php b/modules/gallery/helpers/gallery_task.php
index 3a705027..b3b79e06 100644
--- a/modules/gallery/helpers/gallery_task.php
+++ b/modules/gallery/helpers/gallery_task.php
@@ -122,7 +122,7 @@ class gallery_task_Core {
       $start = microtime(true);
       $data = Cache::instance()->get("update_l10n_cache:{$task->id}");
       if ($data) {
-        list($dirs, $files, $cache) = unserialize($data);
+        list($dirs, $files, $cache, $num_fetched) = unserialize($data);
       }
       $i = 0;
 
@@ -130,6 +130,7 @@ class gallery_task_Core {
       case "init":  // 0%
         $dirs = array("gallery", "modules", "themes", "installer");
         $files = $cache = array();
+        $num_fetched = 0;
         $task->set("mode", "find_files");
         $task->status = t("Finding files");
         break;
@@ -161,7 +162,7 @@ class gallery_task_Core {
         }
         break;
 
-      case "scan_files": // 10% - 90%
+      case "scan_files": // 10% - 70%
         while (($file = array_pop($files)) && microtime(true) - $start < 0.5) {
           $file = DOCROOT . $file;
           switch (pathinfo($file, PATHINFO_EXTENSION)) {
@@ -179,25 +180,31 @@ class gallery_task_Core {
         $task->status = t2("Scanning files: scanned 1 file",
                            "Scanning files: scanned %count files", $total_files - count($files));
 
-        $task->percent_complete = 10 + 80 * ($total_files - count($files)) / $total_files;
+        $task->percent_complete = 10 + 60 * ($total_files - count($files)) / $total_files;
         if (empty($files)) {
           $task->set("mode", "fetch_updates");
           $task->status = t("Fetching updates");
-          $task->percent_complete = 90;
+          $task->percent_complete = 70;
         }
         break;
 
-      case "fetch_updates":  // 90% - 100%
-        l10n_client::fetch_updates();
-        $task->done = true;
-        $task->state = "success";
-        $task->status = t("Translations installed/updated");
-        $task->percent_complete = 100;
+      case "fetch_updates":  // 70% - 100%
+        // Send fetch requests in batches until we're done
+        $num_remaining = l10n_client::fetch_updates($num_fetched);
+        if ($num_remaining) {
+          $total = $num_fetched + $num_remaining;
+          $task->percent_complete = 70 + 30 * ((float) $num_fetched / $total);
+        } else {
+          $task->done = true;
+          $task->state = "success";
+          $task->status = t("Translations installed/updated");
+          $task->percent_complete = 100;
+        }
       }
 
-      if ($task->percent_complete < 100) {
+      if (!$task->done) {
         Cache::instance()->set("update_l10n_cache:{$task->id}",
-                               serialize(array($dirs, $files, $cache)));
+                               serialize(array($dirs, $files, $cache, $num_fetched)));
       } else {
         Cache::instance()->delete("update_l10n_cache:{$task->id}");
       }
diff --git a/modules/gallery/helpers/l10n_client.php b/modules/gallery/helpers/l10n_client.php
index fe70933d..086245e8 100644
--- a/modules/gallery/helpers/l10n_client.php
+++ b/modules/gallery/helpers/l10n_client.php
@@ -68,9 +68,15 @@ class l10n_client_Core {
   }
 
   /**
-   * @return an array of messages that will be written to the task log
+   * Fetches translations for l10n messages. Must be called repeatedly
+   * until 0 is returned (which is a countdown indicating progress).
+   *
+   * @param $num_fetched in/out parameter to specify which batch of
+   *     messages to fetch translations for.
+   * @return The number of messages for which we didn't fetch
+   *     translations for.
    */
-  static function fetch_updates() {
+  static function fetch_updates(&$num_fetched) {
     $request->locales = array();
     $request->messages = new stdClass();
 
@@ -79,23 +85,42 @@ class l10n_client_Core {
       $request->locales[] = $locale;
     }
 
-    // @todo Batch requests (max request size)
-    foreach (db::build()
-             ->select("key", "locale", "revision", "translation")
-             ->from("incoming_translations")
-             ->execute() as $row) {
+    // See the server side code for how we arrive at this
+    // number as a good limit for #locales * #messages.
+    $max_messages = 2000 / count($locales);
+    $num_messages = 0;
+    $rows = db::build()
+        ->select("key", "locale", "revision", "translation")
+        ->from("incoming_translations")
+        ->order_by("key")
+        ->limit(1000000)  // ignore, just there to satisfy SQL syntax
+        ->offset($num_fetched)
+        ->execute();
+    $num_remaining = $rows->count();
+    foreach ($rows as $row) {
       if (!isset($request->messages->{$row->key})) {
+        if ($num_messages >= $max_messages) {
+          break;
+        }
         $request->messages->{$row->key} = 1;
+        $num_messages++;
       }
-      if (!empty($row->revision) && !empty($row->translation)) {
+      if (!empty($row->revision) && !empty($row->translation) &&
+          isset($locales[$row->locale])) {
         if (!is_object($request->messages->{$row->key})) {
           $request->messages->{$row->key} = new stdClass();
         }
-        $request->messages->{$row->key}->{$row->locale} = $row->revision;
+        $request->messages->{$row->key}->{$row->locale} = (int) $row->revision;
       }
+      $num_fetched++;
+      $num_remaining--;
     }
     // @todo Include messages from outgoing_translations?
 
+    if (!$num_messages) {
+      return $num_remaining;
+    }
+
     $request_data = json_encode($request);
     $url = self::_server_url() . "?q=translations/fetch";
     list ($response_data, $response_status) = remote::post($url, array("data" => $request_data));
@@ -103,7 +128,7 @@ class l10n_client_Core {
       throw new Exception("@todo TRANSLATIONS_FETCH_REQUEST_FAILED " . $response_status);
     }
     if (empty($response_data)) {
-      return array(t("Translations fetch request resulted in an empty response"));
+      return $num_remaining;
     }
 
     $response = json_decode($response_data);
@@ -150,6 +175,8 @@ class l10n_client_Core {
       $entry->translation = $translation;
       $entry->save();
     }
+
+    return $num_remaining;
   }
 
   static function submit_translations() {