Move the directory queue into the database as well, otherwise if you

have too many directories it blows out the task queue and the whole
thing falls over.  Fixes #1547.

4 files changed, 93 insertions, 60 deletions

diff --git a/modules/server_add/controllers/server_add.php b/modules/server_add/controllers/server_add.php
index 757481d0..15b92cc0 100644
--- a/modules/server_add/controllers/server_add.php
+++ b/modules/server_add/controllers/server_add.php
@@ -24,6 +24,12 @@ class Server_Add_Controller extends Admin_Controller {
       $files[] = $path;
     }
 
+    // Clean leftover task rows.  There really should be support for this in the task framework
+    db::build()
+      ->where("task_id", "NOT IN", db::build()->select("id")->from("tasks"))
+      ->delete("server_add_entries")
+      ->execute();
+
     $item = ORM::factory("item", $id);
     $view = new View("server_add_tree_dialog.html");
     $view->item = $item;
@@ -79,17 +85,22 @@ class Server_Add_Controller extends Admin_Controller {
     access::verify_csrf();
     $item = ORM::factory("item", Input::instance()->get("item_id"));
 
-    foreach (Input::instance()->post("paths") as $path) {
-      if (server_add::is_valid_path($path)) {
-        $paths[] = array($path, null);
-      }
-    }
-
     $task_def = Task_Definition::factory()
       ->callback("Server_Add_Controller::add")
       ->description(t("Add photos or movies from the local server"))
       ->name(t("Add from server"));
-    $task = task::create($task_def, array("item_id" => $item->id, "queue" => $paths));
+    $task = task::create($task_def, array("item_id" => $item->id));
+
+    foreach (Input::instance()->post("paths") as $path) {
+      if (server_add::is_valid_path($path)) {
+        $entry = ORM::factory("server_add_entry");
+        $entry->path = $path;
+        $entry->is_directory = 1;
+        $entry->parent_id = null;
+        $entry->task_id = $task->id;
+        $entry->save();
+      }
+    }
 
     json::reply(
       array("result" => "started",
@@ -118,7 +129,7 @@ class Server_Add_Controller extends Admin_Controller {
 
   /**
    * This is the task code that adds photos and albums.  It first examines all the target files
-   * and creates a set of Server_Add_File_Models, then runs through the list of models and adds
+   * and creates a set of Server_Add_Entry_Models, then runs through the list of models and adds
    * them one at a time.
    */
   static function add($task) {
@@ -128,6 +139,7 @@ class Server_Add_Controller extends Admin_Controller {
     switch ($mode) {
     case "init":
       $task->set("mode", "build-file-list");
+      $task->set("dirs_scanned", 0);
       $task->percent_complete = 0;
       $task->status = t("Starting up");
       batch::start();
@@ -136,58 +148,63 @@ class Server_Add_Controller extends Admin_Controller {
     case "build-file-list":  // 0% to 10%
       // We can't fit an arbitrary number of paths in a task, so store them in a separate table.
       // Don't use an iterator here because we can't get enough control over it when we're dealing
-      // with a deep hierarchy and we don't want to go over our time quota.  The queue is in the
-      // form [path, parent_id] where the parent_id refers to another Server_Add_File_Model.  We
-      // have this extra level of abstraction because we don't know its Item_Model id yet.
-      $queue = $task->get("queue");
+      // with a deep hierarchy and we don't want to go over our time quota.
       $paths = unserialize(module::get_var("server_add", "authorized_paths"));
+      $dirs_scanned = $task->get("dirs_scanned");
+      while (microtime(true) - $start < 0.5) {
+        // Process every directory that doesn't yet have a parent id, these are the
+        // paths that we're importing.
+        $entry = ORM::factory("server_add_entry")
+          ->where("task_id", "=", $task->id)
+          ->where("is_directory", "=", 1)
+          ->where("checked", "=", 0)
+          ->order_by("id", "ASC")
+          ->find();
 
-      while ($queue && microtime(true) - $start < 0.5) {
-        list($file, $parent_entry_id) = array_shift($queue);
-        // Ignore the staging directories as directories to be imported.
-        if (empty($paths[$file])) {
-          $entry = ORM::factory("server_add_file");
-          $entry->task_id = $task->id;
-          $entry->file = $file;
-          $entry->parent_id = $parent_entry_id;
-          $entry->save();
-          $entry_id = $entry->id;
-        } else {
-          $entry_id = null;
-        }
+        if ($entry->loaded()) {
+          // Ignore the staging directories as directories to be imported.
+          if (!empty($paths[$entry->path])) {
+            $entry->delete();
+          }
 
-        $file = preg_replace("/(\*|\?|\[)/", "[$1]", $file);
-        foreach (glob("$file/*") as $child) {
-          if (is_dir($child)) {
-            $queue[] = array($child, $entry_id);
-          } else {
-            $ext = strtolower(pathinfo($child, PATHINFO_EXTENSION));
-            if (in_array($ext, array("gif", "jpeg", "jpg", "png", "flv", "mp4", "m4v")) &&
-                filesize($child) > 0) {
-              $child_entry = ORM::factory("server_add_file");
-              $child_entry->task_id = $task->id;
-              $child_entry->file = $child;
-              $child_entry->parent_id = $entry_id;
-              $child_entry->save();
+          $path = preg_replace("/(\*|\?|\[)/", "[$1]", $entry->path);
+          foreach (glob("$path/*") as $child_path) {
+            if (!is_dir($child_path)) {
+              $ext = strtolower(pathinfo($child_path, PATHINFO_EXTENSION));
+              if (!in_array($ext, array("gif", "jpeg", "jpg", "png", "flv", "mp4", "m4v")) ||
+                  !filesize($child_path)) {
+                // Not importable, skip it.
+                continue;
+              }
             }
+
+            $child_entry = ORM::factory("server_add_entry");
+            $child_entry->task_id = $task->id;
+            $child_entry->path = $child_path;
+            $child_entry->parent_id = $entry->id;  // null if the parent was a staging dir
+            $child_entry->is_directory = is_dir($child_path);
+            $child_entry->save();
           }
+
+          // We've processed this entry, mark it as done.
+          $entry->checked = 1;
+          $entry->save();
         }
+        $dirs_scanned++;
       }
 
       // We have no idea how long this can take because we have no idea how deep the tree
       // hierarchy rabbit hole goes.  Leave ourselves room here for 100 iterations and don't go
       // over 10% in percent_complete.
-      $task->set("queue", $queue);
+      $task->set("dirs_scanned", $dirs_scanned);
       $task->percent_complete = min($task->percent_complete + 0.1, 10);
-      $task->status = t2(
-        "Found one file", "Found %count files",
-        ORM::factory("server_add_file")->where("task_id", "=", $task->id)->count_all());
+      $task->status = t2("Scanned one directory", "Scanned %count directories", $dirs_scanned);
 
-      if (!$queue) {
+      if (!$entry->loaded()) {
         $task->set("mode", "add-files");
         $task->set(
           "total_files",
-          ORM::factory("server_add_file")->where("task_id", "=", $task->id)->count_all());
+          ORM::factory("server_add_entry")->where("task_id", "=", $task->id)->count_all());
         $task->percent_complete = 10;
       }
       break;
@@ -199,7 +216,7 @@ class Server_Add_Controller extends Admin_Controller {
       // Ordering by id ensures that we add them in the order that we created the entries, which
       // will create albums first.  Ignore entries which already have an Item_Model attached,
       // they're done.
-      $entries = ORM::factory("server_add_file")
+      $entries = ORM::factory("server_add_entry")
         ->where("task_id", "=", $task->id)
         ->where("item_id", "IS", null)
         ->order_by("id", "ASC")
@@ -218,16 +235,16 @@ class Server_Add_Controller extends Admin_Controller {
 
         // Look up the parent item for this entry.  By now it should exist, but if none was
         // specified, then this belongs as a child of the current item.
-        $parent_entry = ORM::factory("server_add_file", $entry->parent_id);
+        $parent_entry = ORM::factory("server_add_entry", $entry->parent_id);
         if (!$parent_entry->loaded()) {
           $parent = ORM::factory("item", $task->get("item_id"));
         } else {
           $parent = ORM::factory("item", $parent_entry->item_id);
         }
 
-        $name = basename($entry->file);
+        $name = basename($entry->path);
         $title = item::convert_filename_to_title($name);
-        if (is_dir($entry->file)) {
+        if ($entry->is_directory) {
           $album = ORM::factory("item");
           $album->type = "album";
           $album->parent_id = $parent->id;
@@ -243,7 +260,7 @@ class Server_Add_Controller extends Admin_Controller {
               $photo = ORM::factory("item");
               $photo->type = "photo";
               $photo->parent_id = $parent->id;
-              $photo->set_data_file($entry->file);
+              $photo->set_data_file($entry->path);
               $photo->name = $name;
               $photo->title = $title;
               $photo->owner_id = $owner_id;
@@ -253,7 +270,7 @@ class Server_Add_Controller extends Admin_Controller {
               $movie = ORM::factory("item");
               $movie->type = "movie";
               $movie->parent_id = $parent->id;
-              $movie->set_data_file($entry->file);
+              $movie->set_data_file($entry->path);
               $movie->name = $name;
               $movie->title = $title;
               $movie->owner_id = $owner_id;
@@ -264,12 +281,12 @@ class Server_Add_Controller extends Admin_Controller {
               // process.  But just in, case.. set this to a non-null value so that we skip this
               // entry.
               $entry->item_id = 0;
-              $task->log("Skipping unknown file type: $entry->file");
+              $task->log("Skipping unknown file type: {$entry->path}");
             }
           } catch (Exception $e) {
             // This can happen if a photo file is invalid, like a BMP masquerading as a .jpg
             $entry->item_id = 0;
-            $task->log("Skipping invalid file: $entry->file");
+            $task->log("Skipping invalid file: {$entry->file}");
           }
         }
 
@@ -288,10 +305,9 @@ class Server_Add_Controller extends Admin_Controller {
       $task->done = true;
       $task->state = "success";
       $task->percent_complete = 100;
-      db::build()
-        ->delete("server_add_files")
+      ORM::factory("server_add_entry")
         ->where("task_id", "=", $task->id)
-        ->execute();
+        ->delete_all();
       message::info(t2("Successfully added one photo / album",
                        "Successfully added %count photos / albums",
                        $task->get("completed_files")));
diff --git a/modules/server_add/helpers/server_add_installer.php b/modules/server_add/helpers/server_add_installer.php
index 676bc3cf..320e205e 100644
--- a/modules/server_add/helpers/server_add_installer.php
+++ b/modules/server_add/helpers/server_add_installer.php
@@ -20,15 +20,17 @@
 class server_add_installer {
   static function install() {
     $db = Database::instance();
-    $db->query("CREATE TABLE {server_add_files} (
+    $db->query("CREATE TABLE {server_add_entries} (
                   `id` int(9) NOT NULL auto_increment,
-                  `file` varchar(255) NOT NULL,
+                  `checked` boolean default 0,
+                  `is_directory` boolean default 0,
                   `item_id` int(9),
                   `parent_id` int(9),
+                  `path` varchar(255) NOT NULL,
                   `task_id` int(9) NOT NULL,
                   PRIMARY KEY (`id`))
                 DEFAULT CHARSET=utf8;");
-    module::set_version("server_add", 3);
+    module::set_version("server_add", 4);
     server_add::check_config();
   }
 
@@ -49,6 +51,21 @@ class server_add_installer {
       $db->query("ALTER TABLE {server_add_files} ADD COLUMN `parent_id` int(9)");
       module::set_version("server_add", $version = 3);
     }
+
+    if ($version == 3) {
+      $db->query("DROP TABLE {server_add_files}");
+      $db->query("CREATE TABLE {server_add_entries} (
+                    `id` int(9) NOT NULL auto_increment,
+                    `checked` boolean default 0,
+                    `is_directory` boolean default 0,
+                    `item_id` int(9),
+                    `parent_id` int(9),
+                    `path` varchar(255) NOT NULL,
+                    `task_id` int(9) NOT NULL,
+                    PRIMARY KEY (`id`))
+                  DEFAULT CHARSET=utf8;");
+      module::set_version("server_add", $version = 4);
+    }
   }
 
   static function deactivate() {
diff --git a/modules/server_add/models/server_add_file.php b/modules/server_add/models/server_add_entry.php
index 92b9f6de..dfeb05a2 100644
--- a/modules/server_add/models/server_add_file.php
+++ b/modules/server_add/models/server_add_entry.php
@@ -17,5 +17,5 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA  02110-1301, USA.
  */
-class Server_Add_File_Model_Core extends ORM {
+class Server_Add_Entry_Model_Core extends ORM {
 }
diff --git a/modules/server_add/module.info b/modules/server_add/module.info
index 23acab94..87b317b1 100644
--- a/modules/server_add/module.info
+++ b/modules/server_add/module.info
@@ -1,3 +1,3 @@
 name = "Server Add"
 description = "Allows authorized users to load images directly from your web server"
-version = 3
+version = 4