From f9e77c4c14c3a213ed8407495d4f71949a3a2339 Mon Sep 17 00:00:00 2001 From: Bharat Mediratta Date: Sat, 21 Jul 2012 10:38:23 -0700 Subject: Fix the access_cache rebuilding code to not load all missing access caches into the stack, if you're missing enough it'll blow the stack (like, if you truncate the access_caches table). Fixes #1895. --- modules/gallery/helpers/gallery_task.php | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'modules/gallery/helpers') diff --git a/modules/gallery/helpers/gallery_task.php b/modules/gallery/helpers/gallery_task.php index 9a35ce67..51aec014 100644 --- a/modules/gallery/helpers/gallery_task.php +++ b/modules/gallery/helpers/gallery_task.php @@ -562,7 +562,7 @@ class gallery_task_Core { case self::FIX_STATE_START_MISSING_ACCESS_CACHES: $stack = array(); - foreach (self::find_missing_access_caches() as $row) { + foreach (self::find_missing_access_caches_limited(500) as $row) { $stack[] = $row->id; } if ($stack) { @@ -582,11 +582,20 @@ class gallery_task_Core { $task->set("stack", implode(" ", $stack)); $completed++; if (empty($stack)) { - // The new cache rows are there, but they're incorrectly populated so we have to fix - // them. If this turns out to be too slow, we'll have to refactor - // access::recalculate_permissions to allow us to do it in slices. - access::recalculate_album_permissions(item::root()); - $state = self::FIX_STATE_DONE; + // Refill the stack + foreach (self::find_missing_access_caches_limited(500) as $row) { + $stack[] = $row->id; + } + + if (empty($stack)) { + // The new cache rows are there, but they're incorrectly populated so we have to fix + // them. If this turns out to be too slow, we'll have to refactor + // access::recalculate_permissions to allow us to do it in slices. + access::recalculate_album_permissions(item::root()); + $state = self::FIX_STATE_DONE; + } else { + $task->set("stack", implode(" ", $stack)); + } } break; } @@ -633,11 +642,16 @@ class gallery_task_Core { } static function find_missing_access_caches() { + return self::find_missing_access_caches_limited(1 << 16); + } + + static function find_missing_access_caches_limited($limit) { return db::build() ->select("items.id") ->from("items") ->join("access_caches", "items.id", "access_caches.item_id", "left") ->where("access_caches.id", "is", null) + ->limit($limit) ->execute(); } } \ No newline at end of file -- cgit v1.2.3 From 3e99c37c35f00f98d4159edbc86da25dd2794a6e Mon Sep 17 00:00:00 2001 From: Bharat Mediratta Date: Sat, 21 Jul 2012 11:11:16 -0700 Subject: Fix comment typo. --- modules/gallery/helpers/gallery_installer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/gallery/helpers') diff --git a/modules/gallery/helpers/gallery_installer.php b/modules/gallery/helpers/gallery_installer.php index e1e4e7a9..2fb2bfd4 100644 --- a/modules/gallery/helpers/gallery_installer.php +++ b/modules/gallery/helpers/gallery_installer.php @@ -405,7 +405,7 @@ class gallery_installer { // for now because we don't want a lengthy operation here. $db->query("UPDATE {items} SET `slug` = `name`"); - // Flush all path caches becuase we're going to start urlencoding them. + // Flush all path caches because we're going to start urlencoding them. $db->query("UPDATE {items} SET `relative_url_cache` = NULL, `relative_path_cache` = NULL"); module::set_version("gallery", $version = 12); } -- cgit v1.2.3 From a98afdb32ce3f625ad52467e09970b91923a5b98 Mon Sep 17 00:00:00 2001 From: Bharat Mediratta Date: Sat, 21 Jul 2012 11:12:51 -0700 Subject: Refill the path and url caches in the item model at the end of this task. Fixes #1896. --- modules/gallery/helpers/gallery_task.php | 90 +++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 20 deletions(-) (limited to 'modules/gallery/helpers') diff --git a/modules/gallery/helpers/gallery_task.php b/modules/gallery/helpers/gallery_task.php index 51aec014..65a72884 100644 --- a/modules/gallery/helpers/gallery_task.php +++ b/modules/gallery/helpers/gallery_task.php @@ -26,9 +26,11 @@ class gallery_task_Core { const FIX_STATE_RUN_DUPE_SLUGS = 5; const FIX_STATE_START_DUPE_NAMES = 6; const FIX_STATE_RUN_DUPE_NAMES = 7; - const FIX_STATE_START_MISSING_ACCESS_CACHES = 8; - const FIX_STATE_RUN_MISSING_ACCESS_CACHES = 9; - const FIX_STATE_DONE = 10; + const FIX_STATE_START_REBUILD_ITEM_CACHES = 8; + const FIX_STATE_RUN_REBUILD_ITEM_CACHES = 9; + const FIX_STATE_START_MISSING_ACCESS_CACHES = 10; + const FIX_STATE_RUN_MISSING_ACCESS_CACHES = 11; + const FIX_STATE_DONE = 12; static function available_tasks() { $dirty_count = graphics::find_dirty_images_query()->count_records(); @@ -337,10 +339,15 @@ class gallery_task_Core { $total = $task->get("total"); if (empty($total)) { + $item_count = db::build()->count_records("items"); + $total = 0; + // mptt: 2 operations for every item - $total = 2 * db::build()->count_records("items"); + $total += 2 * $item_count; + // album audit (permissions and bogus album covers): 1 operation for every album $total += db::build()->where("type", "=", "album")->count_records("items"); + // one operation for each missing slug, name and access cache foreach (array("find_dupe_slugs", "find_dupe_names", "find_missing_access_caches") as $func) { foreach (self::$func() as $row) { @@ -348,6 +355,9 @@ class gallery_task_Core { } } + // one operation to rebuild path and url caches; + $total += 1 * $item_count; + $task->set("total", $total); $task->set("state", $state = self::FIX_STATE_START_MPTT); $task->set("ptr", 1); @@ -556,36 +566,68 @@ class gallery_task_Core { $completed++; if (empty($stack)) { - $state = self::FIX_STATE_START_MISSING_ACCESS_CACHES; + $state = self::FIX_STATE_START_REBUILD_ITEM_CACHES; } break; - case self::FIX_STATE_START_MISSING_ACCESS_CACHES: + case self::FIX_STATE_START_REBUILD_ITEM_CACHES: $stack = array(); - foreach (self::find_missing_access_caches_limited(500) as $row) { + foreach (self::find_empty_item_caches(500) as $row) { $stack[] = $row->id; } - if ($stack) { + $task->set("stack", implode(" ", $stack)); + $state = self::FIX_STATE_RUN_REBUILD_ITEM_CACHES; + break; + + case self::FIX_STATE_RUN_REBUILD_ITEM_CACHES: + $stack = explode(" ", $task->get("stack")); + if (!empty($stack)) { + $id = array_pop($stack); + $item = ORM::factory("item", $id); + $item->relative_path(); // this rebuilds the cache and saves the item as a side-effect $task->set("stack", implode(" ", $stack)); - $state = self::FIX_STATE_RUN_MISSING_ACCESS_CACHES; - } else { - $state = self::FIX_STATE_DONE; + $completed++; + } + + if (empty($stack)) { + // Try refilling the stack + foreach (self::find_empty_item_caches(500) as $row) { + $stack[] = $row->id; + } + $task->set("stack", implode(" ", $stack)); + + if (empty($stack)) { + $state = self::FIX_STATE_START_MISSING_ACCESS_CACHES; + } } break; + case self::FIX_STATE_START_MISSING_ACCESS_CACHES: + $stack = array(); + foreach (self::find_missing_access_caches_limited(500) as $row) { + $stack[] = $row->id; + } + $task->set("stack", implode(" ", $stack)); + $state = self::FIX_STATE_RUN_MISSING_ACCESS_CACHES; + break; + case self::FIX_STATE_RUN_MISSING_ACCESS_CACHES: $stack = explode(" ", $task->get("stack")); - $id = array_pop($stack); - $access_cache = ORM::factory("access_cache"); - $access_cache->item_id = $id; - $access_cache->save(); - $task->set("stack", implode(" ", $stack)); - $completed++; + if (!empty($stack)) { + $id = array_pop($stack); + $access_cache = ORM::factory("access_cache"); + $access_cache->item_id = $id; + $access_cache->save(); + $task->set("stack", implode(" ", $stack)); + $completed++; + } + if (empty($stack)) { - // Refill the stack + // Try refilling the stack foreach (self::find_missing_access_caches_limited(500) as $row) { $stack[] = $row->id; } + $task->set("stack", implode(" ", $stack)); if (empty($stack)) { // The new cache rows are there, but they're incorrectly populated so we have to fix @@ -593,8 +635,6 @@ class gallery_task_Core { // access::recalculate_permissions to allow us to do it in slices. access::recalculate_album_permissions(item::root()); $state = self::FIX_STATE_DONE; - } else { - $task->set("stack", implode(" ", $stack)); } } break; @@ -641,6 +681,16 @@ class gallery_task_Core { ->execute(); } + static function find_empty_item_caches($limit) { + return db::build() + ->select("items.id") + ->from("items") + ->where("relative_path_cache", "is", null) + ->or_where("relative_url_cache", "is", null) + ->limit($limit) + ->execute(); + } + static function find_missing_access_caches() { return self::find_missing_access_caches_limited(1 << 16); } -- cgit v1.2.3 From 467a032f8e4927590a9860adbf3f9a25dd599657 Mon Sep 17 00:00:00 2001 From: Bharat Mediratta Date: Sat, 21 Jul 2012 15:11:27 -0700 Subject: Rename any files that have two dots in them to a legal name. This fixes the bug where we made those files invalid in 3.0.4 without providing a clean upgrade path. Bump gallery module to 50. Fixes --- installer/install.sql | 2 +- modules/gallery/helpers/gallery_installer.php | 23 ++++++++++++++++++++++- modules/gallery/module.info | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) (limited to 'modules/gallery/helpers') diff --git a/installer/install.sql b/installer/install.sql index 2ba168d2..2353a75a 100644 --- a/installer/install.sql +++ b/installer/install.sql @@ -244,7 +244,7 @@ CREATE TABLE {modules} ( KEY `weight` (`weight`) ) AUTO_INCREMENT=10 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO {modules} VALUES (1,1,'gallery',49,1); +INSERT INTO {modules} VALUES (1,1,'gallery',50,1); INSERT INTO {modules} VALUES (2,1,'user',4,2); INSERT INTO {modules} VALUES (3,1,'comment',6,3); INSERT INTO {modules} VALUES (4,1,'organize',4,4); diff --git a/modules/gallery/helpers/gallery_installer.php b/modules/gallery/helpers/gallery_installer.php index 2fb2bfd4..e556b49a 100644 --- a/modules/gallery/helpers/gallery_installer.php +++ b/modules/gallery/helpers/gallery_installer.php @@ -313,7 +313,7 @@ class gallery_installer { module::set_var("gallery", "extra_binary_paths", "/usr/local/bin:/opt/local/bin:/opt/bin"); module::set_var("gallery", "timezone", null); - module::set_version("gallery", 49); + module::set_version("gallery", 50); } static function upgrade($version) { @@ -692,6 +692,27 @@ class gallery_installer { module::set_var("gallery", "timezone", null); module::set_version("gallery", $version = 49); } + + if ($version == 49) { + // In v49 we changed the Item_Model validation code to disallow files with two dots in them, + // but we didn't rename any files which fail to validate, so as soon as you do anything to + // change those files (eg. as a side effect of getting the url or file path) it fails to + // validate. Fix those here. This might be slow, but if it times out it can just pick up + // where it left off. + foreach (db::build() + ->from("items") + ->select("id") + ->where("type", "<>", "album") + ->where(db::expr("`name` REGEXP '\\\\..*\\\\.'"), "=", 1) + ->order_by("id", "asc") + ->execute() as $row) { + set_time_limit(30); + $item = ORM::factory("item", $row->id); + $item->name = legal_file::smash_extensions($item->name); + $item->save(); + } + module::set_version("gallery", $version = 50); + } } static function uninstall() { diff --git a/modules/gallery/module.info b/modules/gallery/module.info index 42345531..a905a241 100644 --- a/modules/gallery/module.info +++ b/modules/gallery/module.info @@ -1,6 +1,6 @@ name = "Gallery 3" description = "Gallery core application" -version = 49 +version = 50 author_name = "Gallery Team" author_url = "http://codex.gallery2.org/Gallery:Team" info_url = "http://codex.gallery2.org/Gallery3:Modules:gallery" -- cgit v1.2.3