From c338d3138e2c93bfb219a8d32ba9110138fcf9c2 Mon Sep 17 00:00:00 2001 From: Joe7 Date: Thu, 6 Jan 2011 02:31:50 +0100 Subject: Importing Gallery2 users in batches Fixes Ticket #1554 --- modules/g2_import/helpers/g2_import_task.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'modules/g2_import/helpers/g2_import_task.php') diff --git a/modules/g2_import/helpers/g2_import_task.php b/modules/g2_import/helpers/g2_import_task.php index f2a13291..dbfe5f77 100644 --- a/modules/g2_import/helpers/g2_import_task.php +++ b/modules/g2_import/helpers/g2_import_task.php @@ -109,8 +109,8 @@ class g2_import_task_Core { case "users": if (empty($queue)) { - $task->set( - "queue", $queue = array_keys(g2(GalleryCoreApi::fetchUsersForGroup(GROUP_EVERYBODY)))); + $task->set("queue", $queue = g2_import::get_user_ids($task->get("last_id", 0))); + $task->set("last_id", end($queue)); } $log_message = g2_import::import_user($queue); if ($log_message) { -- cgit v1.2.3 From 5bfb7fb96a5dc2e0bc511c998a7c055b83b2915a Mon Sep 17 00:00:00 2001 From: Joe7 Date: Thu, 6 Jan 2011 04:49:50 +0100 Subject: Added batch importer for groups Added missing ordering for comments and tags retrieving queries (One might gets the same output without ordering especially if column is the primary key, but the only way to go for sure it by setting it) --- modules/g2_import/helpers/g2_import.php | 30 +++++++++++++++++++++++++--- modules/g2_import/helpers/g2_import_task.php | 4 ++-- 2 files changed, 29 insertions(+), 5 deletions(-) (limited to 'modules/g2_import/helpers/g2_import_task.php') diff --git a/modules/g2_import/helpers/g2_import.php b/modules/g2_import/helpers/g2_import.php index 5e2a61e8..c3d774ea 100644 --- a/modules/g2_import/helpers/g2_import.php +++ b/modules/g2_import/helpers/g2_import.php @@ -1155,7 +1155,8 @@ class g2_import_Core { "SELECT [GalleryComment::id] " . "FROM [GalleryComment] " . "WHERE [GalleryComment::publishStatus] = 0 " . // 0 == COMMENT_PUBLISH_STATUS_PUBLISHED - "AND [GalleryComment::id] > ?", + "AND [GalleryComment::id] > ?" . + "ORDER BY [GalleryComment::id] ASC", array($min_id), array("limit" => array("count" => 100)))); while ($result = $results->nextResult()) { @@ -1174,7 +1175,8 @@ class g2_import_Core { $ids = array(); $results = g2($gallery->search( "SELECT DISTINCT([TagItemMap::itemId]) FROM [TagItemMap] " . - "WHERE [TagItemMap::itemId] > ?", + "WHERE [TagItemMap::itemId] > ?" . + "ORDER BY [TagItemMap::itemId] ASC", array($min_id), array("limit" => array("count" => 100)))); while ($result = $results->nextResult()) { @@ -1194,7 +1196,29 @@ class g2_import_Core { $results = g2($gallery->search( "SELECT [GalleryUser::id] " . "FROM [GalleryUser] " . - "WHERE [GalleryUser::id] > ?", + "WHERE [GalleryUser::id] > ?" . + "ORDER BY [GalleryUser::id] ASC", + array($min_id), + array("limit" => array("count" => 100)))); + while ($result = $results->nextResult()) { + $ids[] = $result[0]; + } + return $ids; + } + + /** + * Get a set of group ids from Gallery 2 greater than $min_id. We use this to get the + * next chunk of groups to import. + */ + static function get_group_ids($min_id) { + global $gallery; + + $ids = array(); + $results = g2($gallery->search( + "SELECT [GalleryGroup::id] " . + "FROM [GalleryGroup] " . + "WHERE [GalleryGroup::id] > ?" . + "ORDER BY [GalleryGroup::id] ASC", array($min_id), array("limit" => array("count" => 100)))); while ($result = $results->nextResult()) { diff --git a/modules/g2_import/helpers/g2_import_task.php b/modules/g2_import/helpers/g2_import_task.php index dbfe5f77..a90b1fc3 100644 --- a/modules/g2_import/helpers/g2_import_task.php +++ b/modules/g2_import/helpers/g2_import_task.php @@ -96,7 +96,8 @@ class g2_import_task_Core { switch($modes[$mode]) { case "groups": if (empty($queue)) { - $task->set("queue", $queue = array_keys(g2(GalleryCoreApi::fetchGroupNames()))); + $task->set("queue", $queue = g2_import::get_group_ids($task->get("last_id", 0))); + $task->set("last_id", end($queue)); } $log_message = g2_import::import_group($queue); if ($log_message) { @@ -141,7 +142,6 @@ class g2_import_task_Core { $task->set("queue", $queue = g2_import::get_item_ids($task->get("last_id", 0))); $task->set("last_id", end($queue)); } - $log_message = g2_import::import_item($queue); if ($log_message) { $task->log($log_message); -- cgit v1.2.3 From 423daa52d55a5298b461384baedc995eee09a0d1 Mon Sep 17 00:00:00 2001 From: Bharat Mediratta Date: Fri, 21 Jan 2011 23:01:06 -0800 Subject: Update copyright to 2011. --- application/Bootstrap.php | 2 +- application/config/config.php | 2 +- index.php | 2 +- installer/cli.php | 2 +- installer/index.php | 2 +- installer/installer.php | 2 +- installer/web.php | 2 +- modules/akismet/controllers/admin_akismet.php | 2 +- modules/akismet/helpers/akismet.php | 2 +- modules/akismet/helpers/akismet_event.php | 2 +- modules/akismet/helpers/akismet_installer.php | 2 +- modules/akismet/tests/Akismet_Helper_Test.php | 2 +- modules/comment/controllers/admin_comments.php | 2 +- modules/comment/controllers/admin_manage_comments.php | 2 +- modules/comment/controllers/comments.php | 2 +- modules/comment/helpers/comment.php | 2 +- modules/comment/helpers/comment_block.php | 2 +- modules/comment/helpers/comment_event.php | 2 +- modules/comment/helpers/comment_installer.php | 2 +- modules/comment/helpers/comment_rest.php | 2 +- modules/comment/helpers/comment_rss.php | 2 +- modules/comment/helpers/comment_theme.php | 2 +- modules/comment/helpers/comments_rest.php | 2 +- modules/comment/helpers/item_comments_rest.php | 2 +- modules/comment/models/comment.php | 2 +- modules/comment/tests/Comment_Event_Test.php | 2 +- modules/comment/tests/Comment_Helper_Test.php | 2 +- modules/comment/tests/Comment_Model_Test.php | 2 +- modules/digibug/config/digibug.php | 2 +- modules/digibug/controllers/admin_digibug.php | 2 +- modules/digibug/controllers/digibug.php | 2 +- modules/digibug/helpers/digibug_event.php | 2 +- modules/digibug/helpers/digibug_installer.php | 2 +- modules/digibug/helpers/digibug_theme.php | 2 +- modules/digibug/models/digibug_proxy.php | 2 +- modules/digibug/tests/Digibug_Controller_Test.php | 2 +- modules/exif/controllers/exif.php | 2 +- modules/exif/helpers/exif.php | 2 +- modules/exif/helpers/exif_event.php | 2 +- modules/exif/helpers/exif_installer.php | 2 +- modules/exif/helpers/exif_task.php | 2 +- modules/exif/helpers/exif_theme.php | 2 +- modules/exif/models/exif_key.php | 2 +- modules/exif/models/exif_record.php | 2 +- modules/exif/tests/Exif_Test.php | 2 +- modules/g2_import/controllers/admin_g2_import.php | 2 +- modules/g2_import/controllers/g2.php | 2 +- modules/g2_import/helpers/g2_import.php | 2 +- modules/g2_import/helpers/g2_import_event.php | 2 +- modules/g2_import/helpers/g2_import_installer.php | 2 +- modules/g2_import/helpers/g2_import_task.php | 2 +- modules/g2_import/libraries/G2_Import_Exception.php | 2 +- modules/g2_import/models/g2_map.php | 2 +- modules/gallery/config/cache.php | 2 +- modules/gallery/config/cookie.php | 2 +- modules/gallery/config/database.php | 2 +- modules/gallery/config/locale.php | 2 +- modules/gallery/config/log_file.php | 2 +- modules/gallery/config/routes.php | 2 +- modules/gallery/config/session.php | 2 +- modules/gallery/config/upload.php | 2 +- modules/gallery/config/user_agents.php | 2 +- modules/gallery/controllers/admin.php | 2 +- modules/gallery/controllers/admin_advanced_settings.php | 2 +- modules/gallery/controllers/admin_dashboard.php | 2 +- modules/gallery/controllers/admin_graphics.php | 2 +- modules/gallery/controllers/admin_languages.php | 2 +- modules/gallery/controllers/admin_maintenance.php | 2 +- modules/gallery/controllers/admin_modules.php | 2 +- modules/gallery/controllers/admin_sidebar.php | 2 +- modules/gallery/controllers/admin_theme_options.php | 2 +- modules/gallery/controllers/admin_themes.php | 2 +- modules/gallery/controllers/admin_upgrade_checker.php | 2 +- modules/gallery/controllers/albums.php | 2 +- modules/gallery/controllers/combined.php | 2 +- modules/gallery/controllers/file_proxy.php | 2 +- modules/gallery/controllers/items.php | 2 +- modules/gallery/controllers/l10n_client.php | 2 +- modules/gallery/controllers/login.php | 2 +- modules/gallery/controllers/logout.php | 2 +- modules/gallery/controllers/movies.php | 2 +- modules/gallery/controllers/packager.php | 2 +- modules/gallery/controllers/permissions.php | 2 +- modules/gallery/controllers/photos.php | 2 +- modules/gallery/controllers/quick.php | 2 +- modules/gallery/controllers/reauthenticate.php | 2 +- modules/gallery/controllers/upgrader.php | 2 +- modules/gallery/controllers/uploader.php | 2 +- modules/gallery/controllers/user_profile.php | 2 +- modules/gallery/controllers/welcome_message.php | 2 +- modules/gallery/helpers/MY_html.php | 2 +- modules/gallery/helpers/MY_num.php | 2 +- modules/gallery/helpers/MY_remote.php | 2 +- modules/gallery/helpers/MY_url.php | 2 +- modules/gallery/helpers/access.php | 2 +- modules/gallery/helpers/album.php | 2 +- modules/gallery/helpers/auth.php | 2 +- modules/gallery/helpers/batch.php | 2 +- modules/gallery/helpers/block_manager.php | 2 +- modules/gallery/helpers/data_rest.php | 2 +- modules/gallery/helpers/dir.php | 2 +- modules/gallery/helpers/gallery.php | 2 +- modules/gallery/helpers/gallery_block.php | 2 +- modules/gallery/helpers/gallery_error.php | 2 +- modules/gallery/helpers/gallery_event.php | 2 +- modules/gallery/helpers/gallery_graphics.php | 2 +- modules/gallery/helpers/gallery_installer.php | 2 +- modules/gallery/helpers/gallery_rss.php | 2 +- modules/gallery/helpers/gallery_task.php | 2 +- modules/gallery/helpers/gallery_theme.php | 2 +- modules/gallery/helpers/graphics.php | 2 +- modules/gallery/helpers/identity.php | 2 +- modules/gallery/helpers/item.php | 2 +- modules/gallery/helpers/item_rest.php | 2 +- modules/gallery/helpers/items_rest.php | 2 +- modules/gallery/helpers/json.php | 2 +- modules/gallery/helpers/l10n_client.php | 2 +- modules/gallery/helpers/l10n_scanner.php | 2 +- modules/gallery/helpers/locales.php | 2 +- modules/gallery/helpers/log.php | 2 +- modules/gallery/helpers/message.php | 2 +- modules/gallery/helpers/model_cache.php | 2 +- modules/gallery/helpers/module.php | 2 +- modules/gallery/helpers/movie.php | 2 +- modules/gallery/helpers/photo.php | 2 +- modules/gallery/helpers/random.php | 2 +- modules/gallery/helpers/site_status.php | 2 +- modules/gallery/helpers/system.php | 2 +- modules/gallery/helpers/task.php | 2 +- modules/gallery/helpers/theme.php | 2 +- modules/gallery/helpers/tree_rest.php | 2 +- modules/gallery/helpers/upgrade_checker.php | 2 +- modules/gallery/helpers/user_profile.php | 2 +- modules/gallery/helpers/xml.php | 2 +- modules/gallery/hooks/init_gallery.php | 2 +- modules/gallery/libraries/Admin_View.php | 2 +- modules/gallery/libraries/Block.php | 2 +- modules/gallery/libraries/Form_Script.php | 2 +- modules/gallery/libraries/Form_Uploadify.php | 2 +- modules/gallery/libraries/Form_Uploadify_buttons.php | 2 +- modules/gallery/libraries/Gallery_I18n.php | 2 +- modules/gallery/libraries/Gallery_View.php | 2 +- modules/gallery/libraries/IdentityProvider.php | 2 +- modules/gallery/libraries/InPlaceEdit.php | 2 +- modules/gallery/libraries/MY_Database.php | 2 +- modules/gallery/libraries/MY_Forge.php | 2 +- modules/gallery/libraries/MY_Input.php | 2 +- modules/gallery/libraries/MY_Kohana_Exception.php | 2 +- modules/gallery/libraries/MY_ORM.php | 2 +- modules/gallery/libraries/MY_Pagination.php | 2 +- modules/gallery/libraries/MY_View.php | 2 +- modules/gallery/libraries/Menu.php | 2 +- modules/gallery/libraries/ORM_MPTT.php | 2 +- modules/gallery/libraries/SafeString.php | 2 +- modules/gallery/libraries/Sendmail.php | 2 +- modules/gallery/libraries/Task_Definition.php | 2 +- modules/gallery/libraries/Theme_View.php | 2 +- modules/gallery/libraries/drivers/Cache/Database.php | 2 +- modules/gallery/libraries/drivers/IdentityProvider.php | 2 +- modules/gallery/models/access_cache.php | 2 +- modules/gallery/models/access_intent.php | 2 +- modules/gallery/models/cache.php | 2 +- modules/gallery/models/failed_auth.php | 2 +- modules/gallery/models/graphics_rule.php | 2 +- modules/gallery/models/incoming_translation.php | 2 +- modules/gallery/models/item.php | 2 +- modules/gallery/models/log.php | 2 +- modules/gallery/models/message.php | 2 +- modules/gallery/models/module.php | 2 +- modules/gallery/models/outgoing_translation.php | 2 +- modules/gallery/models/permission.php | 2 +- modules/gallery/models/task.php | 2 +- modules/gallery/models/theme.php | 2 +- modules/gallery/models/var.php | 2 +- modules/gallery/tests/Access_Helper_Test.php | 2 +- modules/gallery/tests/Albums_Controller_Test.php | 2 +- modules/gallery/tests/Cache_Test.php | 2 +- modules/gallery/tests/Controller_Auth_Test.php | 2 +- modules/gallery/tests/Database_Test.php | 2 +- modules/gallery/tests/Dir_Helper_Test.php | 2 +- modules/gallery/tests/DrawForm_Test.php | 2 +- modules/gallery/tests/File_Structure_Test.php | 4 ++-- modules/gallery/tests/Gallery_Filters.php | 2 +- modules/gallery/tests/Gallery_I18n_Test.php | 2 +- modules/gallery/tests/Gallery_Installer_Test.php | 2 +- modules/gallery/tests/Html_Helper_Test.php | 2 +- modules/gallery/tests/Input_Library_Test.php | 2 +- modules/gallery/tests/Item_Helper_Test.php | 2 +- modules/gallery/tests/Item_Model_Test.php | 2 +- modules/gallery/tests/Item_Rest_Helper_Test.php | 2 +- modules/gallery/tests/Items_Rest_Helper_Test.php | 2 +- modules/gallery/tests/Kohana_Exception_Test.php | 2 +- modules/gallery/tests/Locales_Helper_Test.php | 2 +- modules/gallery/tests/Menu_Test.php | 2 +- modules/gallery/tests/ORM_MPTT_Test.php | 2 +- modules/gallery/tests/Photos_Controller_Test.php | 2 +- modules/gallery/tests/SafeString_Test.php | 2 +- modules/gallery/tests/Sendmail_Test.php | 2 +- modules/gallery/tests/Url_Security_Test.php | 2 +- modules/gallery/tests/Var_Test.php | 2 +- modules/gallery/tests/Xss_Security_Test.php | 2 +- modules/gallery_unit_test/controllers/gallery_unit_test.php | 2 +- modules/gallery_unit_test/helpers/MY_request.php | 2 +- modules/gallery_unit_test/helpers/test.php | 2 +- modules/gallery_unit_test/libraries/Gallery_Unit_Test_Case.php | 2 +- modules/image_block/helpers/image_block_block.php | 2 +- modules/image_block/helpers/image_block_installer.php | 2 +- modules/info/helpers/info_block.php | 2 +- modules/info/helpers/info_installer.php | 2 +- modules/info/helpers/info_theme.php | 2 +- modules/kohana23_compat/config/pagination.php | 2 +- modules/kohana23_compat/libraries/MY_Database_Builder.php | 2 +- modules/kohana23_compat/libraries/Pagination.php | 2 +- modules/notification/controllers/notification.php | 2 +- modules/notification/helpers/notification.php | 2 +- modules/notification/helpers/notification_event.php | 2 +- modules/notification/helpers/notification_installer.php | 2 +- modules/notification/models/pending_notification.php | 2 +- modules/notification/models/subscription.php | 2 +- modules/organize/controllers/organize.php | 2 +- modules/organize/helpers/organize_event.php | 2 +- modules/organize/helpers/organize_installer.php | 2 +- modules/recaptcha/controllers/admin_recaptcha.php | 2 +- modules/recaptcha/helpers/recaptcha.php | 2 +- modules/recaptcha/helpers/recaptcha_event.php | 2 +- modules/recaptcha/helpers/recaptcha_installer.php | 2 +- modules/recaptcha/helpers/recaptcha_theme.php | 2 +- modules/recaptcha/libraries/Form_Recaptcha.php | 2 +- modules/rest/controllers/rest.php | 2 +- modules/rest/helpers/registry_rest.php | 2 +- modules/rest/helpers/rest.php | 2 +- modules/rest/helpers/rest_event.php | 2 +- modules/rest/helpers/rest_installer.php | 2 +- modules/rest/libraries/Rest_Exception.php | 2 +- modules/rest/models/user_access_key.php | 2 +- modules/rest/tests/Rest_Controller_Test.php | 2 +- modules/rss/controllers/rss.php | 2 +- modules/rss/helpers/rss.php | 2 +- modules/rss/helpers/rss_block.php | 2 +- modules/search/controllers/search.php | 2 +- modules/search/helpers/search.php | 2 +- modules/search/helpers/search_event.php | 2 +- modules/search/helpers/search_installer.php | 2 +- modules/search/helpers/search_task.php | 2 +- modules/search/helpers/search_theme.php | 2 +- modules/search/models/search_record.php | 2 +- modules/server_add/controllers/admin_server_add.php | 2 +- modules/server_add/controllers/server_add.php | 2 +- modules/server_add/helpers/server_add.php | 2 +- modules/server_add/helpers/server_add_event.php | 2 +- modules/server_add/helpers/server_add_installer.php | 2 +- modules/server_add/helpers/server_add_theme.php | 2 +- modules/server_add/models/server_add_entry.php | 2 +- modules/slideshow/helpers/slideshow_event.php | 2 +- modules/slideshow/helpers/slideshow_installer.php | 2 +- modules/slideshow/helpers/slideshow_theme.php | 2 +- modules/tag/controllers/admin_tags.php | 2 +- modules/tag/controllers/tag.php | 2 +- modules/tag/controllers/tags.php | 2 +- modules/tag/helpers/item_tags_rest.php | 2 +- modules/tag/helpers/tag.php | 2 +- modules/tag/helpers/tag_block.php | 2 +- modules/tag/helpers/tag_event.php | 2 +- modules/tag/helpers/tag_installer.php | 2 +- modules/tag/helpers/tag_item_rest.php | 2 +- modules/tag/helpers/tag_items_rest.php | 2 +- modules/tag/helpers/tag_rest.php | 2 +- modules/tag/helpers/tag_rss.php | 2 +- modules/tag/helpers/tag_task.php | 2 +- modules/tag/helpers/tag_theme.php | 2 +- modules/tag/helpers/tags_rest.php | 2 +- modules/tag/models/tag.php | 2 +- modules/tag/tests/Tag_Item_Rest_Helper_Test.php | 2 +- modules/tag/tests/Tag_Rest_Helper_Test.php | 2 +- modules/tag/tests/Tag_Test.php | 2 +- modules/tag/tests/Tags_Rest_Helper_Test.php | 2 +- modules/user/config/identity.php | 2 +- modules/user/controllers/admin_users.php | 2 +- modules/user/controllers/password.php | 2 +- modules/user/controllers/users.php | 2 +- modules/user/helpers/group.php | 2 +- modules/user/helpers/user.php | 2 +- modules/user/helpers/user_event.php | 2 +- modules/user/helpers/user_installer.php | 2 +- modules/user/helpers/user_theme.php | 2 +- modules/user/libraries/drivers/IdentityProvider/Gallery.php | 2 +- modules/user/models/group.php | 2 +- modules/user/models/user.php | 2 +- modules/user/tests/No_Direct_ORM_Access_Test.php | 2 +- modules/user/tests/User_Groups_Test.php | 2 +- modules/user/tests/User_Installer_Test.php | 2 +- modules/watermark/controllers/admin_watermarks.php | 2 +- modules/watermark/helpers/watermark.php | 2 +- modules/watermark/helpers/watermark_event.php | 2 +- modules/watermark/helpers/watermark_installer.php | 2 +- 295 files changed, 296 insertions(+), 296 deletions(-) (limited to 'modules/g2_import/helpers/g2_import_task.php') diff --git a/application/Bootstrap.php b/application/Bootstrap.php index fbd83ce1..ff021fd5 100644 --- a/application/Bootstrap.php +++ b/application/Bootstrap.php @@ -1,7 +1,7 @@