From 5d1bd516272396b4c89be74c9d32a7e247c5602d Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 25 Oct 2012 11:36:38 -0700 Subject: [PATCH] Add a script to migrate files between storage engines Summary: Quora requested this (moving to S3) but it's also clearly a good idea. Test Plan: Ran with various valid/invalid options to test options. Error/sanity checking seemed OK. Migrated individual local files. Migrated all my local files back and forth between engines several times. Uploaded some new files. Reviewers: btrahan, vrana Reviewed By: vrana CC: aran Maniphest Tasks: T1950 Differential Revision: https://secure.phabricator.com/D3808 --- bin/files | 1 + scripts/files/manage_files.php | 39 +++++ src/__phutil_library_map__.php | 6 + ...bricatorFilesManagementEnginesWorkflow.php | 46 ++++++ ...bricatorFilesManagementMigrateWorkflow.php | 153 ++++++++++++++++++ .../PhabricatorFilesManagementWorkflow.php | 26 +++ .../files/storage/PhabricatorFile.php | 118 ++++++++++---- .../configuring_file_storage.diviner | 16 ++ 8 files changed, 375 insertions(+), 30 deletions(-) create mode 120000 bin/files create mode 100755 scripts/files/manage_files.php create mode 100644 src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php create mode 100644 src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php create mode 100644 src/applications/files/management/PhabricatorFilesManagementWorkflow.php diff --git a/bin/files b/bin/files new file mode 120000 index 0000000000..bb0c48da5b --- /dev/null +++ b/bin/files @@ -0,0 +1 @@ +../scripts/files/manage_files.php \ No newline at end of file diff --git a/scripts/files/manage_files.php b/scripts/files/manage_files.php new file mode 100755 index 0000000000..3a9ec7707e --- /dev/null +++ b/scripts/files/manage_files.php @@ -0,0 +1,39 @@ +#!/usr/bin/env php +setTagline('manage files'); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = array( + new PhabricatorFilesManagementEnginesWorkflow(), + new PhabricatorFilesManagementMigrateWorkflow(), + new PhutilHelpArgumentWorkflow(), +); + +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index e988a43cfc..a0414b56bc 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -755,6 +755,9 @@ phutil_register_library_map(array( 'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php', 'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php', 'PhabricatorFileUploadView' => 'applications/files/view/PhabricatorFileUploadView.php', + 'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php', + 'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php', + 'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php', 'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php', 'PhabricatorFlagColor' => 'applications/flag/constants/PhabricatorFlagColor.php', 'PhabricatorFlagConstants' => 'applications/flag/constants/PhabricatorFlagConstants.php', @@ -1949,6 +1952,9 @@ phutil_register_library_map(array( 'PhabricatorFileUploadController' => 'PhabricatorFileController', 'PhabricatorFileUploadException' => 'Exception', 'PhabricatorFileUploadView' => 'AphrontView', + 'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow', + 'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow', + 'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow', 'PhabricatorFlag' => 'PhabricatorFlagDAO', 'PhabricatorFlagColor' => 'PhabricatorFlagConstants', 'PhabricatorFlagController' => 'PhabricatorController', diff --git a/src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php new file mode 100644 index 0000000000..02929377b4 --- /dev/null +++ b/src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php @@ -0,0 +1,46 @@ +setName('engines') + ->setSynopsis('List available storage engines.') + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $engines = PhabricatorFile::buildAllEngines(); + if (!$engines) { + throw new Exception("No storage engines are available."); + } + + foreach ($engines as $engine) { + $console->writeOut( + "%s\n", + $engine->getEngineIdentifier()); + } + + return 0; + } + +} diff --git a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php new file mode 100644 index 0000000000..8972ac275e --- /dev/null +++ b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php @@ -0,0 +1,153 @@ +setName('migrate') + ->setSynopsis('Migrate files between storage engines.') + ->setArguments( + array( + array( + 'name' => 'engine', + 'param' => 'storage_engine', + 'help' => 'Migrate to the named storage engine.', + ), + array( + 'name' => 'dry-run', + 'help' => 'Show what would be migrated.', + ), + array( + 'name' => 'all', + 'help' => 'Migrate all files.', + ), + array( + 'name' => 'names', + 'wildcard' => true, + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $engine_id = $args->getArg('engine'); + if (!$engine_id) { + throw new PhutilArgumentUsageException( + "Specify an engine to migrate to with `--engine`. ". + "Use `files engines` to get a list of engines."); + } + + $engine = PhabricatorFile::buildEngine($engine_id); + + if ($args->getArg('all')) { + if ($args->getArg('names')) { + throw new PhutilArgumentUsageException( + "Specify either a list of files or `--all`, but not both."); + } + $iterator = new LiskMigrationIterator(new PhabricatorFile()); + } else if ($args->getArg('names')) { + $iterator = array(); + + foreach ($args->getArg('names') as $name) { + $name = trim($name); + + $id = preg_replace('/^F/i', '', $name); + if (ctype_digit($id)) { + $file = id(new PhabricatorFile())->loadOneWhere( + 'id = %d', + $id); + if (!$file) { + throw new PhutilArgumentUsageException( + "No file exists with id '{$name}'."); + } + } else { + $file = id(new PhabricatorFile())->loadOneWhere( + 'phid = %d', + $name); + if (!$file) { + throw new PhutilArgumentUsageException( + "No file exists with PHID '{$name}'."); + } + } + $iterator[] = $file; + } + } else { + throw new PhutilArgumentUsageException( + "Either specify a list of files to migrate, or use `--all` ". + "to migrate all files."); + } + + $is_dry_run = $args->getArg('dry-run'); + + $failed = array(); + + foreach ($iterator as $file) { + $fid = 'F'.$file->getID(); + + if ($file->getStorageEngine() == $engine_id) { + $console->writeOut( + "%s: Already stored on '%s'\n", + $fid, + $engine_id); + continue; + } + + if ($is_dry_run) { + $console->writeOut( + "%s: Would migrate from '%s' to '%s' (dry run)\n", + $fid, + $file->getStorageEngine(), + $engine_id); + continue; + } + + $console->writeOut( + "%s: Migrating from '%s' to '%s'...", + $fid, + $file->getStorageEngine(), + $engine_id); + + try { + $file->migrateToEngine($engine); + $console->writeOut("done.\n"); + } catch (Exception $ex) { + $console->writeOut("failed!\n"); + $console->writeErr("%s\n", (string)$ex); + $failed[] = $file; + } + } + + if ($failed) { + $console->writeOut("**Failures!**\n"); + $ids = array(); + foreach ($failed as $file) { + $ids[] = 'F'.$file->getID(); + } + $console->writeOut("%s\n", implode(', ', $ids)); + + return 1; + } else { + $console->writeOut("**Success!**\n"); + return 0; + } + } + +} diff --git a/src/applications/files/management/PhabricatorFilesManagementWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementWorkflow.php new file mode 100644 index 0000000000..968f55208f --- /dev/null +++ b/src/applications/files/management/PhabricatorFilesManagementWorkflow.php @@ -0,0 +1,26 @@ +writeFile($data, $params); - if (!$data_handle || strlen($data_handle) > 255) { - // This indicates an improperly implemented storage engine. - throw new PhabricatorFileStorageConfigurationException( - "Storage engine '{$engine_class}' executed writeFile() but did ". - "not return a valid handle ('{$data_handle}') to the data: it ". - "must be nonempty and no longer than 255 characters."); - } - - $engine_identifier = $engine->getEngineIdentifier(); - if (!$engine_identifier || strlen($engine_identifier) > 32) { - throw new PhabricatorFileStorageConfigurationException( - "Storage engine '{$engine_class}' returned an improper engine ". - "identifier '{$engine_identifier}': it must be nonempty ". - "and no longer than 32 characters."); - } + list($engine_identifier, $data_handle) = $file->writeToEngine( + $engine, + $data, + $params); // We stored the file somewhere so stop trying to write it to other // places. break; - } catch (PhabricatorFileStorageConfigurationException $ex) { // If an engine is outright misconfigured (or misimplemented), raise // that immediately since it probably needs attention. throw $ex; - } catch (Exception $ex) { - // If an engine doesn't work, keep trying all the other valid engines - // in case something else works. phlog($ex); + // If an engine doesn't work, keep trying all the other valid engines + // in case something else works. $exceptions[$engine_class] = $ex; } } @@ -204,7 +191,6 @@ final class PhabricatorFile extends PhabricatorFileDAO { // (always the case with newFromFileDownload()), store a '' $authorPHID = idx($params, 'authorPHID'); - $file = new PhabricatorFile(); $file->setName($file_name); $file->setByteSize(strlen($data)); $file->setAuthorPHID($authorPHID); @@ -230,6 +216,63 @@ final class PhabricatorFile extends PhabricatorFileDAO { return $file; } + public function migrateToEngine(PhabricatorFileStorageEngine $engine) { + if (!$this->getID() || !$this->getStorageHandle()) { + throw new Exception( + "You can not migrate a file which hasn't yet been saved."); + } + + $data = $this->loadFileData(); + $params = array( + 'name' => $this->getName(), + ); + + list($new_identifier, $new_handle) = $this->writeToEngine( + $engine, + $data, + $params); + + $old_engine = $this->instantiateStorageEngine(); + $old_handle = $this->getStorageHandle(); + + $this->setStorageEngine($new_identifier); + $this->setStorageHandle($new_handle); + $this->save(); + + $old_engine->deleteFile($old_handle); + + return $this; + } + + private function writeToEngine( + PhabricatorFileStorageEngine $engine, + $data, + array $params) { + + $engine_class = get_class($engine); + + $data_handle = $engine->writeFile($data, $params); + + if (!$data_handle || strlen($data_handle) > 255) { + // This indicates an improperly implemented storage engine. + throw new PhabricatorFileStorageConfigurationException( + "Storage engine '{$engine_class}' executed writeFile() but did ". + "not return a valid handle ('{$data_handle}') to the data: it ". + "must be nonempty and no longer than 255 characters."); + } + + $engine_identifier = $engine->getEngineIdentifier(); + if (!$engine_identifier || strlen($engine_identifier) > 32) { + throw new PhabricatorFileStorageConfigurationException( + "Storage engine '{$engine_class}' returned an improper engine ". + "identifier '{$engine_identifier}': it must be nonempty ". + "and no longer than 32 characters."); + } + + return array($engine_identifier, $data_handle); + } + + public static function newFromFileDownload($uri, $name) { $uri = new PhutilURI($uri); @@ -402,19 +445,34 @@ final class PhabricatorFile extends PhabricatorFileDAO { } protected function instantiateStorageEngine() { - $engines = id(new PhutilSymbolLoader()) - ->setType('class') - ->setAncestorClass('PhabricatorFileStorageEngine') - ->selectAndLoadSymbols(); + return self::buildEngine($this->getStorageEngine()); + } - foreach ($engines as $engine_class) { - $engine = newv($engine_class['name'], array()); - if ($engine->getEngineIdentifier() == $this->getStorageEngine()) { + public static function buildEngine($engine_identifier) { + $engines = self::buildAllEngines(); + foreach ($engines as $engine) { + if ($engine->getEngineIdentifier() == $engine_identifier) { return $engine; } } - throw new Exception("File's storage engine could be located!"); + throw new Exception( + "Storage engine '{$engine_identifier}' could not be located!"); + } + + public static function buildAllEngines() { + $engines = id(new PhutilSymbolLoader()) + ->setType('class') + ->setConcreteOnly(true) + ->setAncestorClass('PhabricatorFileStorageEngine') + ->selectAndLoadSymbols(); + + $results = array(); + foreach ($engines as $engine_class) { + $results[] = newv($engine_class['name'], array()); + } + + return $results; } public function getViewableMimeType() { diff --git a/src/docs/configuration/configuring_file_storage.diviner b/src/docs/configuration/configuring_file_storage.diviner index 6d3af552c9..0484837bfd 100644 --- a/src/docs/configuration/configuring_file_storage.diviner +++ b/src/docs/configuration/configuring_file_storage.diviner @@ -80,6 +80,22 @@ Technical Documentation}. You can test that things are correctly configured by going to the Files application (##/file/##) and uploading files. += Migrating Files Beteeen Engines = + +If you want to move files between storage engines, you can use the `bin/files` +script to perform migrations. For example, suppose you previously used MySQL but +recently set up S3 and want to migrate all your files there. First, migrate one +file to make sure things work: + + phabricator/ $ ./bin/files migrate --engine amazon-s3 F12345 + +If that works properly, you can then migrate everything: + + phabricator/ $ ./bin/files migrate --engine amazon-s3 --all + +You can use `--dry-run` to show which migrations would be performed without +taking any action. Run `bin/files help` for more options and information. + = Next Steps = Continue by: