From b27fd05eef0a20176574c7ff433ab59911738a77 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 05:39:09 -0800 Subject: [PATCH] Add a `bin/bulk export` CLI tool to make debugging and profiling large exports easier Summary: Ref T13049. When stuff executes asynchronously on the bulk workflow it can be hard to inspect directly, and/or a pain to test because you have to go through a bunch of steps to run it again. Make future work here easier by making export triggerable from the CLI. This makes it easy to repeat, inspect with `--trace`, profile with `--xprofile`, etc. Test Plan: - Ran several invalid commands, got sensible error messages. - Ran some valid commands, got exported data. - Used `--xprofile` to look at the profile for a 300MB dump of 100K tasks which took about 40 seconds to export. Nothing jumped out as sketchy to me -- CustomField wrangling is a little slow but most of the time looked like it was being spent legitimately. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18965 --- src/__phutil_library_map__.php | 2 + ...habricatorBulkManagementExportWorkflow.php | 168 ++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 06f619d74e..32985c76c0 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2226,6 +2226,7 @@ phutil_register_library_map(array( 'PhabricatorBulkContentSource' => 'infrastructure/daemon/contentsource/PhabricatorBulkContentSource.php', 'PhabricatorBulkEditGroup' => 'applications/transactions/bulk/PhabricatorBulkEditGroup.php', 'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php', + 'PhabricatorBulkManagementExportWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php', 'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php', 'PhabricatorCSVExportFormat' => 'infrastructure/export/format/PhabricatorCSVExportFormat.php', @@ -7579,6 +7580,7 @@ phutil_register_library_map(array( 'PhabricatorBulkContentSource' => 'PhabricatorContentSource', 'PhabricatorBulkEditGroup' => 'Phobject', 'PhabricatorBulkEngine' => 'Phobject', + 'PhabricatorBulkManagementExportWorkflow' => 'PhabricatorBulkManagementWorkflow', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'PhabricatorBulkManagementWorkflow', 'PhabricatorBulkManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorCSVExportFormat' => 'PhabricatorExportFormat', diff --git a/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php b/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php new file mode 100644 index 0000000000..cd0ed346fc --- /dev/null +++ b/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php @@ -0,0 +1,168 @@ +setName('export') + ->setExamples('**export** [options]') + ->setSynopsis( + pht('Export data to a flat file (JSON, CSV, Excel, etc).')) + ->setArguments( + array( + array( + 'name' => 'class', + 'param' => 'class', + 'help' => pht( + 'SearchEngine class to export data from.'), + ), + array( + 'name' => 'format', + 'param' => 'format', + 'help' => pht('Export format.'), + ), + array( + 'name' => 'query', + 'param' => 'key', + 'help' => pht( + 'Export the data selected by this query.'), + ), + array( + 'name' => 'output', + 'param' => 'path', + 'help' => pht( + 'Write output to a file. If omitted, output will be sent to '. + 'stdout.'), + ), + array( + 'name' => 'overwrite', + 'help' => pht( + 'If the output file already exists, overwrite it instead of '. + 'raising an error.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $class = $args->getArg('class'); + + if (!strlen($class)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a search engine class to export data from with '. + '"--class".')); + } + + if (!is_subclass_of($class, 'PhabricatorApplicationSearchEngine')) { + throw new PhutilArgumentUsageException( + pht( + 'SearchEngine class ("%s") is unknown.', + $class)); + } + + $engine = newv($class, array()) + ->setViewer($viewer); + + if (!$engine->canExport()) { + throw new PhutilArgumentUsageException( + pht( + 'SearchEngine class ("%s") does not support data export.', + $class)); + } + + $query_key = $args->getArg('query'); + if (!strlen($query_key)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a query to export with "--query".')); + } + + if ($engine->isBuiltinQuery($query_key)) { + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); + } else if ($query_key) { + $saved_query = id(new PhabricatorSavedQueryQuery()) + ->setViewer($viewer) + ->withQueryKeys(array($query_key)) + ->executeOne(); + } else { + $saved_query = null; + } + + if (!$saved_query) { + throw new PhutilArgumentUsageException( + pht( + 'Failed to load saved query ("%s").', + $query_key)); + } + + $format_key = $args->getArg('format'); + if (!strlen($format_key)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify an export format with "--format".')); + } + + $all_formats = PhabricatorExportFormat::getAllExportFormats(); + $format = idx($all_formats, $format_key); + if (!$format) { + throw new PhutilArgumentUsageException( + pht( + 'Unknown export format ("%s"). Known formats are: %s.', + $format_key, + implode(', ', array_keys($all_formats)))); + } + + if (!$format->isExportFormatEnabled()) { + throw new PhutilArgumentUsageException( + pht( + 'Export format ("%s") is not enabled.', + $format_key)); + } + + $is_overwrite = $args->getArg('overwrite'); + $output_path = $args->getArg('output'); + + if (!strlen($output_path) && $is_overwrite) { + throw new PhutilArgumentUsageException( + pht( + 'Flag "--overwrite" has no effect without "--output".')); + } + + if (!$is_overwrite) { + if (Filesystem::pathExists($output_path)) { + throw new PhutilArgumentUsageException( + pht( + 'Output path already exists. Use "--overwrite" to overwrite '. + 'it.')); + } + } + + $export_engine = id(new PhabricatorExportEngine()) + ->setViewer($viewer) + ->setTitle(pht('Export')) + ->setFilename(pht('export')) + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setExportFormat($format); + + $file = $export_engine->exportFile(); + + $iterator = $file->getFileDataIterator(); + + if (strlen($output_path)) { + foreach ($iterator as $chunk) { + Filesystem::appendFile($output_path, $chunk); + } + } else { + foreach ($iterator as $chunk) { + echo $chunk; + } + } + + return 0; + } + +}