diff --git a/Command/Catalog/Media/Cleanup.php b/Command/Catalog/Media/Cleanup.php index bad2f44..10e9e42 100644 --- a/Command/Catalog/Media/Cleanup.php +++ b/Command/Catalog/Media/Cleanup.php @@ -6,10 +6,9 @@ */ namespace MagentoCode\CliMediaTool\Command\Catalog\Media; -use mysql_xdevapi\Exception; +use MagentoCode\CliMediaTool\Command\CatalogAbstract; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use MagentoCode\CliMediaTool\Command\CatalogAbstract; class Cleanup extends CatalogAbstract { @@ -38,15 +37,9 @@ protected function execute(InputInterface $input, OutputInterface $output) */ $filePaths = $this->getFilePaths(false); $mediaGalleryPaths = $this->getMediaGalleryPaths(false); - $output->writeln( - sprintf( - ' Files: %d - Media Gallery Entries: %d -===============================================', - count($filePaths), - count($mediaGalleryPaths) - ) - ); + $output->writeln(sprintf(' Files: %d', count($filePaths))); + $output->writeln(sprintf(' Media Gallery Entries: %d', count($mediaGalleryPaths))); + $output->writeln('==============================================='); /* * Cleanup orphaned media gallery entries that no longer have products using them @@ -68,7 +61,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $output->writeln( sprintf( 'Removed %d media gallery paths that no longer have images on the filesystem.', - count($orphanedMediaGalleryPaths) + count($missingFilePaths) ) ); @@ -80,7 +73,21 @@ protected function execute(InputInterface $input, OutputInterface $output) $output->writeln( sprintf( 'Removed %d files which are no longer present in the media gallery database table.', - count($orphanedMediaGalleryPaths) + count($unusedFilePaths) + ) + ); + + /* + * Remove temporary files + */ + $maxLifetimeInHours = Temporary::DEFAULT_MAX_LIFETIME; + $temporaryFilePaths = $this->getTemporaryFilePaths(false, $maxLifetimeInHours); + $this->deleteFilePaths($temporaryFilePaths); + $output->writeln( + sprintf( + 'Removed %d temporary files older that %d hour(s).', + count($unusedFilePaths), + $maxLifetimeInHours ) ); @@ -92,15 +99,9 @@ protected function execute(InputInterface $input, OutputInterface $output) $output->writeln('==============================================='); $output->writeln('=== Done ==='); $output->writeln('==============================================='); - $output->writeln( - sprintf( - ' Files: %d - Media Gallery Entries: %d -===============================================', - count($filePaths), - count($mediaGalleryPaths) - ) - ); + $output->writeln(sprintf(' Files: %d', count($filePaths))); + $output->writeln(sprintf(' Media Gallery Entries: %d', count($mediaGalleryPaths))); + $output->writeln('==============================================='); } catch (\Exception $exception) { $output->writeln(['Exception: ', $exception->getMessage()]); } diff --git a/Command/Catalog/Media/Info.php b/Command/Catalog/Media/Info.php index 1677712..c17c42d 100644 --- a/Command/Catalog/Media/Info.php +++ b/Command/Catalog/Media/Info.php @@ -6,10 +6,9 @@ */ namespace MagentoCode\CliMediaTool\Command\Catalog\Media; -use mysql_xdevapi\Exception; +use MagentoCode\CliMediaTool\Command\CatalogAbstract; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use MagentoCode\CliMediaTool\Command\CatalogAbstract; class Info extends CatalogAbstract { @@ -36,6 +35,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $cacheFilePaths = $this->getCacheFilePaths(); $unusedFiles = $this->getUnusedFilePaths(); $missingFiles = $this->getMissingFilePaths(); + $temporaryFiles = $this->getTemporaryFilePaths(); $output->writeln('==============================================='); $output->writeln(sprintf('Media Gallery entries: %s.', count($mediaGalleryPaths))); @@ -45,6 +45,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $output->writeln(sprintf('Unused files on filesystem: %s.', count($unusedFiles))); $output->writeln(sprintf('Missing files on filesystem: %s.', count($missingFiles))); $output->writeln(sprintf('Orphaned files on filesystem: %s.', count($orphanedFilePaths))); + $output->writeln(sprintf('Temporary files on filesystem: %s.', count($temporaryFiles))); $output->writeln('==============================================='); $output->writeln('To automatically clean up the media gallery, run catalog:media:cleanup'); } catch (\Exception $exception) { diff --git a/Command/Catalog/Media/Temporary.php b/Command/Catalog/Media/Temporary.php new file mode 100644 index 0000000..279a42b --- /dev/null +++ b/Command/Catalog/Media/Temporary.php @@ -0,0 +1,51 @@ + + * @license MIT + */ +namespace MagentoCode\CliMediaTool\Command\Catalog\Media; + +use MagentoCode\CliMediaTool\Command\CatalogAbstract; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class Temporary extends CatalogAbstract +{ + public const OPTION_MAX_LIFETIME = 'max_lifetime'; + public const DEFAULT_MAX_LIFETIME = 24; + + /** + * {@inheritdoc} + */ + protected function configure() + { + $this->setName('catalog:media:temporary') + ->addOption( + self::OPTION_MAX_LIFETIME, + 'm', + InputArgument::OPTIONAL, + 'Max temporary file lifetime in hours?', + self::DEFAULT_MAX_LIFETIME + ) + ->setDescription( + 'Get a list of product media temporary images which exist on the filesystem.' + ); + parent::configure(); + } + + /** + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + try { + $maxLifetimeInHours = (int)$input->getOption(self::OPTION_MAX_LIFETIME); + $temporaryFiles = $this->getTemporaryFilePaths(true, $maxLifetimeInHours); + $output->writeln($temporaryFiles); + } catch (\Exception $exception) { + $output->writeln(['Exception: ', $exception->getMessage()]); + } + } +} diff --git a/Command/Catalog/Media/Temporary/Remove.php b/Command/Catalog/Media/Temporary/Remove.php new file mode 100644 index 0000000..408be9f --- /dev/null +++ b/Command/Catalog/Media/Temporary/Remove.php @@ -0,0 +1,55 @@ + + * @license MIT + */ +namespace MagentoCode\CliMediaTool\Command\Catalog\Media\Temporary; + +use Exception; +use MagentoCode\CliMediaTool\Command\Catalog\Media\Temporary; +use MagentoCode\CliMediaTool\Command\CatalogAbstract; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class Remove extends CatalogAbstract +{ + /** + * {@inheritdoc} + */ + protected function configure() + { + $this->setName('catalog:media:temporary:remove') + ->addOption( + Temporary::OPTION_MAX_LIFETIME, + 'm', + InputArgument::OPTIONAL, + 'Max temporary file lifetime in hours?', + Temporary::DEFAULT_MAX_LIFETIME + ) + ->setDescription( + 'Remove all temporary product media images which exist on the filesystem.' + ); + parent::configure(); + } + + /** + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + try { + $maxLifetimeInHours = (int)$input->getOption(Temporary::OPTION_MAX_LIFETIME); + $temporaryFilePaths = $this->getTemporaryFilePaths(true, $maxLifetimeInHours); + $this->deleteFilePaths($temporaryFilePaths); + $output->writeln(sprintf( + 'Removed %d temporary files older that %d hour(s).', + count($temporaryFilePaths), + $maxLifetimeInHours + )); + } catch (Exception $exception) { + $output->writeln(['Exception: ', $exception->getMessage()]); + } + } +} diff --git a/Command/CatalogAbstract.php b/Command/CatalogAbstract.php index ba45487..3f4107d 100644 --- a/Command/CatalogAbstract.php +++ b/Command/CatalogAbstract.php @@ -4,17 +4,17 @@ * @author Brian Neff * @license MIT */ + namespace MagentoCode\CliMediaTool\Command; -use Magento\Framework\DB\Select; -use Symfony\Component\Console\Command\Command; -use Symfony\Component\Console\Input\InputInterface; -use Symfony\Component\Console\Output\OutputInterface; -use Magento\Framework\App\ResourceConnection; +use Magento\Catalog\Model\Product\Media\Config as ProductMediaConfig; use Magento\Catalog\Model\ResourceModel\Product\Gallery; -use Magento\Framework\Filesystem\Driver\File; use Magento\Framework\App\Filesystem\DirectoryList; -use Symfony\Component\Console\Input\InputOption; +use Magento\Framework\App\ResourceConnection; +use Magento\Framework\DB\Select; +use Magento\Framework\Exception\FileSystemException; +use Magento\Framework\Filesystem\Driver\File; +use Symfony\Component\Console\Command\Command; abstract class CatalogAbstract extends Command { @@ -53,11 +53,21 @@ abstract class CatalogAbstract extends Command */ protected $driverFile; + /** + * @var ProductMediaConfig + */ + private $productMediaConfig; + /** * @var array The file paths from the media directory */ private $filePaths = []; + /** + * @var array The file paths from the tmp media directory + */ + private $tmpFilePaths = []; + /** * @var array The cache file paths from the media directory */ @@ -88,16 +98,29 @@ abstract class CatalogAbstract extends Command */ private $unusedFiles = []; + /** + * @var array The temporary file paths + */ + private $temporaryFiles = []; + /** * Constructor * * @param ResourceConnection $resource + * @param DirectoryList $directoryList + * @param File $driverFile + * @param ProductMediaConfig $productMediaConfig */ - public function __construct(ResourceConnection $resource, DirectoryList $directoryList, File $driverFile) - { + public function __construct( + ResourceConnection $resource, + DirectoryList $directoryList, + File $driverFile, + ProductMediaConfig $productMediaConfig + ) { $this->resource = $resource; $this->directoryList = $directoryList; $this->driverFile = $driverFile; + $this->productMediaConfig = $productMediaConfig; parent::__construct(); } @@ -112,6 +135,17 @@ protected function getMediaDirectoryPath() .DIRECTORY_SEPARATOR.'catalog'.DIRECTORY_SEPARATOR.'product'; } + /** + * Gets the temporary product media directory path + * @return string + * @throws FileSystemException + */ + protected function getTmpMediaDirectoryPath() + { + return $this->directoryList->getPath(DirectoryList::MEDIA) + . DIRECTORY_SEPARATOR . $this->productMediaConfig->getBaseTmpMediaPath(); + } + /** * Gets the product media directory path * @return string @@ -123,6 +157,28 @@ protected function getCacheDirectoryPath() .DIRECTORY_SEPARATOR.'catalog'.DIRECTORY_SEPARATOR.'product'.DIRECTORY_SEPARATOR.'cache'; } + /** + * Gets the product placeholder directory path + * @return string + * @throws \Magento\Framework\Exception\FileSystemException + */ + protected function getPlaceholderDirectoryPath(): string + { + return $this->directoryList->getPath(DirectoryList::MEDIA) + .DIRECTORY_SEPARATOR.'catalog'.DIRECTORY_SEPARATOR.'product'.DIRECTORY_SEPARATOR.'placeholder'; + } + + /** + * Gets the Mirasvit SEO-friendly product image directory path + * @return string + * @throws \Magento\Framework\Exception\FileSystemException + */ + protected function getMirasvitSEODirectoryPath(): string + { + return $this->directoryList->getPath(DirectoryList::MEDIA) + .DIRECTORY_SEPARATOR.'catalog'.DIRECTORY_SEPARATOR.'product'.DIRECTORY_SEPARATOR.'image'; + } + /** * Get the file paths for all media files exclusing cache files * @param $useCache boolean @@ -134,20 +190,62 @@ protected function getFilePaths($useCache = true) if ($useCache && $this->filePaths) { return $this->filePaths; } - if ($this->driverFile->isExists($this->getMediaDirectoryPath())) { - $this->filePaths = $this->driverFile->readDirectoryRecursively($this->getMediaDirectoryPath()); - $cacheFiles = $this->getCacheFilePaths(); - $this->filePaths = array_diff($this->filePaths, $cacheFiles); - //remove anything that's a directory - foreach ($this->filePaths as $k => $filePath) { - if ($this->driverFile->isDirectory($filePath)) { - unset($this->filePaths[$k]); + $mediaDirectoryPath = $this->getMediaDirectoryPath(); + if (!$this->driverFile->isExists($mediaDirectoryPath)) { + return []; + } + $this->filePaths = []; + $excludedPaths = [ + $this->getCacheDirectoryPath(), + $this->getMirasvitSEODirectoryPath(), + $this->getPlaceholderDirectoryPath() + ]; + foreach ($this->driverFile->readDirectory($mediaDirectoryPath) as $dirsPath) { + if (!$this->driverFile->isDirectory($dirsPath) || in_array($dirsPath, $excludedPaths, true)) { + continue; // Skip files and excluded dirs + } + + foreach ($this->driverFile->readDirectoryRecursively($dirsPath) as $filePath) { + if (!$this->driverFile->isDirectory($filePath)) { + $this->filePaths[] = $filePath; } } } return $this->filePaths; } + /** + * Get the file paths for all tmp media files + * @param $useCache boolean + * @return array|string[] + * @throws FileSystemException + */ + protected function getTmpFilePaths($useCache = true, int $maxLifetimeInHours = 0) + { + if ($useCache && $this->tmpFilePaths) { + return $this->tmpFilePaths; + } + if ($this->driverFile->isExists($this->getTmpMediaDirectoryPath())) { + $this->tmpFilePaths = $this->driverFile->readDirectoryRecursively($this->getTmpMediaDirectoryPath()); + + $maxLifetime = $maxLifetimeInHours * 60 * 60; // in seconds + + foreach ($this->tmpFilePaths as $k => $filePath) { + if ($this->driverFile->isDirectory($filePath)) { + // Skip directories + unset($this->tmpFilePaths[$k]); + } elseif ($this->driverFile->isFile($filePath) + && $this->driverFile->isReadable($filePath) + && (time() - $this->driverFile->stat($filePath)['mtime']) < $maxLifetime + ) { + // Skip newly created files + unset($this->tmpFilePaths[$k]); + } + } + } + return $this->tmpFilePaths; + } + /** * Get the file paths for all cache files * @param $useCache boolean @@ -160,13 +258,12 @@ protected function getCacheFilePaths($useCache = true) return $this->cacheFilePaths; } if ($this->driverFile->isExists($this->getCacheDirectoryPath())) { - $this->cacheFilePaths = $this->driverFile->readDirectoryRecursively($this->getCacheDirectoryPath()); - //remove anything that's a directory - foreach ($this->cacheFilePaths as $k => $cacheFilePath) { - if ($this->driverFile->isDirectory($cacheFilePath)) { - unset($this->cacheFilePaths[$k]); + $this->cacheFilePaths = array_filter( + $this->driverFile->readDirectoryRecursively($this->getCacheDirectoryPath()), + function ($filePath) { + return !$this->driverFile->isDirectory($filePath); // skip directories } - } + ); } return $this->cacheFilePaths; } @@ -192,7 +289,7 @@ protected function getOrphanedMediaGalleryPaths($useCache = true) ) ->reset(Select::COLUMNS) ->columns('value') - ->where('ent.row_id IS NULL'); + ->where('ent.entity_id IS NULL'); $this->orphanedMediaGalleryPaths = $connection->fetchCol($select); //add the directory path to each value because the DB only stores the path relative to the media directory $mediaDirectoryPath = $this->getMediaDirectoryPath(); @@ -273,6 +370,21 @@ protected function getUnusedFilePaths($useCache = true) return $this->unusedFiles; } + /** + * Gets an array of all the files that are not present in the media gallery table + * @param $useCache boolean + * @return array + * @throws FileSystemException + */ + protected function getTemporaryFilePaths($useCache = true, int $maxLifetimeInHours = 0) + { + if ($useCache && $this->temporaryFiles) { + return $this->temporaryFiles; + } + $this->temporaryFiles = $this->getTmpFilePaths($useCache, $maxLifetimeInHours); + return $this->temporaryFiles; + } + /** * Deletes the provided media gallery paths from the database * @param array $mediaGalleryPaths @@ -287,15 +399,14 @@ protected function deleteMediaGalleryPaths(array $mediaGalleryPaths) }, $mediaGalleryPaths); $connection = $this->resource->getConnection(); - $select = $connection->select() - ->from($connection->getTableName(Gallery::GALLERY_TABLE)) - ->where('value IN(?)', $mediaGalleryPaths); + $chunkSize = 500; - $delete = $connection->deleteFromSelect( - $select, - $connection->getTableName(Gallery::GALLERY_TABLE) - ); - $connection->query($delete); + foreach (array_chunk($mediaGalleryPaths, $chunkSize) as $mediaGalleryPathsChunk) { + $connection->delete( + $connection->getTableName(Gallery::GALLERY_TABLE), + ['value IN (?)' => $mediaGalleryPathsChunk] + ); + } } /** @@ -305,6 +416,12 @@ protected function deleteMediaGalleryPaths(array $mediaGalleryPaths) */ protected function deleteFilePaths(array $filePaths) { - array_map([$this->driverFile, 'deleteFile'], $filePaths); + foreach ($filePaths as $filePath) { + try { + $this->driverFile->deleteFile($filePath); + } catch (\Magento\Framework\Exception\FileSystemException $e) { + // Skip this exceptions + } + } } } diff --git a/README.md b/README.md index dac3b12..81e2b10 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Within this module and the commands herein, we use the following terminology to * **Missing** - an image which is referenced in the database, but does not exist on the filesystem * **Orphaned** - an image which is referenced in the database, but is not in use on any current products * **Unused** - an image which is present on the filesystem, but not in the database, and as such is not currently in use +* **Temporary** - a temporary image which is present on the filesystem ## Installation @@ -34,8 +35,8 @@ bin/magento setup:upgrade Use this command to retrieve a summary of information about the current state of your product media gallery. This can be used as an easy way to run a checkup on your media gallery to see if you're wasting any disk space on old images. In general, a "healthy" media gallery should have the same number of `Media Gallery entries` as it does -`Non-cache media files` with no `Unused`, `Missing`, or `Orphaned` files or media gallery entries. The number of -`Cache media files` should not be taken into consideration. +`Non-cache media files` with no `Unused`, `Missing`, `Orphaned` or `Temporary` files or media gallery entries. The +number of `Cache media files` should not be taken into consideration. ``` bin/magento catalog:media:info @@ -47,6 +48,7 @@ Cache media files on filesystem: 4813. Unused files on filesystem: 0. Missing files on filesystem: 0. Orphaned files on filesystem: 0. +Temporary files on filesystem: 0. =============================================== ``` @@ -55,10 +57,11 @@ Orphaned files on filesystem: 0. ##### This is a destructive operation and will make changes to your database and filesystem when run. Use this command to automatically clean up your media gallery database and filesystem. Running this command is the same -as running `catalog:media:orphaned:remove`, `catalog:media:missing:remove`, and `catalog:media:unused:remove` except it -removes **all** orphaned records from the database instead of just those with files on the filesystem. It also does all -of this for you in a single command, and is particularly useful for a quick cleanup of your media gallery. **It is -highly recommended that you make a backup of your site files and database before running this command!** +as running `catalog:media:orphaned:remove`, `catalog:media:missing:remove`, `catalog:media:unused:remove` +and `catalog:media:temporary:remove` except it removes **all** orphaned records from the database instead of just those +with files on the filesystem. It also does all of this for you in a single command, and is particularly useful for a +quick cleanup of your media gallery. **It is highly recommended that you make a backup of your site files and database +before running this command!** ``` bin/magento catalog:media:cleanup @@ -71,6 +74,7 @@ bin/magento catalog:media:cleanup Removed 0 orphaned media gallery paths which are no longer in use on any products. Removed 0 media gallery paths that no longer have images on the filesystem. Removed 0 files which are no longer present in the media gallery database table. +Removed 0 temporary files older that 24 hour(s). =============================================== === Done === =============================================== @@ -172,4 +176,27 @@ This command will remove all current `Unused` files from the filesystem and outp ``` bin/magento catalog:media:unused:remove Removed 0 unused files. -``` \ No newline at end of file +``` + +### catalog:media:temporary + +This command simply outputs the full absolute file path to each of the `Temporary` file paths in the media gallery. It is +mostly intended for use with custom shell scripts or external programs which may want to grab this output for their own +use. + +``` +bin/magento catalog:media:temporary -m 24 +/path/to/pub/media/tmp/catalog/product/i/m/image1.jpg +/path/to/pub/media/tmp/catalog/product/i/m/image2.jpg +/path/to/pub/media/tmp/catalog/product/i/m/image3.jpg +/path/to/pub/media/tmp/catalog/product/i/m/image4.jpg +``` + +### catalog:media:temporary:remove + +This command will remove all current `Temporary` files from the filesystem and output the number of items removed. + +``` +bin/magento catalog:media:temporary:remove -m 24 +Removed 0 temporary files older that 24 hour(s). +``` diff --git a/composer.json b/composer.json index ae8d94c..2eb4b59 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,6 @@ "name": "magentocode/magento2-cli-media-tool", "description": "Magento 2 CLI media analysis and cleanup tool.", "type": "magento2-module", - "version": "1.1.0", "license": "MIT", "authors": [ { diff --git a/etc/di.xml b/etc/di.xml index 17422e6..6587d36 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -21,6 +21,8 @@ MagentoCode\CliMediaTool\Command\Catalog\Media\Orphaned\Remove MagentoCode\CliMediaTool\Command\Catalog\Media\Unused MagentoCode\CliMediaTool\Command\Catalog\Media\Unused\Remove + MagentoCode\CliMediaTool\Command\Catalog\Media\Temporary + MagentoCode\CliMediaTool\Command\Catalog\Media\Temporary\Remove