diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 0d4018b5..5523c037 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -181,23 +181,57 @@ public function uploadData(string $data, string $path, string $contentType, int return $chunksReceived; } - private function joinChunks(string $path, int $chunks) + private function joinChunks(string $path, int $chunks): void { - $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; + $tmpDir = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path); + $tmp = $tmpDir.DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; + $tmpAssemble = \dirname($path).DIRECTORY_SEPARATOR.'tmp_assemble_'.\basename($path); + + $dest = \fopen($tmpAssemble, 'wb'); + if ($dest === false) { + throw new Exception('Failed to open temporary assembly file '.$tmpAssemble); + } + + $partsToUnlink = []; for ($i = 1; $i <= $chunks; $i++) { - $part = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; - $data = file_get_contents($part); - if (! $data) { - throw new Exception('Failed to read chunk '.$part); + $part = $tmpDir.DIRECTORY_SEPARATOR.\pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; + $src = @\fopen($part, 'rb'); + if ($src === false) { + \fclose($dest); + \unlink($tmpAssemble); + throw new Exception('Failed to open chunk '.$part); } - if (! file_put_contents($path, $data, FILE_APPEND)) { - throw new Exception('Failed to append chunk '.$part); + if (\stream_copy_to_stream($src, $dest) === false) { + \fclose($src); + \fclose($dest); + \unlink($tmpAssemble); + throw new Exception('Failed to copy chunk '.$part); + } + \fclose($src); + $partsToUnlink[] = $part; + } + + \fclose($dest); + + if (! \rename($tmpAssemble, $path)) { + \unlink($tmpAssemble); + throw new Exception('Failed to finalize assembled file '.$path); + } + + foreach ($partsToUnlink as $part) { + if (! \unlink($part)) { + \trigger_error('Failed to remove chunk part '.$part, E_USER_WARNING); } - \unlink($part); } - \unlink($tmp); - \rmdir(dirname($tmp)); + + if (! \unlink($tmp)) { + \trigger_error('Failed to remove chunk log '.$tmp, E_USER_WARNING); + } + + if (! \rmdir($tmpDir)) { + \trigger_error('Failed to remove temporary chunk directory '.$tmpDir, E_USER_WARNING); + } } /** diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index b9e54759..0cfe048f 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -2,6 +2,7 @@ namespace Utopia\Tests\Storage\Device; +use Exception; use PHPUnit\Framework\TestCase; use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Local; @@ -429,4 +430,117 @@ public function testNestedDeletePath() $this->assertTrue($this->object->deletePath('nested-delete-path-test')); $this->assertFalse($this->object->exists($dir)); } + + // ------------------------------------------------------------------------- + // joinChunks tests + // ------------------------------------------------------------------------- + + /** + * Create a self-contained Local storage instance in a fresh temp directory. + * The caller is responsible for deleting the root after the test. + */ + private function makeJoinTestStorage(): Local + { + $dir = \sys_get_temp_dir().DIRECTORY_SEPARATOR.'utopia-join-test-'.uniqid(); + \mkdir($dir, 0755, true); + + return new Local($dir); + } + + public function testJoinChunksAssemblesContentCorrectly(): void + { + $storage = $this->makeJoinTestStorage(); + $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'test.dat'; + + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + + $this->assertTrue(\file_exists($dest)); + $this->assertSame('AAAABBBBCCCC', \file_get_contents($dest)); + + $storage->delete($storage->getRoot(), true); + } + + public function testJoinChunksCleansUpTempFilesOnSuccess(): void + { + $storage = $this->makeJoinTestStorage(); + $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'test.dat'; + $tmpDir = $storage->getRoot().DIRECTORY_SEPARATOR.'tmp_test.dat'; + $tmpAssemble = $storage->getRoot().DIRECTORY_SEPARATOR.'tmp_assemble_test.dat'; + + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + + $this->assertFalse(\is_dir($tmpDir), 'Temp chunk directory should be removed after assembly'); + $this->assertFalse(\file_exists($tmpAssemble), 'Temp assembly file should be removed after rename'); + for ($i = 1; $i <= 3; $i++) { + $this->assertFalse( + \file_exists($tmpDir.DIRECTORY_SEPARATOR.'test.part.'.$i), + "Part file $i should be removed after assembly" + ); + } + + $storage->delete($storage->getRoot(), true); + } + + public function testJoinChunksMissingPartThrowsAndPreservesState(): void + { + $storage = $this->makeJoinTestStorage(); + $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'test.dat'; + $tmpDir = $storage->getRoot().DIRECTORY_SEPARATOR.'tmp_test.dat'; + $tmpAssemble = $storage->getRoot().DIRECTORY_SEPARATOR.'tmp_assemble_test.dat'; + + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + + // Simulate a missing/corrupted chunk by deleting part 1 before the + // final upload triggers assembly. + \unlink($tmpDir.DIRECTORY_SEPARATOR.'test.part.1'); + + $exceptionThrown = false; + try { + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + } catch (Exception $e) { + $exceptionThrown = true; + $this->assertStringContainsString('Failed to open chunk', $e->getMessage()); + } + + $this->assertTrue($exceptionThrown, 'Exception should be thrown when a chunk is missing'); + $this->assertFalse(\file_exists($dest), 'Final file must not be created on assembly failure'); + $this->assertFalse(\file_exists($tmpAssemble), 'Temp assembly file must be cleaned up on failure'); + // Surviving parts must remain so the upload can be retried. + $this->assertTrue( + \file_exists($tmpDir.DIRECTORY_SEPARATOR.'test.part.2'), + 'Part 2 must be preserved for retry' + ); + $this->assertTrue( + \file_exists($tmpDir.DIRECTORY_SEPARATOR.'test.part.3'), + 'Part 3 must be preserved for retry' + ); + + $storage->delete($storage->getRoot(), true); + } + + public function testJoinChunksStaleAssemblyFileIsOverwritten(): void + { + $storage = $this->makeJoinTestStorage(); + $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'test.dat'; + $tmpAssemble = $storage->getRoot().DIRECTORY_SEPARATOR.'tmp_assemble_test.dat'; + + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + + // Simulate a stale assembly file left by a previously crashed attempt. + \file_put_contents($tmpAssemble, 'STALE_GARBAGE_DATA'); + + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + + $this->assertTrue(\file_exists($dest)); + $this->assertSame('AAAABBBBCCCC', \file_get_contents($dest), 'Stale assembly file must not corrupt output'); + $this->assertFalse(\file_exists($tmpAssemble), 'Temp assembly file should be removed after successful rename'); + + $storage->delete($storage->getRoot(), true); + } }