diff --git a/src/Volume/Volume.cpp b/src/Volume/Volume.cpp index ceca072ef7..838d579c9c 100644 --- a/src/Volume/Volume.cpp +++ b/src/Volume/Volume.cpp @@ -14,6 +14,8 @@ #include #endif #include "EncryptionModeXTS.h" +#include "EncryptionThreadPool.h" +#include "Platform/Finally.h" #include "Volume.h" #include "VolumeHeader.h" #include "VolumeLayout.h" @@ -120,6 +122,114 @@ namespace VeraCrypt bool skipLayoutV1Normal = false; + // Finalizes the volume once a layout's header has been decrypted. + // Shared by the serial and parallel header-detection paths below. + auto setupFromDecryptedHeader = [&](shared_ptr layout, shared_ptr header) + { + if (typeid (*layout) == typeid (VolumeLayoutV2Normal) && header->GetRequiredMinProgramVersion() < 0x10b) + { + // VolumeLayoutV1Normal has been opened as VolumeLayoutV2Normal + layout.reset (new VolumeLayoutV1Normal); + header->SetSize (layout->GetHeaderSize()); + layout->SetHeader (header); + } + + Pim = pim; + Type = layout->GetType(); + SectorSize = header->GetSectorSize(); + + VolumeDataOffset = layout->GetDataOffset (VolumeHostSize); + VolumeDataSize = layout->GetDataSize (VolumeHostSize); + EncryptedDataSize = header->GetEncryptedAreaLength(); + + Header = header; + Layout = layout; + EA = header->GetEncryptionAlgorithm(); + EncryptionMode &mode = *EA->GetMode(); + + if (layout->HasDriveHeader()) + { + if (header->GetEncryptedAreaLength() != header->GetVolumeDataSize()) + { + EncryptionNotCompleted = true; + // we avoid writing data to the partition since it is only partially encrypted + Protection = VolumeProtection::ReadOnly; + } + + uint64 partitionStartOffset = VolumeFile->GetPartitionDeviceStartOffset(); + + if (partitionStartOffset < header->GetEncryptedAreaStart() + || partitionStartOffset >= header->GetEncryptedAreaStart() + header->GetEncryptedAreaLength()) + throw PasswordIncorrect (SRC_POS); + + EncryptedDataSize -= partitionStartOffset - header->GetEncryptedAreaStart(); + + mode.SetSectorOffset (partitionStartOffset / ENCRYPTION_DATA_UNIT_SIZE); + } + + // Volume protection + if (Protection == VolumeProtection::HiddenVolumeReadOnly) + { + if (Type == VolumeType::Hidden) + throw PasswordIncorrect (SRC_POS); + else + { + try + { + Volume protectedVolume; + + protectedVolume.Open (VolumeFile, + protectionPassword, protectionPim, protectionKdf, protectionKeyfiles, + emvSupportEnabled, + VolumeProtection::ReadOnly, + shared_ptr (), 0, shared_ptr (),shared_ptr (), + VolumeType::Hidden, + useBackupHeaders); + + if (protectedVolume.GetType() != VolumeType::Hidden) + ParameterIncorrect (SRC_POS); + + ProtectedRangeStart = protectedVolume.VolumeDataOffset; + ProtectedRangeEnd = protectedVolume.VolumeDataOffset + protectedVolume.VolumeDataSize; + } + catch (PasswordException&) + { + if (protectionKeyfiles && !protectionKeyfiles->empty()) + throw ProtectionPasswordKeyfilesIncorrect (SRC_POS); + throw ProtectionPasswordIncorrect (SRC_POS); + } + } + } + }; + + // When no specific KDF is requested and the encryption thread pool is + // available, gather every candidate layout header and derive their keys + // in parallel (VolumeHeader::DecryptHeaderParallel) so expensive KDFs + // (e.g. Argon2) for different layouts run concurrently instead of one + // layout at a time. Otherwise use the original serial detection. + // Start the encryption thread pool for parallel key derivation only if + // it is not already running (the GUI keeps a persistent pool). When we + // start it here -- e.g. in the elevated core service -- we also stop it + // before returning, so it is NOT running when the caller forks to launch + // the FUSE daemon (fork() in a multithreaded process is unsafe). The + // FUSE daemon starts its own pool after that fork. + bool encryptionPoolStartedHere = false; + if (!kdf && !EncryptionThreadPool::IsRunning()) + { + try + { + EncryptionThreadPool::Start(); + encryptionPoolStartedHere = true; + } + catch (...) { } + } + finally_do_arg (bool, encryptionPoolStartedHere, { if (finally_arg) EncryptionThreadPool::Stop(); }); + + bool useParallelHeaderDetection = (!kdf && EncryptionThreadPool::IsRunning()); + vector < shared_ptr > candidateLayouts; + vector < shared_ptr > candidateBuffers; // keep header buffers alive for the parallel call + vector candidates; + // Test volume layouts foreach (shared_ptr layout, VolumeLayout::GetAvailableLayouts (volumeType)) { @@ -132,7 +242,7 @@ namespace VeraCrypt if (useBackupHeaders && !layout->HasBackupHeader()) continue; - SecureBuffer headerBuffer (layout->GetHeaderSize()); + shared_ptr headerBuffer (new SecureBuffer (layout->GetHeaderSize())); if (layout->HasDriveHeader()) { @@ -152,7 +262,7 @@ namespace VeraCrypt else driveDevice.SeekEnd (headerOffset); - if (driveDevice.Read (headerBuffer) != layout->GetHeaderSize()) + if (driveDevice.Read (*headerBuffer) != layout->GetHeaderSize()) continue; } else @@ -167,7 +277,7 @@ namespace VeraCrypt else VolumeFile->SeekEnd (headerOffset); - if (VolumeFile->Read (headerBuffer) != layout->GetHeaderSize()) + if (VolumeFile->Read (*headerBuffer) != layout->GetHeaderSize()) continue; } @@ -185,84 +295,32 @@ namespace VeraCrypt shared_ptr header = layout->GetHeader(); - if (header->Decrypt (headerBuffer, *passwordKey, pim, kdf, layout->GetSupportedKeyDerivationFunctions(), layoutEncryptionAlgorithms, layoutEncryptionModes)) + if (useParallelHeaderDetection) { - // Header decrypted - - if (typeid (*layout) == typeid (VolumeLayoutV2Normal) && header->GetRequiredMinProgramVersion() < 0x10b) - { - // VolumeLayoutV1Normal has been opened as VolumeLayoutV2Normal - layout.reset (new VolumeLayoutV1Normal); - header->SetSize (layout->GetHeaderSize()); - layout->SetHeader (header); - } - - Pim = pim; - Type = layout->GetType(); - SectorSize = header->GetSectorSize(); - - VolumeDataOffset = layout->GetDataOffset (VolumeHostSize); - VolumeDataSize = layout->GetDataSize (VolumeHostSize); - EncryptedDataSize = header->GetEncryptedAreaLength(); - - Header = header; - Layout = layout; - EA = header->GetEncryptionAlgorithm(); - EncryptionMode &mode = *EA->GetMode(); - - if (layout->HasDriveHeader()) - { - if (header->GetEncryptedAreaLength() != header->GetVolumeDataSize()) - { - EncryptionNotCompleted = true; - // we avoid writing data to the partition since it is only partially encrypted - Protection = VolumeProtection::ReadOnly; - } - - uint64 partitionStartOffset = VolumeFile->GetPartitionDeviceStartOffset(); - - if (partitionStartOffset < header->GetEncryptedAreaStart() - || partitionStartOffset >= header->GetEncryptedAreaStart() + header->GetEncryptedAreaLength()) - throw PasswordIncorrect (SRC_POS); - - EncryptedDataSize -= partitionStartOffset - header->GetEncryptedAreaStart(); - - mode.SetSectorOffset (partitionStartOffset / ENCRYPTION_DATA_UNIT_SIZE); - } + VolumeHeader::DecryptCandidate candidate; + candidate.Header = header; + candidate.EncryptedData = *headerBuffer; + candidate.KeyDerivationFunctions = layout->GetSupportedKeyDerivationFunctions(); + candidate.EncryptionAlgorithms = layoutEncryptionAlgorithms; + candidate.EncryptionModes = layoutEncryptionModes; + + candidates.push_back (candidate); + candidateLayouts.push_back (layout); + candidateBuffers.push_back (headerBuffer); + } + else if (header->Decrypt (*headerBuffer, *passwordKey, pim, kdf, layout->GetSupportedKeyDerivationFunctions(), layoutEncryptionAlgorithms, layoutEncryptionModes)) + { + setupFromDecryptedHeader (layout, header); + return; + } + } - // Volume protection - if (Protection == VolumeProtection::HiddenVolumeReadOnly) - { - if (Type == VolumeType::Hidden) - throw PasswordIncorrect (SRC_POS); - else - { - try - { - Volume protectedVolume; - - protectedVolume.Open (VolumeFile, - protectionPassword, protectionPim, protectionKdf, protectionKeyfiles, - emvSupportEnabled, - VolumeProtection::ReadOnly, - shared_ptr (), 0, shared_ptr (),shared_ptr (), - VolumeType::Hidden, - useBackupHeaders); - - if (protectedVolume.GetType() != VolumeType::Hidden) - ParameterIncorrect (SRC_POS); - - ProtectedRangeStart = protectedVolume.VolumeDataOffset; - ProtectedRangeEnd = protectedVolume.VolumeDataOffset + protectedVolume.VolumeDataSize; - } - catch (PasswordException&) - { - if (protectionKeyfiles && !protectionKeyfiles->empty()) - throw ProtectionPasswordKeyfilesIncorrect (SRC_POS); - throw ProtectionPasswordIncorrect (SRC_POS); - } - } - } + if (useParallelHeaderDetection && !candidates.empty()) + { + int winningCandidate = VolumeHeader::DecryptHeaderParallel (candidates, *passwordKey, pim); + if (winningCandidate >= 0) + { + setupFromDecryptedHeader (candidateLayouts[winningCandidate], candidates[winningCandidate].Header); return; } } diff --git a/src/Volume/VolumeHeader.cpp b/src/Volume/VolumeHeader.cpp index a690057f7d..bcfb14009b 100644 --- a/src/Volume/VolumeHeader.cpp +++ b/src/Volume/VolumeHeader.cpp @@ -201,6 +201,145 @@ namespace VeraCrypt return false; } + int VolumeHeader::DecryptHeaderParallel (const vector &candidates, const VolumePassword &password, int pim) + { + if (password.Size() < 1) + throw PasswordEmpty (SRC_POS); + + if (!EncryptionThreadPool::IsRunning()) + return -1; + + typedef EncryptionThreadPool::KeyDerivationWorkItem KeyDerivationWorkItem; + + // One work item per (candidate x KDF). 'Tested' guards DecryptWithHeaderKey + // against being re-run on the same item across resolution passes. + struct Entry + { + shared_ptr Item; + bool Tested; + }; + + // Grouped by candidate so candidates can be resolved in their original + // (priority) order even though the derivations complete concurrently. + vector < vector > candidateEntries (candidates.size()); + SharedVal outstandingWorkItemCount (0); + SyncEvent keyDerivationCompletedEvent; + SyncEvent noOutstandingWorkItemEvent; + long volatile abortKeyDerivation = 0; + size_t enqueuedWorkItemCount = 0; + bool workItemsDrained = false; + + try + { + for (size_t ci = 0; ci < candidates.size(); ++ci) + { + const DecryptCandidate &candidate = candidates[ci]; + // The salt is a view into the candidate's header buffer, which the + // caller keeps alive for the duration of this call. + ConstBufferPtr salt (candidate.EncryptedData.GetRange (SaltOffset, SaltSize)); + + foreach (shared_ptr pkcs5, candidate.KeyDerivationFunctions) + { + Entry entry; + entry.Item = shared_ptr (new KeyDerivationWorkItem (pkcs5, GetHeaderKeyDerivationSize (pkcs5))); + entry.Tested = false; + candidateEntries[ci].push_back (entry); + + EncryptionThreadPool::BeginKeyDerivation (*entry.Item, password, pim, salt, keyDerivationCompletedEvent, noOutstandingWorkItemEvent, outstandingWorkItemCount, &abortKeyDerivation); + ++enqueuedWorkItemCount; + } + } + + // Resolve candidates strictly in priority order, preserving the serial + // detection semantics: candidate N is only considered once every + // higher-priority candidate is known not to decrypt (and not to throw). + // Within a candidate, the first KDF whose derived key decrypts the header + // wins, so a fast match does not wait on that candidate's slow KDFs. + size_t nextCandidate = 0; + while (nextCandidate < candidates.size()) + { + bool recordedCompletion = false; + + // Mark newly completed work items as done across all remaining + // candidates so the completion signal is fully consumed each pass. + for (size_t ci = nextCandidate; ci < candidates.size(); ++ci) + { + for (size_t i = 0; i < candidateEntries[ci].size(); ++i) + { + Entry &entry = candidateEntries[ci][i]; + if (!entry.Item->Processed && entry.Item->Completed.Get()) + { + entry.Item->Processed = true; + recordedCompletion = true; + } + } + } + + // Try to resolve the current (highest remaining priority) candidate. + const DecryptCandidate &candidate = candidates[nextCandidate]; + vector &entries = candidateEntries[nextCandidate]; + bool candidateComplete = true; + + for (size_t i = 0; i < entries.size(); ++i) + { + Entry &entry = entries[i]; + if (!entry.Item->Processed) + { + candidateComplete = false; + continue; + } + + if (entry.Item->ItemException.get()) + { + // KDF exceptions are fatal; surfaced in candidate priority order. + abortKeyDerivation = 1; + DrainKeyDerivationWorkItems (noOutstandingWorkItemEvent, enqueuedWorkItemCount, workItemsDrained); + entry.Item->ItemException->Throw(); + } + + if (entry.Tested) + continue; + + entry.Tested = true; + + if (entry.Item->Result != 0) + continue; + + // DecryptWithHeaderKey may throw (e.g. HigherVersionRequired); it + // runs only for the highest-priority unresolved candidate, so that + // exception keeps the same priority as the serial path. + if (candidate.Header->DecryptWithHeaderKey (candidate.EncryptedData, entry.Item->Kdf, entry.Item->DerivedKey, candidate.EncryptionAlgorithms, candidate.EncryptionModes)) + { + abortKeyDerivation = 1; + DrainKeyDerivationWorkItems (noOutstandingWorkItemEvent, enqueuedWorkItemCount, workItemsDrained); + return (int) nextCandidate; + } + } + + if (candidateComplete) + { + // All of this candidate's KDFs finished without a match. + ++nextCandidate; + continue; + } + + // Current candidate still has outstanding derivations; if nothing new + // was recorded this pass, block until another work item completes. + if (!recordedCompletion) + keyDerivationCompletedEvent.Wait(); + } + } + catch (...) + { + abortKeyDerivation = 1; + DrainKeyDerivationWorkItems (noOutstandingWorkItemEvent, enqueuedWorkItemCount, workItemsDrained); + throw; + } + + DrainKeyDerivationWorkItems (noOutstandingWorkItemEvent, enqueuedWorkItemCount, workItemsDrained); + return -1; + } + bool VolumeHeader::DecryptWithHeaderKey (const ConstBufferPtr &encryptedData, shared_ptr pkcs5, const ConstBufferPtr &headerKey, const EncryptionAlgorithmList &encryptionAlgorithms, const EncryptionModeList &encryptionModes) { SecureBuffer header (EncryptedHeaderDataSize); diff --git a/src/Volume/VolumeHeader.h b/src/Volume/VolumeHeader.h index 7d6844f330..5f25743574 100644 --- a/src/Volume/VolumeHeader.h +++ b/src/Volume/VolumeHeader.h @@ -61,6 +61,24 @@ namespace VeraCrypt void Create (const BufferPtr &headerBuffer, VolumeHeaderCreationOptions &options); bool Decrypt (const ConstBufferPtr &encryptedData, const VolumePassword &password, int pim, shared_ptr kdf, const Pkcs5KdfList &keyDerivationFunctions, const EncryptionAlgorithmList &encryptionAlgorithms, const EncryptionModeList &encryptionModes); + + // A candidate header (one volume layout) for DecryptHeaderParallel(). + struct DecryptCandidate + { + shared_ptr Header; + ConstBufferPtr EncryptedData; // must remain valid for the call + Pkcs5KdfList KeyDerivationFunctions; + EncryptionAlgorithmList EncryptionAlgorithms; + EncryptionModeList EncryptionModes; + }; + + // Tries to decrypt several candidate headers (different volume layouts) + // concurrently: every (candidate x KDF) derivation is dispatched to the + // encryption thread pool at once, so expensive KDFs (e.g. Argon2) for + // different layouts run in parallel instead of serially. On success the + // matching candidate's Header is populated and its index is returned; + // returns -1 if none match. Requires EncryptionThreadPool::IsRunning(). + static int DecryptHeaderParallel (const vector &candidates, const VolumePassword &password, int pim); void EncryptNew (const BufferPtr &newHeaderBuffer, const ConstBufferPtr &newSalt, const ConstBufferPtr &newHeaderKey, shared_ptr newPkcs5Kdf); uint64 GetEncryptedAreaStart () const { return EncryptedAreaStart; } uint64 GetEncryptedAreaLength () const { return EncryptedAreaLength; }