Pipelined bnk extraction

BnkExtractor/BnkExtr/BnkParser.cs

@@ -38,9 +38,95 @@ public static string CreateOutputDirectory(string bnk_filename)
         internal static void Parse(string bnkFilepath, bool swapByteOrder, bool noDirectory, bool dumpObjectsTxt)
         {
             using var bnkReader = new BinaryReader(File.OpenRead(bnkFilepath));
+            var files = new List<Index>();
+            var outputDirectory = Path.GetDirectoryName(bnkFilepath);
 
-            long dataOffset = 0;
+            if (!noDirectory)
+            {
+                outputDirectory = CreateOutputDirectory(bnkFilepath);
+            }
+
+            long dataOffset = CoreParse(bnkReader, outputDirectory, swapByteOrder, dumpObjectsTxt, ref files);
+
+            // Extract WEM files
+            if (dataOffset == 0U || files.Count == 0)
+            {
+                Logger.LogError("No WEM files discovered to be extracted");
+                return;
+            }
+
+            Logger.LogVerbose($"Found {files.Count} WEM files");
+            Logger.LogVerbose("Start extracting...");
+
+            foreach (Index index in files)
+            {
+                if (swapByteOrder)
+                {
+                    index.size = Swap32(index.size);
+                    index.offset = Swap32(index.offset);
+                }
+
+                bnkReader.BaseStream.Position = dataOffset + index.offset;
+                byte[] data = bnkReader.ReadBytes((int)index.size);
+                string wemFilepath = Path.Combine(outputDirectory, $"{index.id}.wem");
+                File.WriteAllBytes(wemFilepath, data);
+                //Logger.LogVerbose(wem_filename);
+            }
+
+            Logger.LogVerbose($"Files were extracted to: {outputDirectory}");
+        }
+
+        /// <summary>
+        /// Extracts a Wwise *.BNK File to memory
+        /// </summary>
+        /// <param name="bnkFilepath">The path to the bnk file to test</param>
+        /// <param name="swapByteOrder">Swap byte order (use it for unpacking 'Army of Two')</param>
+        internal static Dictionary<uint, MemoryStream> ParseToMemory(string bnkFilepath, bool swapByteOrder, bool noDirectory, bool dumpObjectsTxt)
+        {
+            using var bnkReader = new BinaryReader(File.OpenRead(bnkFilepath));
             var files = new List<Index>();
+            var outputDirectory = Path.GetDirectoryName(bnkFilepath);
+
+            if (!noDirectory)
+            {
+                outputDirectory = CreateOutputDirectory(bnkFilepath);
+            }
+
+            long dataOffset = CoreParse(bnkReader, outputDirectory, swapByteOrder, dumpObjectsTxt, ref files);
+            var wemDataMap = new Dictionary<uint, MemoryStream>();
+
+            // Extract WEM files
+            if (dataOffset == 0U || files.Count == 0)
+            {
+                Logger.LogError("No WEM files discovered to be extracted");
+                return wemDataMap;
+            }
+
+            Logger.LogVerbose($"Found {files.Count} WEM files");
+            Logger.LogVerbose("Start extracting...");
+
+            foreach (Index index in files)
+            {
+                if (swapByteOrder)
+                {
+                    index.size = Swap32(index.size);
+                    index.offset = Swap32(index.offset);
+                }
+
+                bnkReader.BaseStream.Position = dataOffset + index.offset;
+                byte[] data = bnkReader.ReadBytes((int)index.size);
+                wemDataMap.Add(index.id, new MemoryStream(data));
+                //Logger.LogVerbose(wem_filename);
+            }
+
+            Logger.LogVerbose($"Files were extracted to memory");
+            return wemDataMap;
+        }
+
+
+        private static long CoreParse(BinaryReader bnkReader, string outputDirectory, bool swapByteOrder, bool dumpObjectsTxt, ref List<Index> files)
+        {
+            long dataOffset = 0;
             var contentSection = new Section();
             var bankHeader = new BankHeader();
             var objects = new List<Object>();
@@ -230,13 +316,6 @@ internal static void Parse(string bnkFilepath, bool swapByteOrder, bool noDirect
             // Reset EOF
             bnkReader.BaseStream.Position = 0;
 
-            var outputDirectory = Path.GetDirectoryName(bnkFilepath);
-
-            if (!noDirectory)
-            {
-                outputDirectory = CreateOutputDirectory(bnkFilepath);
-            }
-
             // Dump objects information
             if (dumpObjectsTxt)
             {
@@ -280,32 +359,7 @@ internal static void Parse(string bnkFilepath, bool swapByteOrder, bool noDirect
                 Logger.LogVerbose($"Objects file was written to: {objectFilepath}");
             }
 
-            // Extract WEM files
-            if (dataOffset == 0U || files.Count == 0)
-            {
-                Logger.LogError("No WEM files discovered to be extracted");
-                return;
-            }
-
-            Logger.LogVerbose($"Found {files.Count} WEM files");
-            Logger.LogVerbose("Start extracting...");
-
-            foreach (Index index in files)
-            {
-                if (swapByteOrder)
-                {
-                    index.size = Swap32(index.size);
-                    index.offset = Swap32(index.offset);
-                }
-
-                bnkReader.BaseStream.Position = dataOffset + index.offset;
-                byte[] data = bnkReader.ReadBytes((int)index.size);
-                string wemFilepath = Path.Combine(outputDirectory, $"{index.id}.wem");
-                File.WriteAllBytes(wemFilepath, data);
-                //Logger.LogVerbose(wem_filename);
-            }
-
-            Logger.LogVerbose($"Files were extracted to: {outputDirectory}");
+            return dataOffset;
         }
     }
 }

BnkExtractor/Extractor.cs

@@ -1,4 +1,6 @@
 using BnkExtractor.Ww2ogg;
+using System;
+using System.Collections.Generic;
 using System.IO;
 
 namespace BnkExtractor
@@ -15,5 +17,75 @@ public static void ConvertWem(string filePath)
             options.CodebooksFilename = "packed_codebooks_aoTuV_603.bin";
             Ww2oggConverter.Main(options);
         }
+
+        /// <summary>
+        /// Extracts BNK, converts all WEM to Revorbed Oggs, writes to disk
+        /// </summary>
+        /// <param name="filePath">Input .bnk file path</param>
+        /// <param name="noDirectory">Optionally create subdirectory for ogg files</param>
+        public static void BnkToOgg(string filePath, bool noDirectory = false)
+        {
+            Ww2oggOptions options = new Ww2oggOptions();
+            options.CodebooksFilename = "packed_codebooks_aoTuV_603.bin";
+            Dictionary<uint, MemoryStream> wemFiles = BnkExtr.BnkParser.ParseToMemory(filePath, false, noDirectory, true);
+            Dictionary<uint, MemoryStream> oggFiles = Ww2oggConverter.Main(wemFiles, options);
+
+            string outDirectory = noDirectory
+                ? Path.GetDirectoryName(filePath)
+                : BnkExtr.BnkParser.CreateOutputDirectory(filePath);
+
+            foreach (var (key, oggFile) in oggFiles) 
+            {
+                try
+                {
+                    var outFile = Path.Combine(outDirectory, $"{key}.ogg");
+                    using var fs = new FileStream(outFile, FileMode.Create, FileAccess.Write);
+                    Revorb.RevorbSharp.Convert(oggFile, fs);
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogVerbose(ex.ToString());
+                }
+                finally
+                {
+                    oggFile.Dispose();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Extracts BNK and converts all WEM to Revorbed Oggs, keeps in memory
+        /// </summary>
+        /// <param name="filePath">Input .bnk file path</param>
+        public static Dictionary<uint, MemoryStream> BnkToOggMemory(string filePath)
+        {
+            Ww2oggOptions options = new Ww2oggOptions();
+            options.CodebooksFilename = "packed_codebooks_aoTuV_603.bin";
+            Dictionary<uint, MemoryStream> wemFiles = BnkExtr.BnkParser.ParseToMemory(filePath, false, true, false);
+            Dictionary<uint, MemoryStream> oggFiles = Ww2oggConverter.Main(wemFiles, options);
+
+            Dictionary<uint, MemoryStream> result = new Dictionary<uint, MemoryStream>();
+
+            foreach (var (key, oggStream) in oggFiles)
+            {
+                try
+                {
+                    var outStream = new MemoryStream();
+                    Revorb.RevorbSharp.Convert(oggStream, outStream, true);
+                    outStream.Position = 0;
+                    result.Add(key, outStream);
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogVerbose(ex.ToString());
+                }
+                finally
+                {
+                    oggStream.Dispose();
+                }
+            }
+
+            return result;
+        }
     }
 }

BnkExtractor/Revorb/RevorbSharp.cs

@@ -44,10 +44,11 @@ public static byte[] Convert(byte[] inputData)
         /// </summary>
         /// <param name="inputStream">A stream to read the input data from</param>
         /// <param name="outputStream">A stream to write the output data to</param>
-        public static void Convert(Stream inputStream, Stream outputStream)
+        /// <param name="leaveOpen">Leave the streams open</param>
+        public static void Convert(Stream inputStream, Stream outputStream, bool leaveOpen = false)
         {
-            using BinaryReader fi = new BinaryReader(inputStream);
-            using BinaryWriter fo = new BinaryWriter(outputStream);
+            using BinaryReader fi = new BinaryReader(inputStream, System.Text.Encoding.UTF8, leaveOpen);
+            using BinaryWriter fo = new BinaryWriter(outputStream, System.Text.Encoding.UTF8, leaveOpen);
 
             ogg_sync_state sync_in = new ogg_sync_state();
             ogg_sync_state sync_out = new ogg_sync_state();

BnkExtractor/Ww2ogg/Ww2oggConverter.cs

@@ -1,4 +1,5 @@
 using BnkExtractor.Ww2ogg.Exceptions;
+using System.Collections.Generic;
 using System.IO;
 
 namespace BnkExtractor.Ww2ogg;
@@ -64,4 +65,44 @@ internal static void Main(Ww2oggOptions opt)
             Logger.LogError(pe.ToString());
         }
     }
+
+    /// <summary>
+    /// Input wem streams are disposed after consumption
+    /// </summary>
+    internal static Dictionary<uint, MemoryStream> Main(Dictionary<uint, MemoryStream> wemStreams, Ww2oggOptions opt)
+    {
+        var outputStreams = new Dictionary<uint, MemoryStream>();
+
+        foreach (var (id, stream) in wemStreams) 
+        { 
+            try
+            {
+                MemoryStream ms = new MemoryStream();
+                using (var writer = new BinaryWriter(ms, System.Text.Encoding.UTF8, leaveOpen: true))
+                {
+                    Wwise_RIFF_Vorbis ww = new Wwise_RIFF_Vorbis(stream, opt.CodebooksFilename, opt.InlineCodebooks, opt.FullSetup, opt.ForcePacketFormat);
+                    ww.GenerateOgg(writer);
+                }
+
+                ms.Position = 0;
+                outputStreams.Add(id, ms);
+                Logger.LogVerbose($"Converted {id} to OGG in memory.");
+            }
+            catch (FileOpenException fe)
+            {
+                Logger.LogError(fe.ToString());
+            }
+            catch (ParseException pe)
+            {
+                Logger.LogError(pe.ToString());
+            }
+            finally
+            {
+                stream.Dispose();
+            }
+        }
+
+        return outputStreams;
+    }
+
 }

BnkExtractor/Ww2ogg/Wwise_RIFF_Vorbis.cs

@@ -63,6 +63,9 @@ public class Wwise_RIFF_Vorbis
     private Func<BinaryReader, ushort> read16Delegate = null;
     private Func<BinaryReader, uint> read32Delegate = null;
 
+    /// <summary>
+    /// Read from file
+    /// </summary>
     public Wwise_RIFF_Vorbis(string name, string codebooks_name, bool inline_codebooks, bool full_setup, ForcePacketFormat force_packet_format)
     {
         this._file_name = name;
@@ -75,6 +78,24 @@ public Wwise_RIFF_Vorbis(string name, string codebooks_name, bool inline_codeboo
             throw new FileOpenException(name);
         }
 
+        Configure(force_packet_format);
+    }
+
+    /// <summary>
+    /// Read from stream
+    /// </summary>
+    public Wwise_RIFF_Vorbis(Stream input, string codebooks_name, bool inline_codebooks, bool full_setup, ForcePacketFormat force_packet_format)
+    {
+        this._codebooks_name = codebooks_name;
+        this._infile = new BinaryReader(input);
+        this._inline_codebooks = inline_codebooks;
+        this._full_setup = full_setup;
+
+        Configure(force_packet_format);
+    }
+
+    private void Configure(ForcePacketFormat force_packet_format)
+    {
         _infile.seekg(0, StreamPosition.End);
         _file_size = _infile.tellg();