Add extraction benchmarks; stop spilling entries to temp files and leaking 7z/rar decoders

.gitignore

@@ -4,3 +4,4 @@ obj/
 .vs
 RecursiveExtractor.sln.DotSettings.user
 .DS_Store
+BenchmarkDotNet.Artifacts/

README.md

@@ -303,6 +303,11 @@ foreach(var file in results)
 } 
 ```
 
+### Controlling Memory Usage
+The `Content` stream of each `FileEntry` is held in memory while it is smaller than `ExtractorOptions.MemoryStreamCutoff` (100MB by default) and backed by a temporary file once it grows past it. Entries whose size is not known before extraction begins start in memory and move to disk only if they actually exceed the cutoff.
+
+Because entries are held concurrently when `ExtractorOptions.Parallel` is set, peak memory is bounded by `BatchSize * MemoryStreamCutoff`. Lower `MemoryStreamCutoff` if you are extracting in a memory constrained environment, or raise it to trade memory for fewer temporary files.
+
 # Feedback
 
 If you have any issues or feature requests (for example, supporting other formats) you can open a new [Issue](https://github.com/microsoft/RecursiveExtractor/issues/new).  

RecursiveExtractor.Benchmarks/ArchiveFormatBenchmarks.cs

@@ -0,0 +1,129 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+using BenchmarkDotNet.Attributes;
+using SharpCompress.Archives;
+using SharpCompress.Readers;
+using System;
+using System.IO;
+
+namespace Microsoft.CST.RecursiveExtractor.Benchmarks
+{
+    /// <summary>
+    /// Extracts the same payload from every supported container so the cost attributable to the
+    /// underlying format implementation can be separated from RecursiveExtractor's own overhead.
+    /// </summary>
+    [MemoryDiagnoser]
+    public class ArchiveFormatBenchmarks
+    {
+        private readonly Extractor _extractor = new();
+        private readonly ExtractorOptions _options = new();
+
+        private byte[] _archive = Array.Empty<byte>();
+        private byte[] _copyBuffer = Array.Empty<byte>();
+        private string _fileName = string.Empty;
+
+        /// <summary>
+        /// The container and compression under test.
+        /// </summary>
+        [Params(
+            BenchmarkArchiveKind.ZipStore,
+            BenchmarkArchiveKind.ZipDeflate,
+            BenchmarkArchiveKind.Tar,
+            BenchmarkArchiveKind.TarGz,
+            BenchmarkArchiveKind.TarBz2,
+            BenchmarkArchiveKind.SevenZipLzma)]
+        public BenchmarkArchiveKind Kind { get; set; }
+
+        /// <summary>
+        /// Builds the archive under test.
+        /// </summary>
+        [GlobalSetup]
+        public void Setup()
+        {
+            _archive = SyntheticArchives.Create(Kind, EntryCount, EntrySize);
+            _fileName = SyntheticArchives.FileNameFor(Kind);
+            _copyBuffer = new byte[81920];
+        }
+
+        /// <summary>
+        /// Number of entries in each generated archive.
+        /// </summary>
+        public const int EntryCount = 200;
+
+        /// <summary>
+        /// Size in bytes of each entry in each generated archive.
+        /// </summary>
+        public const int EntrySize = 8192;
+
+        /// <summary>
+        /// SharpCompress reading the archive and discarding the bytes. Forward-only readers handle
+        /// the outer compression layer of the tar variants transparently, so they stay comparable
+        /// with what RecursiveExtractor produces; 7z has no forward-only reader and is read through
+        /// the random access archive API instead.
+        /// </summary>
+        /// <returns>Bytes decompressed.</returns>
+        [Benchmark(Baseline = true, Description = "SharpCompress reader")]
+        public long SharpCompressReader()
+        {
+            using var source = new MemoryStream(_archive, false);
+            return Kind == BenchmarkArchiveKind.SevenZipLzma
+                ? ReadWithArchiveApi(source)
+                : ReadWithReaderApi(source);
+        }
+
+        private long ReadWithReaderApi(Stream source)
+        {
+            long total = 0;
+            using var reader = ReaderFactory.OpenReader(source, new ReaderOptions { LeaveStreamOpen = true });
+            while (reader.MoveToNextEntry())
+            {
+                if (reader.Entry.IsDirectory)
+                {
+                    continue;
+                }
+
+                using var entryStream = reader.OpenEntryStream();
+                total += entryStream.CopyToCounting(Stream.Null, _copyBuffer);
+            }
+
+            return total;
+        }
+
+        private long ReadWithArchiveApi(Stream source)
+        {
+            long total = 0;
+            using var archive = ArchiveFactory.OpenArchive(source, new ReaderOptions { LeaveStreamOpen = true });
+            foreach (var entry in archive.Entries)
+            {
+                if (entry.IsDirectory)
+                {
+                    continue;
+                }
+
+                using var entryStream = entry.OpenEntryStream();
+                total += entryStream.CopyToCounting(Stream.Null, _copyBuffer);
+            }
+
+            return total;
+        }
+
+        /// <summary>
+        /// RecursiveExtractor extracting the same archive.
+        /// </summary>
+        /// <returns>Bytes extracted.</returns>
+        [Benchmark(Description = "RecursiveExtractor.Extract")]
+        public long RecursiveExtractorExtract()
+        {
+            long total = 0;
+            using var source = new MemoryStream(_archive, false);
+            var root = new FileEntry(_fileName, source, passthroughStream: true);
+            foreach (var entry in _extractor.Extract(root, _options))
+            {
+                total += entry.Content.Length;
+                entry.Content.Dispose();
+            }
+
+            return total;
+        }
+    }
+}

RecursiveExtractor.Benchmarks/BenchmarkConfig.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Reports;
+
+namespace Microsoft.CST.RecursiveExtractor.Benchmarks
+{
+    /// <summary>
+    /// Shared BenchmarkDotNet configuration.
+    /// </summary>
+    public static class BenchmarkConfig
+    {
+        /// <summary>
+        /// Builds the configuration used by every benchmark in this assembly.
+        /// </summary>
+        /// <returns>The configuration.</returns>
+        public static IConfig Create() =>
+            DefaultConfig.Instance
+                .AddDiagnoser(MemoryDiagnoser.Default)
+                .AddColumn(RankColumn.Arabic)
+                .WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(40));
+    }
+}

RecursiveExtractor.Benchmarks/ExtractToDirectoryBenchmarks.cs

@@ -0,0 +1,98 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Engines;
+using System;
+using System.IO;
+
+namespace Microsoft.CST.RecursiveExtractor.Benchmarks
+{
+    /// <summary>
+    /// Measures writing an extracted archive to disk with and without
+    /// <see cref="ExtractorOptions.Parallel"/>. Only the write side is parallelized; the
+    /// extraction enumeration itself is sequential because entries are produced by a single
+    /// forward pass over the source archive.
+    /// </summary>
+    [MemoryDiagnoser]
+    [SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 1, iterationCount: 5)]
+    public class ExtractToDirectoryBenchmarks
+    {
+        private readonly Extractor _extractor = new();
+
+        private byte[] _archive = Array.Empty<byte>();
+        private string _outputRoot = string.Empty;
+        private string _outputDirectory = string.Empty;
+
+        /// <summary>
+        /// Whether entries are written to disk in parallel.
+        /// </summary>
+        [Params(false, true)]
+        public bool Parallel { get; set; }
+
+        /// <summary>
+        /// Number of entries in the archive being written out.
+        /// </summary>
+        [Params(500)]
+        public int EntryCount { get; set; }
+
+        /// <summary>
+        /// Builds the archive and the root output folder.
+        /// </summary>
+        [GlobalSetup]
+        public void Setup()
+        {
+            _archive = SyntheticArchives.Create(BenchmarkArchiveKind.ZipDeflate, EntryCount, 8192);
+            _outputRoot = Path.Combine(Path.GetTempPath(), $"re-bench-{Guid.NewGuid():N}");
+            Directory.CreateDirectory(_outputRoot);
+        }
+
+        /// <summary>
+        /// Removes the root output folder.
+        /// </summary>
+        [GlobalCleanup]
+        public void Cleanup() => TryDelete(_outputRoot);
+
+        /// <summary>
+        /// Gives each iteration a clean output folder.
+        /// </summary>
+        [IterationSetup]
+        public void IterationSetup()
+        {
+            _outputDirectory = Path.Combine(_outputRoot, Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        /// <summary>
+        /// Deletes the folder written by the previous iteration.
+        /// </summary>
+        [IterationCleanup]
+        public void IterationCleanup() => TryDelete(_outputDirectory);
+
+        /// <summary>
+        /// Extracts the archive to disk.
+        /// </summary>
+        /// <returns>The extraction status.</returns>
+        [Benchmark(Description = "ExtractToDirectory")]
+        public ExtractionStatusCode ExtractToDirectory()
+        {
+            using var source = new MemoryStream(_archive, false);
+            var root = new FileEntry("benchmark.zip", source, passthroughStream: true);
+            return _extractor.ExtractToDirectory(_outputDirectory, root, new ExtractorOptions { Parallel = Parallel });
+        }
+
+        private static void TryDelete(string directory)
+        {
+            try
+            {
+                if (Directory.Exists(directory))
+                {
+                    Directory.Delete(directory, true);
+                }
+            }
+            catch (IOException)
+            {
+                // Best effort cleanup of temporary output.
+            }
+        }
+    }
+}

RecursiveExtractor.Benchmarks/FileTypeDetectionBenchmarks.cs

@@ -0,0 +1,80 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+using BenchmarkDotNet.Attributes;
+using System;
+using System.IO;
+
+namespace Microsoft.CST.RecursiveExtractor.Benchmarks
+{
+    /// <summary>
+    /// Measures <see cref="MiniMagic.DetectFileType(Stream)"/>, which runs once per entry during
+    /// recursion via <see cref="FileEntry.ArchiveType"/>. It seeks to the end of the stream for the
+    /// DMG footer check, so the backing store matters.
+    /// </summary>
+    [MemoryDiagnoser]
+    public class FileTypeDetectionBenchmarks
+    {
+        private MemoryStream? _plainMemory;
+        private MemoryStream? _zipMemory;
+        private FileStream? _plainFile;
+        private string _tempFile = string.Empty;
+
+        /// <summary>
+        /// Size in bytes of the content being sniffed.
+        /// </summary>
+        [Params(4096, 1048576)]
+        public int ContentSize { get; set; }
+
+        /// <summary>
+        /// Creates the streams under test.
+        /// </summary>
+        [GlobalSetup]
+        public void Setup()
+        {
+            _plainMemory = new MemoryStream(SyntheticArchives.CreatePayload(ContentSize, true), false);
+            _zipMemory = new MemoryStream(SyntheticArchives.Create(BenchmarkArchiveKind.ZipDeflate, 4, 1024), false);
+
+            _tempFile = Path.Combine(Path.GetTempPath(), $"re-bench-{Guid.NewGuid():N}.bin");
+            File.WriteAllBytes(_tempFile, SyntheticArchives.CreatePayload(ContentSize, true));
+            _plainFile = new FileStream(_tempFile, FileMode.Open, FileAccess.Read, FileShare.Read);
+        }
+
+        /// <summary>
+        /// Disposes the streams and removes the temporary file.
+        /// </summary>
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            _plainMemory?.Dispose();
+            _zipMemory?.Dispose();
+            _plainFile?.Dispose();
+
+            if (File.Exists(_tempFile))
+            {
+                File.Delete(_tempFile);
+            }
+        }
+
+        /// <summary>
+        /// Detection against an in-memory non-archive, the common case for leaf files.
+        /// </summary>
+        /// <returns>The detected type.</returns>
+        [Benchmark(Baseline = true, Description = "MemoryStream, not an archive")]
+        public ArchiveFileType MemoryStreamPlain() => MiniMagic.DetectFileType(_plainMemory!);
+
+        /// <summary>
+        /// Detection against an in-memory zip, which matches on the first four bytes.
+        /// </summary>
+        /// <returns>The detected type.</returns>
+        [Benchmark(Description = "MemoryStream, zip")]
+        public ArchiveFileType MemoryStreamZip() => MiniMagic.DetectFileType(_zipMemory!);
+
+        /// <summary>
+        /// Detection against a FileStream, which is what backs entries larger than
+        /// <see cref="ExtractorOptions.MemoryStreamCutoff"/>.
+        /// </summary>
+        /// <returns>The detected type.</returns>
+        [Benchmark(Description = "FileStream, not an archive")]
+        public ArchiveFileType FileStreamPlain() => MiniMagic.DetectFileType(_plainFile!);
+    }
+}