Fix performance of tarfile reading with "r|*"

Author: TomiBelan

Lib/tarfile.py

@@ -382,7 +382,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize,
                 except ImportError:
                     raise CompressionError("bz2 module is not available") from None
                 if mode == "r":
-                    self.dbuf = b""
                     self.cmp = bz2.BZ2Decompressor()
                     self.exception = OSError
                 else:
@@ -394,7 +393,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize,
                 except ImportError:
                     raise CompressionError("lzma module is not available") from None
                 if mode == "r":
-                    self.dbuf = b""
                     self.cmp = lzma.LZMADecompressor()
                     self.exception = lzma.LZMAError
                 else:
@@ -475,7 +473,6 @@ def _init_read_gz(self):
         """Initialize for reading a gzip compressed fileobj.
         """
         self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
-        self.dbuf = b""
 
         # taken from gzip.GzipFile with some alterations
         if self.__read(2) != b"\037\213":
@@ -533,26 +530,31 @@ def _read(self, size):
         if self.comptype == "tar":
             return self.__read(size)
 
-        c = len(self.dbuf)
-        t = [self.dbuf]
+        c = 0
+        t = []
         while c < size:
             # Skip underlying buffer to avoid unaligned double buffering.
             if self.buf:
                 buf = self.buf
                 self.buf = b""
+            elif self.comptype != "gz" and not self.cmp.needs_input:
+                buf = b""
             else:
                 buf = self.fileobj.read(self.bufsize)
                 if not buf:
                     break
             try:
-                buf = self.cmp.decompress(buf)
+                buf = self.cmp.decompress(buf, size - c)
+                if self.comptype == "gz":
+                    self.buf = self.cmp.unconsumed_tail
             except self.exception as e:
                 raise ReadError("invalid compressed data") from e
             t.append(buf)
             c += len(buf)
         t = b"".join(t)
-        self.dbuf = t[size:]
-        return t[:size]
+        if len(t) > size:
+            raise ReadError("decompress() returned too much data")
+        return t
 
     def __read(self, size):
         """Return size bytes from stream. If internal buffer is empty,

Misc/ACKS

@@ -141,6 +141,7 @@ Bas van Beek
 Ian Beer
 Stefan Behnel
 Reimer Behrends
+Tomi Belan
 Maxime Bélanger
 Ben Bell
 Thomas Bellman

Misc/NEWS.d/next/Library/2024-07-02-20-57-43.gh-issue-121109.Tp6R2s.rst

@@ -0,0 +1,2 @@
+Fix :mod:`tarfile` performance issue when reading archives in streaming mode
+(e.g. ``r|*``).