encoding: improve UTF-8 decode strategy

Author: ralian

pygit2/__init__.py

@@ -365,7 +365,7 @@
 from .settings import Settings
 from .submodules import Submodule
 from .transaction import ReferenceTransaction
-from .utils import to_bytes, to_str
+from .utils import decode_fs_path, encode_fs_path, to_bytes, to_str
 
 # Features
 features = enums.Feature(C.git_libgit2_features())
@@ -430,15 +430,15 @@ def init_repository(
     options.mode = mode
 
     if workdir_path:
-        workdir_path_ref = ffi.new('char []', to_bytes(workdir_path))
+        workdir_path_ref = ffi.new('char []', encode_fs_path(workdir_path))
         options.workdir_path = workdir_path_ref
 
     if description:
         description_ref = ffi.new('char []', to_bytes(description))
         options.description = description_ref
 
     if template_path:
-        template_path_ref = ffi.new('char []', to_bytes(template_path))
+        template_path_ref = ffi.new('char []', encode_fs_path(template_path))
         options.template_path = template_path_ref
 
     if initial_head:
@@ -451,7 +451,7 @@ def init_repository(
 
     # Call
     crepository = ffi.new('git_repository **')
-    err = C.git_repository_init_ext(crepository, to_bytes(path), options)
+    err = C.git_repository_init_ext(crepository, encode_fs_path(path), options)
     check_error(err)
 
     # Ok
@@ -536,7 +536,7 @@ def clone_repository(
         with git_fetch_options(payload, opts=opts.fetch_opts):
             with git_proxy_options(payload, opts.fetch_opts.proxy_opts, proxy):
                 crepo = ffi.new('git_repository **')
-                err = C.git_clone(crepo, to_bytes(url), to_bytes(path), opts)
+                err = C.git_clone(crepo, to_bytes(url), encode_fs_path(path), opts)
                 payload.check_error(err)
 
     # Ok

pygit2/blame.py

@@ -30,7 +30,7 @@
 
 # Import from pygit2
 from .ffi import C, ffi
-from .utils import GenericIterator
+from .utils import GenericIterator, maybe_string
 
 if TYPE_CHECKING:
     from ._libgit2.ffi import GitBlameC, GitHunkC, GitSignatureC
@@ -110,7 +110,7 @@ def orig_path(self) -> None | str:
         if not path:
             return None
 
-        return ffi.string(path).decode('utf-8')
+        return maybe_string(path)
 
 
 class Blame:

pygit2/callbacks.py

@@ -74,7 +74,14 @@
 from .enums import CheckoutNotify, CheckoutStrategy, CredentialType, StashApplyProgress
 from .errors import Passthrough, check_error
 from .ffi import C, ffi
-from .utils import StrArray, maybe_string, ptr_to_bytes, to_bytes
+from .utils import (
+    StrArray,
+    decode_fs_path,
+    encode_fs_path,
+    maybe_string,
+    ptr_to_bytes,
+    to_bytes,
+)
 
 _Credentials = Username | UserPass | Keypair
 
@@ -784,13 +791,13 @@ def get_credentials(fn, url, username, allowed):
 def _checkout_notify_cb(
     why, path_cstr, baseline, target, workdir, data: CheckoutCallbacks
 ):
-    pypath = maybe_string(path_cstr)
+    pypath = decode_fs_path(path_cstr)
     pybaseline = DiffFile.from_c(ptr_to_bytes(baseline))
     pytarget = DiffFile.from_c(ptr_to_bytes(target))
     pyworkdir = DiffFile.from_c(ptr_to_bytes(workdir))
 
     try:
-        data.checkout_notify(why, pypath, pybaseline, pytarget, pyworkdir)  # type: ignore[arg-type]
+        data.checkout_notify(why, pypath, pybaseline, pytarget, pyworkdir)
     except Passthrough:
         # Unlike most other operations with optional callbacks, checkout
         # doesn't support the GIT_PASSTHROUGH return code, so we must bypass
@@ -805,7 +812,7 @@ def _checkout_notify_cb(
 
 @libgit2_callback_void
 def _checkout_progress_cb(path, completed_steps, total_steps, data: CheckoutCallbacks):
-    data.checkout_progress(maybe_string(path), completed_steps, total_steps)  # type: ignore[arg-type]
+    data.checkout_progress(decode_fs_path(path), completed_steps, total_steps)
 
 
 def _git_checkout_options(
@@ -839,7 +846,7 @@ def _git_checkout_options(
     opts.checkout_strategy = int(strategy)
 
     if directory:
-        target_dir = ffi.new('char[]', to_bytes(directory))
+        target_dir = ffi.new('char[]', encode_fs_path(directory))
         refs.append(target_dir)
         opts.target_directory = target_dir
 

pygit2/config.py

@@ -35,7 +35,7 @@
 # Import from pygit2
 from .errors import check_error
 from .ffi import C, ffi
-from .utils import to_bytes
+from .utils import encode_fs_path, to_bytes
 
 if TYPE_CHECKING:
     from ._libgit2.ffi import GitConfigC, GitConfigEntryC
@@ -116,7 +116,7 @@ def __init__(self, path: PathLike | str | None = None) -> None:
         if not path:
             err = C.git_config_new(cconfig)
         else:
-            path_bytes = to_bytes(path)
+            path_bytes = encode_fs_path(path)
             err = C.git_config_open_ondisk(cconfig, path_bytes)
 
         check_error(err, io=True)
@@ -282,7 +282,7 @@ def add_file(self, path: str | PathLike, level: int = 0, force: int = 0) -> None
         """Add a config file instance to an existing config."""
 
         err = C.git_config_add_file_ondisk(
-            self._config, to_bytes(path), level, ffi.NULL, force
+            self._config, encode_fs_path(path), level, ffi.NULL, force
         )
         check_error(err)
 

pygit2/filter.py

@@ -32,7 +32,7 @@
 from ._pygit2 import Blob, FilterSource
 from .errors import check_error
 from .ffi import C, ffi
-from .utils import to_bytes
+from .utils import encode_fs_path, to_bytes
 
 if TYPE_CHECKING:
     from ._libgit2.ffi import GitFilterListC
@@ -178,7 +178,7 @@ def apply_to_file(self, repo: BaseRepository, path: str) -> bytes:
         Return the filtered contents.
         """
         buf = ffi.new('git_buf *')
-        c_path = to_bytes(path)
+        c_path = encode_fs_path(path)
         err = C.git_filter_list_apply_to_file(buf, self._pointer, repo._repo, c_path)
         check_error(err)
         try:

pygit2/index.py

@@ -33,7 +33,7 @@
 from .enums import DiffOption, FileMode
 from .errors import check_error
 from .ffi import C, ffi
-from .utils import GenericIterator, StrArray, to_bytes, to_str
+from .utils import GenericIterator, StrArray, decode_fs_path, encode_fs_path
 
 if typing.TYPE_CHECKING:
     from .repository import Repository
@@ -52,7 +52,7 @@ def __init__(self, path: str | PathLike[str] | None = None) -> None:
         to read from and write to.
         """
         cindex = ffi.new('git_index **')
-        err = C.git_index_open(cindex, to_bytes(path))
+        err = C.git_index_open(cindex, encode_fs_path(path))
         check_error(err)
 
         self._repo = None
@@ -79,7 +79,7 @@ def __len__(self) -> int:
         return C.git_index_entrycount(self._index)
 
     def __contains__(self, path) -> bool:
-        err = C.git_index_find(ffi.NULL, self._index, to_bytes(path))
+        err = C.git_index_find(ffi.NULL, self._index, encode_fs_path(path))
         if err == C.GIT_ENOTFOUND:
             return False
 
@@ -89,7 +89,7 @@ def __contains__(self, path) -> bool:
     def __getitem__(self, key: str | int | PathLike[str]) -> 'IndexEntry':
         centry = ffi.NULL
         if isinstance(key, str) or hasattr(key, '__fspath__'):
-            centry = C.git_index_get_bypath(self._index, to_bytes(key), 0)
+            centry = C.git_index_get_bypath(self._index, encode_fs_path(key), 0)
         elif isinstance(key, int):
             if key >= 0:
                 centry = C.git_index_get_byindex(self._index, key)
@@ -180,12 +180,12 @@ def write_tree(self, repo: 'Repository | None' = None) -> Oid:
 
     def remove(self, path: PathLike[str] | str, level: int = 0) -> None:
         """Remove an entry from the Index."""
-        err = C.git_index_remove(self._index, to_bytes(path), level)
+        err = C.git_index_remove(self._index, encode_fs_path(path), level)
         check_error(err, io=True)
 
     def remove_directory(self, path: PathLike[str] | str, level: int = 0) -> None:
         """Remove a directory from the Index."""
-        err = C.git_index_remove_directory(self._index, to_bytes(path), level)
+        err = C.git_index_remove_directory(self._index, encode_fs_path(path), level)
         check_error(err, io=True)
 
     def remove_all(self, pathspecs: typing.Sequence[str | PathLike[str]]) -> None:
@@ -221,7 +221,7 @@ def add(self, path_or_entry: 'IndexEntry | str | PathLike[str]') -> None:
             err = C.git_index_add(self._index, centry)
         elif isinstance(path_or_entry, str) or hasattr(path_or_entry, '__fspath__'):
             path = path_or_entry
-            err = C.git_index_add_bypath(self._index, to_bytes(path))
+            err = C.git_index_add_bypath(self._index, encode_fs_path(path))
         else:
             raise TypeError('argument must be string, Path or IndexEntry')
 
@@ -420,7 +420,7 @@ def _from_c(cls, centry):
             return None
 
         automergeable = centry.automergeable != 0
-        path = to_str(ffi.string(centry.path)) if centry.path else None
+        path = decode_fs_path(ffi.string(centry.path)) if centry.path else None
         mode = FileMode(centry.mode)
         contents = ffi.string(centry.ptr, centry.len).decode('utf-8')
 
@@ -475,7 +475,7 @@ def _to_c(self) -> tuple['ffi.GitIndexEntryC', 'ffi.ArrayC[ffi.char]']:
         # basically memcpy()
         ffi.buffer(ffi.addressof(centry, 'id'))[:] = self.id.raw[:]
         centry.mode = int(self.mode)
-        path = ffi.new('char[]', to_bytes(self.path))
+        path = ffi.new('char[]', encode_fs_path(self.path))
         centry.path = path
 
         return centry, path
@@ -486,7 +486,7 @@ def _from_c(cls, centry):
             return None
 
         entry = cls.__new__(cls)
-        entry.path = to_str(ffi.string(centry.path))
+        entry.path = decode_fs_path(ffi.string(centry.path))
         entry.mode = FileMode(centry.mode)
         entry.id = Oid(raw=bytes(ffi.buffer(ffi.addressof(centry, 'id'))[:]))
 
@@ -503,7 +503,7 @@ def __getitem__(self, path):
         ctheirs = ffi.new('git_index_entry **')
 
         err = C.git_index_conflict_get(
-            cancestor, cours, ctheirs, self._index._index, to_bytes(path)
+            cancestor, cours, ctheirs, self._index._index, encode_fs_path(path)
         )
         check_error(err)
 
@@ -514,7 +514,7 @@ def __getitem__(self, path):
         return ancestor, ours, theirs
 
     def __delitem__(self, path):
-        err = C.git_index_conflict_remove(self._index._index, to_bytes(path))
+        err = C.git_index_conflict_remove(self._index._index, encode_fs_path(path))
         check_error(err)
 
     def __iter__(self):
@@ -526,7 +526,7 @@ def __contains__(self, path):
         ctheirs = ffi.new('git_index_entry **')
 
         err = C.git_index_conflict_get(
-            cancestor, cours, ctheirs, self._index._index, to_bytes(path)
+            cancestor, cours, ctheirs, self._index._index, encode_fs_path(path)
         )
         if err == C.GIT_ENOTFOUND:
             return False

pygit2/options.py

@@ -34,7 +34,7 @@
 
 from .errors import check_error
 from .ffi import C, ffi
-from .utils import to_bytes, to_str
+from .utils import decode_fs_path, encode_fs_path, to_bytes, to_str
 
 if TYPE_CHECKING:
     from ._libgit2.ffi import NULL_TYPE, ArrayC, char, char_pointer
@@ -348,7 +348,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
 
         try:
             if buf.ptr != ffi.NULL:
-                result = to_str(ffi.string(buf.ptr))
+                result = decode_fs_path(ffi.string(buf.ptr))
             else:
                 result = None
         finally:
@@ -366,7 +366,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
         if path is None:
             path_cdata = ffi.NULL
         else:
-            path_bytes = to_bytes(path)
+            path_bytes = encode_fs_path(path)
             path_cdata = ffi.new('char[]', path_bytes)
 
         err = C.git_libgit2_opts(option_type, ffi.cast('int', level), path_cdata)
@@ -423,14 +423,14 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
         if cert_file is None:
             cert_file_cdata = ffi.NULL
         else:
-            cert_file_bytes = to_bytes(cert_file)
+            cert_file_bytes = encode_fs_path(cert_file)
             cert_file_cdata = ffi.new('char[]', cert_file_bytes)
 
         cert_dir_cdata: ArrayC[char] | NULL_TYPE
         if cert_dir is None:
             cert_dir_cdata = ffi.NULL
         else:
-            cert_dir_bytes = to_bytes(cert_dir)
+            cert_dir_bytes = encode_fs_path(cert_dir)
             cert_dir_cdata = ffi.new('char[]', cert_dir_bytes)
 
         err = C.git_libgit2_opts(option_type, cert_file_cdata, cert_dir_cdata)
@@ -476,7 +476,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
 
         try:
             if buf.ptr != ffi.NULL:
-                result = to_str(ffi.string(buf.ptr))
+                result = decode_fs_path(ffi.string(buf.ptr))
             else:
                 result = None
         finally:
@@ -492,7 +492,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
         if path is None:
             template_path_cdata = ffi.NULL
         else:
-            path_bytes = to_bytes(path)
+            path_bytes = encode_fs_path(path)
             template_path_cdata = ffi.new('char[]', path_bytes)
 
         err = C.git_libgit2_opts(option_type, template_path_cdata)
@@ -688,7 +688,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
 
         try:
             if buf.ptr != ffi.NULL:
-                result = to_str(ffi.string(buf.ptr))
+                result = decode_fs_path(ffi.string(buf.ptr))
             else:
                 result = None
         finally:
@@ -705,7 +705,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED)
         if path is None:
             homedir_cdata = ffi.NULL
         else:
-            path_bytes = to_bytes(path)
+            path_bytes = encode_fs_path(path)
             homedir_cdata = ffi.new('char[]', path_bytes)
 
         err = C.git_libgit2_opts(option_type, homedir_cdata)

pygit2/packbuilder.py

@@ -29,7 +29,7 @@
 # Import from pygit2
 from .errors import check_error
 from .ffi import C, ffi
-from .utils import to_bytes
+from .utils import encode_fs_path
 
 if TYPE_CHECKING:
     from pygit2 import Oid, Repository
@@ -76,7 +76,7 @@ def set_threads(self, n_threads: int) -> int:
         return C.git_packbuilder_set_threads(self._packbuilder, n_threads)
 
     def write(self, path: str | bytes | PathLike[str] | None = None) -> None:
-        path_bytes = ffi.NULL if path is None else to_bytes(path)
+        path_bytes = ffi.NULL if path is None else encode_fs_path(path)
         err = C.git_packbuilder_write(
             self._packbuilder, path_bytes, 0, ffi.NULL, ffi.NULL
         )

pygit2/repository.py

@@ -81,7 +81,7 @@
 from .remotes import RemoteCollection
 from .submodules import SubmoduleCollection
 from .transaction import ReferenceTransaction
-from .utils import StrArray, maybe_string, to_bytes
+from .utils import StrArray, decode_fs_path, encode_fs_path, maybe_string, to_bytes
 
 if TYPE_CHECKING:
     from pygit2._libgit2.ffi import (
@@ -219,7 +219,7 @@ def hashfile(
             If this is `None` and the `path` parameter is a file within the
             repository's working directory, then the `path` will be used.
         """
-        c_path = to_bytes(path)
+        c_path = encode_fs_path(path)
 
         c_as_path: ffi.NULL_TYPE | bytes
         if as_path is None:
@@ -1824,7 +1824,7 @@ def __init__(
             if hasattr(path, '__fspath__'):
                 path = path.__fspath__()
             if not isinstance(path, str):
-                path = path.decode('utf-8')
+                path = decode_fs_path(path)
             path_backend = init_file_backend(path, int(flags))
             super().__init__(path_backend)
         else: