From 84fb50c9baf02fd57c3747132b25f6873df643dc Mon Sep 17 00:00:00 2001 From: sbnair Date: Wed, 8 Jul 2026 18:13:23 +0530 Subject: [PATCH] fix: resolve pylint W1514 (unspecified-encoding) and re-enable it in lint Add explicit encoding="utf-8" to all text-mode open() calls in qlib/, scripts/ and example notebooks, so file reads/writes no longer depend on the platform's locale encoding (e.g. cp936/GBK on Chinese Windows, where reading UTF-8 YAML configs or calendar files could fail or corrupt text). - qlib/utils/file.py: get_io_object() now defaults to UTF-8 only when the file is opened in text mode without an explicit encoding; binary mode and caller-provided encodings are left untouched. - qlib/data/storage/file_storage.py: _write_calendar() keeps its binary mode (np.savetxt already encodes via its own encoding argument) with an explanatory pragma. - Makefile: remove W1514 from the pylint/nbqa disable lists so CI enforces the rule from now on. make pylint now scores 10.00/10 on qlib and scripts, and nbqa reports no W1514 in notebooks. Contributes to #1007 --- Makefile | 6 +++--- examples/benchmarks/TRA/Reports.ipynb | 2 +- qlib/__init__.py | 2 +- qlib/cli/run.py | 4 ++-- qlib/contrib/model/pytorch_tra.py | 2 +- qlib/contrib/rolling/base.py | 2 +- qlib/data/storage/file_storage.py | 8 +++++--- qlib/rl/contrib/naive_config_parser.py | 2 +- qlib/rl/contrib/train_onpolicy.py | 2 +- qlib/utils/__init__.py | 2 +- qlib/utils/file.py | 6 +++++- 11 files changed, 22 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 40c80431875..e8c376de0e6 100644 --- a/Makefile +++ b/Makefile @@ -148,8 +148,8 @@ black: # We use sys.setrecursionlimit(2000) to make the recursion depth larger to ensure that pylint works properly (the default recursion depth is 1000). # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 pylint: - pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,W4904,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1730,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}' qlib --init-hook="import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)" - pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,E1123,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0246,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}' scripts --init-hook="import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)" + pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W4904,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1730,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}' qlib --init-hook="import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)" + pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,R1720,W0105,W0123,W0201,W0511,W0613,W1113,E0401,E1121,E1123,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0246,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}' scripts --init-hook="import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)" # Check code with flake8. # The following flake8 error codes were ignored: @@ -183,7 +183,7 @@ mypy: # Check ipynb with nbqa. nbqa: nbqa black . -l 120 --check --diff - nbqa pylint . --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136,W0719,W0104,W0404,C0412,W0611,C0410 --const-rgx='[a-z_][a-z0-9_]{2,30}' + nbqa pylint . --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136,W0719,W0104,W0404,C0412,W0611,C0410 --const-rgx='[a-z_][a-z0-9_]{2,30}' # Check ipynb with nbconvert.(Run after data downloads) # TODO: Add more ipynb files in future diff --git a/examples/benchmarks/TRA/Reports.ipynb b/examples/benchmarks/TRA/Reports.ipynb index bd1534433a6..002551256a1 100644 --- a/examples/benchmarks/TRA/Reports.ipynb +++ b/examples/benchmarks/TRA/Reports.ipynb @@ -685,7 +685,7 @@ "for k, v in exps.items():\n", " tmp = dict()\n", " for fname in v:\n", - " with open(fname) as f:\n", + " with open(fname, encoding=\"utf-8\") as f:\n", " info = json.load(f)\n", " tmp[fname] = {\"IC\": info[\"metric\"][\"IC\"], \"MSE\": info[\"metric\"][\"MSE\"]}\n", " tmp = pd.DataFrame(tmp).T\n", diff --git a/qlib/__init__.py b/qlib/__init__.py index b43dd6fc42a..bf2c9723b40 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -194,7 +194,7 @@ def init_from_yaml_conf(conf_path, **kwargs): if conf_path is None: config = {} else: - with open(conf_path) as f: + with open(conf_path, encoding="utf-8") as f: yaml = YAML(typ="safe", pure=True) config = yaml.load(f) config.update(kwargs) diff --git a/qlib/cli/run.py b/qlib/cli/run.py index c2dcdc2ef0c..f5594b98138 100644 --- a/qlib/cli/run.py +++ b/qlib/cli/run.py @@ -63,7 +63,7 @@ def render_template(config_path: str) -> str: str the rendered content """ - with open(config_path, "r") as f: + with open(config_path, "r", encoding="utf-8") as f: config = f.read() # Set up the Jinja2 environment template = Template(config) @@ -126,7 +126,7 @@ def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"): else: raise FileNotFoundError(f"Can't find the BASE_CONFIG file: {base_config_path}") - with open(path) as fp: + with open(path, encoding="utf-8") as fp: yaml = YAML(typ="safe", pure=True) base_config = yaml.load(fp) logger.info(f"Load BASE_CONFIG_PATH succeed: {path.resolve()}") diff --git a/qlib/contrib/model/pytorch_tra.py b/qlib/contrib/model/pytorch_tra.py index bc9a6aa9779..ac4e25a88c8 100644 --- a/qlib/contrib/model/pytorch_tra.py +++ b/qlib/contrib/model/pytorch_tra.py @@ -493,7 +493,7 @@ def fit(self, dataset, evals_result=dict()): "best_eval_metric": -best_score, # NOTE: -1 for minimize "metrics": {"train": train_metrics, "valid": valid_metrics, "test": test_metrics}, } - with open(self.logdir + "/info.json", "w") as f: + with open(self.logdir + "/info.json", "w", encoding="utf-8") as f: json.dump(info, f) def predict(self, dataset, segment="test"): diff --git a/qlib/contrib/rolling/base.py b/qlib/contrib/rolling/base.py index 5f17c05623d..16b30a1a443 100644 --- a/qlib/contrib/rolling/base.py +++ b/qlib/contrib/rolling/base.py @@ -116,7 +116,7 @@ def __init__( # - So we have to design a priority mechanism to solve this issue. def _raw_conf(self) -> dict: - with self.conf_path.open("r") as f: + with self.conf_path.open("r", encoding="utf-8") as f: yaml = YAML(typ="safe", pure=True) return yaml.load(f) diff --git a/qlib/data/storage/file_storage.py b/qlib/data/storage/file_storage.py index e2bc5c3679a..e2a096a834c 100644 --- a/qlib/data/storage/file_storage.py +++ b/qlib/data/storage/file_storage.py @@ -111,7 +111,7 @@ def _read_calendar(self) -> List[CalVT]: if not self.uri.exists(): self._write_calendar(values=[]) - with self.uri.open("r") as fp: + with self.uri.open("r", encoding="utf-8") as fp: res = [] for line in fp.readlines(): line = line.strip() @@ -120,7 +120,9 @@ def _read_calendar(self) -> List[CalVT]: return res def _write_calendar(self, values: Iterable[CalVT], mode: str = "wb"): - with self.uri.open(mode=mode) as fp: + # `mode` is always a binary mode ("wb"/"ab"), so no `encoding` is needed; + # `np.savetxt` encodes the text itself via its own `encoding` argument. + with self.uri.open(mode=mode) as fp: # pylint: disable=unspecified-encoding np.savetxt(fp, values, fmt="%s", encoding="utf-8") @property @@ -219,7 +221,7 @@ def _read_instrument(self) -> Dict[InstKT, InstVT]: def _write_instrument(self, data: Dict[InstKT, InstVT] = None) -> None: if not data: - with self.uri.open("w") as _: + with self.uri.open("w", encoding="utf-8") as _: pass return diff --git a/qlib/rl/contrib/naive_config_parser.py b/qlib/rl/contrib/naive_config_parser.py index 3b8d3912f28..d816cfcaa06 100644 --- a/qlib/rl/contrib/naive_config_parser.py +++ b/qlib/rl/contrib/naive_config_parser.py @@ -54,7 +54,7 @@ def parse_backtest_config(path: str) -> dict: del sys.modules[tmp_module_name] else: - with open(tmp_config_file.name) as input_stream: + with open(tmp_config_file.name, encoding="utf-8") as input_stream: yaml = YAML(typ="safe", pure=True) config = yaml.load(input_stream) diff --git a/qlib/rl/contrib/train_onpolicy.py b/qlib/rl/contrib/train_onpolicy.py index 83dd924103f..420ac3be5f1 100644 --- a/qlib/rl/contrib/train_onpolicy.py +++ b/qlib/rl/contrib/train_onpolicy.py @@ -262,7 +262,7 @@ def main(config: dict, run_training: bool, run_backtest: bool) -> None: parser.add_argument("--run_backtest", action="store_true", help="Run backtest workflow.") args = parser.parse_args() - with open(args.config_path, "r") as input_stream: + with open(args.config_path, "r", encoding="utf-8") as input_stream: yaml = YAML(typ="safe", pure=True) config = yaml.load(input_stream) diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 2a94ebd555b..543e1b656a7 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -246,7 +246,7 @@ def parse_config(config): # Check whether config is file yaml = YAML(typ="safe", pure=True) if os.path.exists(config): - with open(config, "r") as f: + with open(config, "r", encoding="utf-8") as f: return yaml.load(f) # Check whether the str can be parsed try: diff --git a/qlib/utils/file.py b/qlib/utils/file.py index 1e17a574a9d..bf7e825ab3e 100644 --- a/qlib/utils/file.py +++ b/qlib/utils/file.py @@ -186,5 +186,9 @@ def get_io_object(file: Union[IO, str, Path], *args, **kwargs) -> IO: file = Path(file) if not isinstance(file, Path): raise NotImplementedError(f"This type[{type(file)}] of input is not supported") - with file.open(*args, **kwargs) as f: + # Default to UTF-8 when the file is opened in text mode without an explicit encoding, + # so the behavior does not depend on the platform's locale (e.g. GBK on Chinese Windows). + if "b" not in (args[0] if args else kwargs.get("mode", "r")): + kwargs.setdefault("encoding", "utf-8") + with file.open(*args, **kwargs) as f: # pylint: disable=unspecified-encoding yield f