From e47eb8775340ea9ab28554a78c84fde065cb0e6c Mon Sep 17 00:00:00 2001 From: ty <123@wisdri.com> Date: Wed, 8 Jul 2026 14:46:38 +0800 Subject: [PATCH] fix: resolve some type annotation issues in scripts directory --- scripts/data_collector/baostock_5min/collector.py | 8 ++++---- scripts/data_collector/base.py | 14 +++++++------- scripts/data_collector/br_index/collector.py | 7 ++++--- .../contrib/fill_cn_1min_data/fill_cn_1min_data.py | 5 +++-- .../future_trading_date_collector.py | 4 ++-- scripts/data_collector/crypto/collector.py | 12 +++++++----- scripts/data_collector/fund/collector.py | 9 +++++---- .../data_collector/future_calendar_collector.py | 2 +- scripts/data_collector/index.py | 4 ++-- scripts/data_collector/pit/collector.py | 2 +- scripts/data_collector/us_index/collector.py | 8 ++++---- scripts/data_collector/utils.py | 12 ++++++------ scripts/data_collector/yahoo/collector.py | 14 +++++++------- scripts/dump_bin.py | 10 +++++----- scripts/dump_pit.py | 2 +- 15 files changed, 59 insertions(+), 54 deletions(-) diff --git a/scripts/data_collector/baostock_5min/collector.py b/scripts/data_collector/baostock_5min/collector.py index 0a69beefb82..5b2b92475dd 100644 --- a/scripts/data_collector/baostock_5min/collector.py +++ b/scripts/data_collector/baostock_5min/collector.py @@ -11,7 +11,7 @@ from tqdm import tqdm from pathlib import Path from loguru import logger -from typing import Iterable, List +from typing import Iterable, List, Union import qlib from qlib.data import D @@ -26,7 +26,7 @@ class BaostockCollectorHS3005min(BaseCollector): def __init__( self, - save_dir: [str, Path], + save_dir: Union[str, Path], start=None, end=None, interval="5min", @@ -151,7 +151,7 @@ class BaostockNormalizeHS3005min(BaseNormalize): PM_RANGE = ("13:00:00", "14:59:00") def __init__( - self, qlib_data_1d_dir: [str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs + self, qlib_data_1d_dir: Union[str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs ): """ @@ -267,7 +267,7 @@ def normalize_class_name(self): return f"BaostockNormalize{self.region.upper()}{self.interval}" @property - def default_base_dir(self) -> [Path, str]: + def default_base_dir(self) -> Union[Path, str]: return CUR_DIR def download_data( diff --git a/scripts/data_collector/base.py b/scripts/data_collector/base.py index 672e593fa5a..94a8b66498e 100644 --- a/scripts/data_collector/base.py +++ b/scripts/data_collector/base.py @@ -7,7 +7,7 @@ import datetime import importlib from pathlib import Path -from typing import Type, Iterable +from typing import Type, Iterable, Union from concurrent.futures import ProcessPoolExecutor import pandas as pd @@ -31,7 +31,7 @@ class BaseCollector(abc.ABC): def __init__( self, - save_dir: [str, Path], + save_dir: Union[str, Path], start=None, end=None, interval="1d", @@ -85,14 +85,14 @@ def __init__( except Exception as e: logger.warning(f"Cannot use limit_nums={limit_nums}, the parameter will be ignored") - def normalize_start_datetime(self, start_datetime: [str, pd.Timestamp] = None): + def normalize_start_datetime(self, start_datetime: Union[str, pd.Timestamp] = None): return ( pd.Timestamp(str(start_datetime)) if start_datetime else getattr(self, f"DEFAULT_START_DATETIME_{self.interval.upper()}") ) - def normalize_end_datetime(self, end_datetime: [str, pd.Timestamp] = None): + def normalize_end_datetime(self, end_datetime: Union[str, pd.Timestamp] = None): return ( pd.Timestamp(str(end_datetime)) if end_datetime @@ -246,8 +246,8 @@ def _get_calendar_list(self) -> Iterable[pd.Timestamp]: class Normalize: def __init__( self, - source_dir: [str, Path], - target_dir: [str, Path], + source_dir: Union[str, Path], + target_dir: Union[str, Path], normalize_class: Type[BaseNormalize], max_workers: int = 16, date_field_name: str = "date", @@ -373,7 +373,7 @@ def normalize_class_name(self): @property @abc.abstractmethod - def default_base_dir(self) -> [Path, str]: + def default_base_dir(self) -> Union[Path, str]: raise NotImplementedError("rewrite default_base_dir") def download_data( diff --git a/scripts/data_collector/br_index/collector.py b/scripts/data_collector/br_index/collector.py index 04b2f96d9fb..d6768ee1da5 100644 --- a/scripts/data_collector/br_index/collector.py +++ b/scripts/data_collector/br_index/collector.py @@ -4,6 +4,7 @@ import sys from pathlib import Path import datetime +from typing import Union, Optional import fire import pandas as pd @@ -26,7 +27,7 @@ class IBOVIndex(IndexBase): def __init__( self, index_name: str, - qlib_dir: [str, Path] = None, + qlib_dir: Union[str, Path] = None, freq: str = "day", request_retry: int = 5, retry_sleep: int = 3, @@ -35,7 +36,7 @@ def __init__( index_name=index_name, qlib_dir=qlib_dir, freq=freq, request_retry=request_retry, retry_sleep=retry_sleep ) - self.today: datetime = datetime.date.today() + self.today: datetime.date = datetime.date.today() self.current_4_month_period = self.get_current_4_month_period(self.today.month) self.year = str(self.today.year) self.years_4_month_periods = self.get_four_month_period() @@ -276,7 +277,7 @@ def get_new_companies(self): except Exception as E: logger.error("An error occured while getting new companies - {}".format(E)) - def filter_df(self, df: pd.DataFrame) -> pd.DataFrame: + def filter_df(self, df: pd.DataFrame) -> Optional[pd.DataFrame]: if "Código" in df.columns: return df.loc[:, ["Código"]].copy() diff --git a/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py b/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py index 0a721298d3c..63a926cc0ab 100644 --- a/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py +++ b/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from concurrent.futures import ThreadPoolExecutor +from typing import Union import fire import qlib @@ -40,8 +41,8 @@ def get_symbols(data_1min_dir: Path): def fill_1min_using_1d( - data_1min_dir: [str, Path], - qlib_data_1d_dir: [str, Path], + data_1min_dir: Union[str, Path], + qlib_data_1d_dir: Union[str, Path], max_workers: int = 16, date_field_name: str = "date", symbol_field_name: str = "symbol", diff --git a/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py b/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py index 939ba7f6ad0..101ddfaa109 100644 --- a/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py +++ b/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import sys -from typing import List +from typing import List, Union from pathlib import Path import fire @@ -45,7 +45,7 @@ def generate_qlib_calendar(date_list: List[str], freq: str) -> List[str]: raise ValueError(f"Unsupported freq: {freq}") -def future_calendar_collector(qlib_dir: [str, Path], freq: str = "day"): +def future_calendar_collector(qlib_dir: Union[str, Path], freq: str = "day"): """get future calendar Parameters diff --git a/scripts/data_collector/crypto/collector.py b/scripts/data_collector/crypto/collector.py index d936fb9d979..686fee1cabd 100644 --- a/scripts/data_collector/crypto/collector.py +++ b/scripts/data_collector/crypto/collector.py @@ -3,11 +3,13 @@ import datetime from abc import ABC from pathlib import Path +from typing import Union, Optional import fire import pandas as pd from loguru import logger from dateutil.tz import tzlocal +from pandas import DataFrame CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent)) @@ -22,7 +24,7 @@ _CG_CRYPTO_SYMBOLS = None -def get_cg_crypto_symbols(qlib_data_path: [str, Path] = None) -> list: +def get_cg_crypto_symbols(qlib_data_path: Union[str, Path] = None) -> list: """get crypto symbols in coingecko Returns @@ -56,7 +58,7 @@ def _get_coingecko(): class CryptoCollector(BaseCollector): def __init__( self, - save_dir: [str, Path], + save_dir: Union[str, Path], start=None, end=None, interval="1d", @@ -115,7 +117,7 @@ def init_datetime(self): self.end_datetime = self.convert_datetime(self.end_datetime, self._timezone) @staticmethod - def convert_datetime(dt: [pd.Timestamp, datetime.date, str], timezone): + def convert_datetime(dt: Union[pd.Timestamp, datetime.date, str], timezone): try: dt = pd.Timestamp(dt, tz=timezone).timestamp() dt = pd.Timestamp(dt, tz=tzlocal(), unit="s") @@ -150,7 +152,7 @@ def get_data_from_remote(symbol, interval, start, end): def get_data( self, symbol: str, interval: str, start_datetime: pd.Timestamp, end_datetime: pd.Timestamp - ) -> [pd.DataFrame]: + ) -> Optional[DataFrame]: def _get_simple(start_, end_): self.sleep() _remote_interval = interval @@ -249,7 +251,7 @@ def normalize_class_name(self): return f"CryptoNormalize{self.interval}" @property - def default_base_dir(self) -> [Path, str]: + def default_base_dir(self) -> Union[Path, str]: return CUR_DIR def download_data( diff --git a/scripts/data_collector/fund/collector.py b/scripts/data_collector/fund/collector.py index 937d3931dbb..102525ac582 100644 --- a/scripts/data_collector/fund/collector.py +++ b/scripts/data_collector/fund/collector.py @@ -7,6 +7,7 @@ import json from abc import ABC from pathlib import Path +from typing import Union, Optional import fire import requests @@ -26,7 +27,7 @@ class FundCollector(BaseCollector): def __init__( self, - save_dir: [str, Path], + save_dir: Union[str, Path], start=None, end=None, interval="1d", @@ -85,7 +86,7 @@ def init_datetime(self): self.end_datetime = self.convert_datetime(self.end_datetime, self._timezone) @staticmethod - def convert_datetime(dt: [pd.Timestamp, datetime.date, str], timezone): + def convert_datetime(dt: Union[pd.Timestamp, datetime.date, str], timezone): try: dt = pd.Timestamp(dt, tz=timezone).timestamp() dt = pd.Timestamp(dt, tz=tzlocal(), unit="s") @@ -129,7 +130,7 @@ def get_data_from_remote(symbol, interval, start, end): def get_data( self, symbol: str, interval: str, start_datetime: pd.Timestamp, end_datetime: pd.Timestamp - ) -> [pd.DataFrame]: + ) -> Optional[pd.DataFrame]: def _get_simple(start_, end_): self.sleep() _remote_interval = interval @@ -244,7 +245,7 @@ def normalize_class_name(self): return f"FundNormalize{self.region.upper()}{self.interval}" @property - def default_base_dir(self) -> [Path, str]: + def default_base_dir(self) -> Union[Path, str]: return CUR_DIR def download_data( diff --git a/scripts/data_collector/future_calendar_collector.py b/scripts/data_collector/future_calendar_collector.py index 4dfd24e4b7b..c9305e65bd6 100644 --- a/scripts/data_collector/future_calendar_collector.py +++ b/scripts/data_collector/future_calendar_collector.py @@ -48,7 +48,7 @@ def calendar_list(self) -> List[pd.Timestamp]: calendar_df["date"] = pd.to_datetime(calendar_df["date"]) return calendar_df["date"].to_list() - def _format_datetime(self, datetime_d: [str, pd.Timestamp]): + def _format_datetime(self, datetime_d: Union[str, pd.Timestamp]): datetime_d = pd.Timestamp(datetime_d) return datetime_d.strftime(self.calendar_format) diff --git a/scripts/data_collector/index.py b/scripts/data_collector/index.py index caaaee53af5..5a559355fa3 100644 --- a/scripts/data_collector/index.py +++ b/scripts/data_collector/index.py @@ -1,7 +1,7 @@ import sys import abc from pathlib import Path -from typing import List +from typing import List, Union import pandas as pd from tqdm import tqdm @@ -29,7 +29,7 @@ class IndexBase: def __init__( self, index_name: str, - qlib_dir: [str, Path] = None, + qlib_dir: Union[str, Path] = None, freq: str = "day", request_retry: int = 5, retry_sleep: int = 3, diff --git a/scripts/data_collector/pit/collector.py b/scripts/data_collector/pit/collector.py index c34b31348dc..8fefdae8fc1 100644 --- a/scripts/data_collector/pit/collector.py +++ b/scripts/data_collector/pit/collector.py @@ -252,7 +252,7 @@ def normalize_class_name(self) -> str: return f"PitNormalize" @property - def default_base_dir(self) -> [Path, str]: + def default_base_dir(self) -> Union[Path, str]: return BASE_DIR diff --git a/scripts/data_collector/us_index/collector.py b/scripts/data_collector/us_index/collector.py index 9d0e43717b9..e9819c1046a 100644 --- a/scripts/data_collector/us_index/collector.py +++ b/scripts/data_collector/us_index/collector.py @@ -6,7 +6,7 @@ import sys from pathlib import Path from concurrent.futures import ThreadPoolExecutor -from typing import List +from typing import List, Union, Optional from io import StringIO import fire @@ -41,7 +41,7 @@ class WIKIIndex(IndexBase): def __init__( self, index_name: str, - qlib_dir: [str, Path] = None, + qlib_dir: Union[str, Path] = None, freq: str = "day", request_retry: int = 5, retry_sleep: int = 3, @@ -149,7 +149,7 @@ class NASDAQ100Index(WIKIIndex): ) MAX_WORKERS = 16 - def filter_df(self, df: pd.DataFrame) -> pd.DataFrame: + def filter_df(self, df: pd.DataFrame) -> Optional[pd.DataFrame]: if len(df) >= 100 and "Ticker" in df.columns: return df.loc[:, ["Ticker"]].copy() @@ -208,7 +208,7 @@ def bench_start_date(self) -> pd.Timestamp: def get_changes(self) -> pd.DataFrame: pass - def filter_df(self, df: pd.DataFrame) -> pd.DataFrame: + def filter_df(self, df: pd.DataFrame) -> Optional[pd.DataFrame]: if "Symbol" in df.columns: _df = df.loc[:, ["Symbol"]].copy() _df["Symbol"] = _df["Symbol"].apply(lambda x: x.split(":")[-1]) diff --git a/scripts/data_collector/utils.py b/scripts/data_collector/utils.py index b4e6708b1a6..2a23a70bef4 100644 --- a/scripts/data_collector/utils.py +++ b/scripts/data_collector/utils.py @@ -10,7 +10,7 @@ import requests import functools from pathlib import Path -from typing import Iterable, Tuple, List +from typing import Iterable, Tuple, List, Union import numpy as np import pandas as pd @@ -111,7 +111,7 @@ def return_date_list(date_field_name: str, file_path: Path): def get_calendar_list_by_ratio( - source_dir: [str, Path], + source_dir: Union[str, Path], date_field_name: str = "date", threshold: float = 0.5, minimum_count: int = 10, @@ -286,7 +286,7 @@ def _get_symbol(): return _HS_SYMBOLS -def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list: +def get_us_stock_symbols(qlib_data_path: Union[str, Path] = None) -> list: """get US stock symbols Returns @@ -367,7 +367,7 @@ def _format(s_): return _US_SYMBOLS -def get_in_stock_symbols(qlib_data_path: [str, Path] = None) -> list: +def get_in_stock_symbols(qlib_data_path: Union[str, Path] = None) -> list: """get IN stock symbols Returns @@ -408,7 +408,7 @@ def _format(s_): return _IN_SYMBOLS -def get_br_stock_symbols(qlib_data_path: [str, Path] = None) -> list: +def get_br_stock_symbols(qlib_data_path: Union[str, Path] = None) -> list: """get Brazil(B3) stock symbols Returns @@ -459,7 +459,7 @@ def _format(s_): return _BR_SYMBOLS -def get_en_fund_symbols(qlib_data_path: [str, Path] = None) -> list: +def get_en_fund_symbols(qlib_data_path: Union[str, Path] = None) -> list: """get en fund symbols Returns diff --git a/scripts/data_collector/yahoo/collector.py b/scripts/data_collector/yahoo/collector.py index 82660f1112b..98ab3798684 100644 --- a/scripts/data_collector/yahoo/collector.py +++ b/scripts/data_collector/yahoo/collector.py @@ -10,7 +10,7 @@ from abc import ABC import multiprocessing from pathlib import Path -from typing import Iterable +from typing import Iterable, Union import fire import requests @@ -50,7 +50,7 @@ class YahooCollector(BaseCollector): def __init__( self, - save_dir: [str, Path], + save_dir: Union[str, Path], start=None, end=None, interval="1d", @@ -109,7 +109,7 @@ def init_datetime(self): self.end_datetime = self.convert_datetime(self.end_datetime, self._timezone) @staticmethod - def convert_datetime(dt: [pd.Timestamp, datetime.date, str], timezone): + def convert_datetime(dt: Union[pd.Timestamp, datetime.date, str], timezone): try: dt = pd.Timestamp(dt, tz=timezone).timestamp() dt = pd.Timestamp(dt, tz=tzlocal(), unit="s") @@ -509,7 +509,7 @@ def _manual_adj_data(self, df: pd.DataFrame) -> pd.DataFrame: class YahooNormalize1dExtend(YahooNormalize1d): def __init__( - self, old_qlib_data_dir: [str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs + self, old_qlib_data_dir: Union[str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs ): """ @@ -526,7 +526,7 @@ def __init__( self.column_list = ["open", "high", "low", "close", "volume", "factor", "change"] self.old_qlib_data = self._get_old_data(old_qlib_data_dir) - def _get_old_data(self, qlib_data_dir: [str, Path]): + def _get_old_data(self, qlib_data_dir: Union[str, Path]): qlib_data_dir = str(Path(qlib_data_dir).expanduser().resolve()) qlib.init(provider_uri=qlib_data_dir, expression_cache=None, dataset_cache=None) df = D.features(D.instruments("all"), ["$" + col for col in self.column_list]) @@ -564,7 +564,7 @@ class YahooNormalize1min(YahooNormalize, ABC): CALC_PAUSED_NUM = True def __init__( - self, qlib_data_1d_dir: [str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs + self, qlib_data_1d_dir: Union[str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs ): """ @@ -749,7 +749,7 @@ def normalize_class_name(self): return f"YahooNormalize{self.region.upper()}{self.interval}" @property - def default_base_dir(self) -> [Path, str]: + def default_base_dir(self) -> Union[Path, str]: return CUR_DIR def download_data( diff --git a/scripts/dump_bin.py b/scripts/dump_bin.py index cb8ed72dabb..96ff38d080e 100644 --- a/scripts/dump_bin.py +++ b/scripts/dump_bin.py @@ -5,7 +5,7 @@ import shutil import traceback from pathlib import Path -from typing import Iterable, List, Union +from typing import Iterable, List, Union, Tuple from functools import partial from concurrent.futures import ThreadPoolExecutor, as_completed, ProcessPoolExecutor @@ -141,13 +141,13 @@ def __init__( def _backup_qlib_dir(self, target_dir: Path): shutil.copytree(str(self.qlib_dir.resolve()), str(target_dir.resolve())) - def _format_datetime(self, datetime_d: [str, pd.Timestamp]): + def _format_datetime(self, datetime_d: Union[str, pd.Timestamp]): datetime_d = pd.Timestamp(datetime_d) return datetime_d.strftime(self.calendar_format) def _get_date( - self, file_or_df: [Path, pd.DataFrame], *, is_begin_end: bool = False, as_set: bool = False - ) -> Iterable[pd.Timestamp]: + self, file_or_df: Union[Path, pd.DataFrame], *, is_begin_end: bool = False, as_set: bool = False + ) -> Iterable[pd.Timestamp] | Tuple[Tuple[pd.Timestamp, pd.Timestamp], Iterable[pd.Timestamp]]: if not isinstance(file_or_df, pd.DataFrame): df = self._get_source_data(file_or_df) else: @@ -268,7 +268,7 @@ def _data_to_bin(self, df: pd.DataFrame, calendar_list: List[pd.Timestamp], feat # append; self._mode == self.ALL_MODE or not bin_path.exists() np.hstack([date_index, _df[field]]).astype("