fix some type annotation issues in scripts directory

scripts/data_collector/baostock_5min/collector.py

@@ -11,7 +11,7 @@
 from tqdm import tqdm
 from pathlib import Path
 from loguru import logger
-from typing import Iterable, List
+from typing import Iterable, List, Union
 
 import qlib
 from qlib.data import D
@@ -26,7 +26,7 @@
 class BaostockCollectorHS3005min(BaseCollector):
     def __init__(
         self,
-        save_dir: [str, Path],
+        save_dir: Union[str, Path],
         start=None,
         end=None,
         interval="5min",
@@ -151,7 +151,7 @@ class BaostockNormalizeHS3005min(BaseNormalize):
     PM_RANGE = ("13:00:00", "14:59:00")
 
     def __init__(
-        self, qlib_data_1d_dir: [str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs
+        self, qlib_data_1d_dir: Union[str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs
     ):
         """
 
@@ -267,7 +267,7 @@ def normalize_class_name(self):
         return f"BaostockNormalize{self.region.upper()}{self.interval}"
 
     @property
-    def default_base_dir(self) -> [Path, str]:
+    def default_base_dir(self) -> Union[Path, str]:
         return CUR_DIR
 
     def download_data(

scripts/data_collector/base.py

@@ -7,7 +7,7 @@
 import datetime
 import importlib
 from pathlib import Path
-from typing import Type, Iterable
+from typing import Type, Iterable, Union
 from concurrent.futures import ProcessPoolExecutor
 
 import pandas as pd
@@ -31,7 +31,7 @@ class BaseCollector(abc.ABC):
 
     def __init__(
         self,
-        save_dir: [str, Path],
+        save_dir: Union[str, Path],
         start=None,
         end=None,
         interval="1d",
@@ -85,14 +85,14 @@ def __init__(
             except Exception as e:
                 logger.warning(f"Cannot use limit_nums={limit_nums}, the parameter will be ignored")
 
-    def normalize_start_datetime(self, start_datetime: [str, pd.Timestamp] = None):
+    def normalize_start_datetime(self, start_datetime: Union[str, pd.Timestamp] = None):
         return (
             pd.Timestamp(str(start_datetime))
             if start_datetime
             else getattr(self, f"DEFAULT_START_DATETIME_{self.interval.upper()}")
         )
 
-    def normalize_end_datetime(self, end_datetime: [str, pd.Timestamp] = None):
+    def normalize_end_datetime(self, end_datetime: Union[str, pd.Timestamp] = None):
         return (
             pd.Timestamp(str(end_datetime))
             if end_datetime
@@ -246,8 +246,8 @@ def _get_calendar_list(self) -> Iterable[pd.Timestamp]:
 class Normalize:
     def __init__(
         self,
-        source_dir: [str, Path],
-        target_dir: [str, Path],
+        source_dir: Union[str, Path],
+        target_dir: Union[str, Path],
         normalize_class: Type[BaseNormalize],
         max_workers: int = 16,
         date_field_name: str = "date",
@@ -373,7 +373,7 @@ def normalize_class_name(self):
 
     @property
     @abc.abstractmethod
-    def default_base_dir(self) -> [Path, str]:
+    def default_base_dir(self) -> Union[Path, str]:
         raise NotImplementedError("rewrite default_base_dir")
 
     def download_data(

scripts/data_collector/br_index/collector.py

@@ -4,6 +4,7 @@
 import sys
 from pathlib import Path
 import datetime
+from typing import Union, Optional
 
 import fire
 import pandas as pd
@@ -26,7 +27,7 @@ class IBOVIndex(IndexBase):
     def __init__(
         self,
         index_name: str,
-        qlib_dir: [str, Path] = None,
+        qlib_dir: Union[str, Path] = None,
         freq: str = "day",
         request_retry: int = 5,
         retry_sleep: int = 3,
@@ -35,7 +36,7 @@ def __init__(
             index_name=index_name, qlib_dir=qlib_dir, freq=freq, request_retry=request_retry, retry_sleep=retry_sleep
         )
 
-        self.today: datetime = datetime.date.today()
+        self.today: datetime.date = datetime.date.today()
         self.current_4_month_period = self.get_current_4_month_period(self.today.month)
         self.year = str(self.today.year)
         self.years_4_month_periods = self.get_four_month_period()
@@ -276,7 +277,7 @@ def get_new_companies(self):
         except Exception as E:
             logger.error("An error occured while getting new companies - {}".format(E))
 
-    def filter_df(self, df: pd.DataFrame) -> pd.DataFrame:
+    def filter_df(self, df: pd.DataFrame) -> Optional[pd.DataFrame]:
         if "Código" in df.columns:
             return df.loc[:, ["Código"]].copy()
 

scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py

@@ -4,6 +4,7 @@
 import sys
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
+from typing import Union
 
 import fire
 import qlib
@@ -40,8 +41,8 @@ def get_symbols(data_1min_dir: Path):
 
 
 def fill_1min_using_1d(
-    data_1min_dir: [str, Path],
-    qlib_data_1d_dir: [str, Path],
+    data_1min_dir: Union[str, Path],
+    qlib_data_1d_dir: Union[str, Path],
     max_workers: int = 16,
     date_field_name: str = "date",
     symbol_field_name: str = "symbol",

scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 import sys
-from typing import List
+from typing import List, Union
 from pathlib import Path
 
 import fire
@@ -45,7 +45,7 @@ def generate_qlib_calendar(date_list: List[str], freq: str) -> List[str]:
         raise ValueError(f"Unsupported freq: {freq}")
 
 
-def future_calendar_collector(qlib_dir: [str, Path], freq: str = "day"):
+def future_calendar_collector(qlib_dir: Union[str, Path], freq: str = "day"):
     """get future calendar
 
     Parameters

scripts/data_collector/crypto/collector.py

@@ -3,11 +3,13 @@
 import datetime
 from abc import ABC
 from pathlib import Path
+from typing import Union, Optional
 
 import fire
 import pandas as pd
 from loguru import logger
 from dateutil.tz import tzlocal
+from pandas import DataFrame
 
 CUR_DIR = Path(__file__).resolve().parent
 sys.path.append(str(CUR_DIR.parent.parent))
@@ -22,7 +24,7 @@
 _CG_CRYPTO_SYMBOLS = None
 
 
-def get_cg_crypto_symbols(qlib_data_path: [str, Path] = None) -> list:
+def get_cg_crypto_symbols(qlib_data_path: Union[str, Path] = None) -> list:
     """get crypto symbols in coingecko
 
     Returns
@@ -56,7 +58,7 @@ def _get_coingecko():
 class CryptoCollector(BaseCollector):
     def __init__(
         self,
-        save_dir: [str, Path],
+        save_dir: Union[str, Path],
         start=None,
         end=None,
         interval="1d",
@@ -115,7 +117,7 @@ def init_datetime(self):
         self.end_datetime = self.convert_datetime(self.end_datetime, self._timezone)
 
     @staticmethod
-    def convert_datetime(dt: [pd.Timestamp, datetime.date, str], timezone):
+    def convert_datetime(dt: Union[pd.Timestamp, datetime.date, str], timezone):
         try:
             dt = pd.Timestamp(dt, tz=timezone).timestamp()
             dt = pd.Timestamp(dt, tz=tzlocal(), unit="s")
@@ -150,7 +152,7 @@ def get_data_from_remote(symbol, interval, start, end):
 
     def get_data(
         self, symbol: str, interval: str, start_datetime: pd.Timestamp, end_datetime: pd.Timestamp
-    ) -> [pd.DataFrame]:
+    ) -> Optional[DataFrame]:
         def _get_simple(start_, end_):
             self.sleep()
             _remote_interval = interval
@@ -249,7 +251,7 @@ def normalize_class_name(self):
         return f"CryptoNormalize{self.interval}"
 
     @property
-    def default_base_dir(self) -> [Path, str]:
+    def default_base_dir(self) -> Union[Path, str]:
         return CUR_DIR
 
     def download_data(

scripts/data_collector/fund/collector.py

@@ -7,6 +7,7 @@
 import json
 from abc import ABC
 from pathlib import Path
+from typing import Union, Optional
 
 import fire
 import requests
@@ -26,7 +27,7 @@
 class FundCollector(BaseCollector):
     def __init__(
         self,
-        save_dir: [str, Path],
+        save_dir: Union[str, Path],
         start=None,
         end=None,
         interval="1d",
@@ -85,7 +86,7 @@ def init_datetime(self):
         self.end_datetime = self.convert_datetime(self.end_datetime, self._timezone)
 
     @staticmethod
-    def convert_datetime(dt: [pd.Timestamp, datetime.date, str], timezone):
+    def convert_datetime(dt: Union[pd.Timestamp, datetime.date, str], timezone):
         try:
             dt = pd.Timestamp(dt, tz=timezone).timestamp()
             dt = pd.Timestamp(dt, tz=tzlocal(), unit="s")
@@ -129,7 +130,7 @@ def get_data_from_remote(symbol, interval, start, end):
 
     def get_data(
         self, symbol: str, interval: str, start_datetime: pd.Timestamp, end_datetime: pd.Timestamp
-    ) -> [pd.DataFrame]:
+    ) -> Optional[pd.DataFrame]:
         def _get_simple(start_, end_):
             self.sleep()
             _remote_interval = interval
@@ -244,7 +245,7 @@ def normalize_class_name(self):
         return f"FundNormalize{self.region.upper()}{self.interval}"
 
     @property
-    def default_base_dir(self) -> [Path, str]:
+    def default_base_dir(self) -> Union[Path, str]:
         return CUR_DIR
 
     def download_data(

scripts/data_collector/future_calendar_collector.py

@@ -48,7 +48,7 @@ def calendar_list(self) -> List[pd.Timestamp]:
         calendar_df["date"] = pd.to_datetime(calendar_df["date"])
         return calendar_df["date"].to_list()
 
-    def _format_datetime(self, datetime_d: [str, pd.Timestamp]):
+    def _format_datetime(self, datetime_d: Union[str, pd.Timestamp]):
         datetime_d = pd.Timestamp(datetime_d)
         return datetime_d.strftime(self.calendar_format)
 

scripts/data_collector/index.py

@@ -1,7 +1,7 @@
 import sys
 import abc
 from pathlib import Path
-from typing import List
+from typing import List, Union
 
 import pandas as pd
 from tqdm import tqdm
@@ -29,7 +29,7 @@ class IndexBase:
     def __init__(
         self,
         index_name: str,
-        qlib_dir: [str, Path] = None,
+        qlib_dir: Union[str, Path] = None,
         freq: str = "day",
         request_retry: int = 5,
         retry_sleep: int = 3,

scripts/data_collector/pit/collector.py

@@ -252,7 +252,7 @@ def normalize_class_name(self) -> str:
         return f"PitNormalize"
 
     @property
-    def default_base_dir(self) -> [Path, str]:
+    def default_base_dir(self) -> Union[Path, str]:
         return BASE_DIR
 
 

scripts/data_collector/us_index/collector.py

@@ -6,7 +6,7 @@
 import sys
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
-from typing import List
+from typing import List, Union, Optional
 from io import StringIO
 
 import fire
@@ -41,7 +41,7 @@ class WIKIIndex(IndexBase):
     def __init__(
         self,
         index_name: str,
-        qlib_dir: [str, Path] = None,
+        qlib_dir: Union[str, Path] = None,
         freq: str = "day",
         request_retry: int = 5,
         retry_sleep: int = 3,
@@ -149,7 +149,7 @@ class NASDAQ100Index(WIKIIndex):
     )
     MAX_WORKERS = 16
 
-    def filter_df(self, df: pd.DataFrame) -> pd.DataFrame:
+    def filter_df(self, df: pd.DataFrame) -> Optional[pd.DataFrame]:
         if len(df) >= 100 and "Ticker" in df.columns:
             return df.loc[:, ["Ticker"]].copy()
 
@@ -208,7 +208,7 @@ def bench_start_date(self) -> pd.Timestamp:
     def get_changes(self) -> pd.DataFrame:
         pass
 
-    def filter_df(self, df: pd.DataFrame) -> pd.DataFrame:
+    def filter_df(self, df: pd.DataFrame) -> Optional[pd.DataFrame]:
         if "Symbol" in df.columns:
             _df = df.loc[:, ["Symbol"]].copy()
             _df["Symbol"] = _df["Symbol"].apply(lambda x: x.split(":")[-1])

scripts/data_collector/utils.py

@@ -10,7 +10,7 @@
 import requests
 import functools
 from pathlib import Path
-from typing import Iterable, Tuple, List
+from typing import Iterable, Tuple, List, Union
 
 import numpy as np
 import pandas as pd
@@ -111,7 +111,7 @@ def return_date_list(date_field_name: str, file_path: Path):
 
 
 def get_calendar_list_by_ratio(
-    source_dir: [str, Path],
+    source_dir: Union[str, Path],
     date_field_name: str = "date",
     threshold: float = 0.5,
     minimum_count: int = 10,
@@ -286,7 +286,7 @@ def _get_symbol():
     return _HS_SYMBOLS
 
 
-def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
+def get_us_stock_symbols(qlib_data_path: Union[str, Path] = None) -> list:
     """get US stock symbols
 
     Returns
@@ -367,7 +367,7 @@ def _format(s_):
     return _US_SYMBOLS
 
 
-def get_in_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
+def get_in_stock_symbols(qlib_data_path: Union[str, Path] = None) -> list:
     """get IN stock symbols
 
     Returns
@@ -408,7 +408,7 @@ def _format(s_):
     return _IN_SYMBOLS
 
 
-def get_br_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
+def get_br_stock_symbols(qlib_data_path: Union[str, Path] = None) -> list:
     """get Brazil(B3) stock symbols
 
     Returns
@@ -459,7 +459,7 @@ def _format(s_):
     return _BR_SYMBOLS
 
 
-def get_en_fund_symbols(qlib_data_path: [str, Path] = None) -> list:
+def get_en_fund_symbols(qlib_data_path: Union[str, Path] = None) -> list:
     """get en fund symbols
 
     Returns