Move various imports inside methods/functions to load them lazily when initializing the app

DashAI/back/converters/hugging_face/embedding.py

@@ -1,7 +1,4 @@
-import pyarrow as pa
-import torch
-from datasets import Dataset, concatenate_datasets
-from transformers import AutoModel, AutoTokenizer
+"""HuggingFace embedding converter with lazy-loaded dependencies."""
 
 from DashAI.back.converters.category.advanced_preprocessing import (
     AdvancedPreprocessingConverter,
@@ -104,16 +101,23 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float32 as the output type for embeddings."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float32())
 
     def _load_model(self):
         """Load the embedding model and tokenizer."""
+        from transformers import AutoModel, AutoTokenizer
+
         self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
         self.model = AutoModel.from_pretrained(self.model_name).to(self.device)
         self.model.eval()
 
     def _process_batch(self, batch: DashAIDataset) -> DashAIDataset:
         """Process a batch of text into embeddings."""
+        import torch
+        from datasets import Dataset, concatenate_datasets
+
         all_column_embeddings = []
 
         for column in batch.column_names:

DashAI/back/converters/hugging_face/tokenizer.py

@@ -1,6 +1,3 @@
-from datasets import Dataset, concatenate_datasets
-from transformers import AutoTokenizer
-
 from DashAI.back.converters.category.advanced_preprocessing import (
     AdvancedPreprocessingConverter,
 )
@@ -87,12 +84,16 @@ def __init__(self, **kwargs):
 
     def _load_model(self):
         """Load tokenizer only."""
+        from transformers import AutoTokenizer
+
         self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
 
     def _process_batch(self, batch: DashAIDataset) -> DashAIDataset:
         """
         Tokenize a batch of text columns and store each input_id in a separate column.
         """
+        from datasets import Dataset, concatenate_datasets
+
         all_column_tokens = []
 
         for column in batch.column_names:

DashAI/back/converters/hugging_face_wrapper.py

@@ -1,8 +1,6 @@
 from abc import ABCMeta, abstractmethod
 from typing import Type
 
-from datasets import concatenate_datasets
-
 from DashAI.back.converters.base_converter import BaseConverter
 from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset
 from DashAI.back.types.dashai_data_type import DashAIDataType
@@ -49,6 +47,8 @@ def fit(self, x: DashAIDataset, y: DashAIDataset = None) -> Type[BaseConverter]:
 
     def transform(self, x: DashAIDataset, y: DashAIDataset = None) -> DashAIDataset:
         """Transform the input data using the model."""
+        from datasets import concatenate_datasets
+
         all_results = []
 
         # Process in batches

DashAI/back/converters/imbalanced_learn/smoteenn_converter.py

@@ -1,5 +1,4 @@
 from imblearn.combine import SMOTEENN
-from imblearn.over_sampling import SMOTE
 
 from DashAI.back.converters.category.sampling import SamplingConverter
 from DashAI.back.converters.imbalanced_learn_wrapper import ImbalancedLearnWrapper
@@ -61,6 +60,8 @@ class SMOTEENNConverter(SamplingConverter, ImbalancedLearnWrapper, SMOTEENN):
     IMAGE_PREVIEW = "smoteenn.png"
 
     def __init__(self, **kwargs):
+        from imblearn.over_sampling import SMOTE
+
         self.smote = SMOTE(
             sampling_strategy=kwargs.get("sampling_strategy", "auto"),
             random_state=kwargs.get("random_state"),

DashAI/back/converters/imbalanced_learn_wrapper.py

@@ -1,10 +1,6 @@
 from abc import ABCMeta
 from typing import Type, Union
 
-import numpy as np
-import pandas as pd
-import pyarrow as pa
-
 from DashAI.back.converters.base_converter import BaseConverter
 from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset
 from DashAI.back.job.base_job import JobError
@@ -20,7 +16,7 @@ class ImbalancedLearnWrapper(BaseConverter, metaclass=ABCMeta):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.fitted = False
-        self._resampled_table: Union[pa.Table, None] = None
+        self._resampled_table = None
         self.original_X_column_names_: list = []
         self.original_target_column_name_: str = ""
 
@@ -44,6 +40,10 @@ def fit(self, x: DashAIDataset, y: DashAIDataset) -> Type[BaseConverter]:
         Fit the sampler using imbalanced-learn's fit_resample and store the combined
         result.
         """
+        import numpy as np
+        import pandas as pd
+        import pyarrow as pa
+
         if y is None or len(y) == 0:
             raise ValueError(
                 "Imbalanced-learn samplers require a non-empty target dataset (y)."

DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.kernel_approximation import (
     AdditiveChi2Sampler as AdditiveChi2SamplerOperation,
 )
@@ -57,4 +56,6 @@ class AdditiveChi2Sampler(
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for transformed data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/bag_of_words.py

@@ -1,6 +1,3 @@
-import pandas as pd
-from sklearn.feature_extraction.text import CountVectorizer
-
 from DashAI.back.converters.base_converter import BaseConverter
 from DashAI.back.converters.category.advanced_preprocessing import (
     AdvancedPreprocessingConverter,
@@ -90,6 +87,8 @@ class BagOfWordsConverter(AdvancedPreprocessingConverter, BaseConverter):
 
     def __init__(self, **kwargs):
         super().__init__()
+        from sklearn.feature_extraction.text import CountVectorizer
+
         self.vectorizer = CountVectorizer(
             max_features=kwargs.get("max_features", 1000),
             lowercase=kwargs.get("lowercase", True),
@@ -111,6 +110,8 @@ def fit(self, x: DashAIDataset, y=None) -> "BagOfWordsConverter":
 
     def transform(self, x: DashAIDataset, y=None) -> DashAIDataset:
         """Transform text into Bag-of-Words frequency columns."""
+        import pandas as pd
+
         if not self.fitted:
             raise RuntimeError("The converter must be fitted before calling transform.")
 

DashAI/back/converters/scikit_learn/binarizer.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import Binarizer as BinarizerOperation
 
 from DashAI.back.converters.category.encoding import EncodingConverter
@@ -51,4 +50,6 @@ class Binarizer(EncodingConverter, SklearnWrapper, BinarizerOperation):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Integer64 as the output type for binarized data."""
+        import pyarrow as pa
+
         return Integer(arrow_type=pa.int64())

DashAI/back/converters/scikit_learn/cca.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.cross_decomposition import CCA as CCAOPERATION
 
 from DashAI.back.converters.category.advanced_preprocessing import (
@@ -72,4 +71,6 @@ class CCA(AdvancedPreprocessingConverter, SklearnWrapper, CCAOPERATION):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for transformed data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/fast_ica.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.decomposition import FastICA as FastICAOperation
 
 from DashAI.back.api.utils import (
@@ -161,4 +160,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for transformed data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/generic_univariate_select.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.feature_selection import (
     GenericUnivariateSelect as GenericUnivariateSelectOperation,
 )
@@ -62,4 +61,6 @@ class GenericUnivariateSelect(
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for selected features."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/knn_imputer.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.impute import KNNImputer as KNNImputerOperation
 
 from DashAI.back.converters.category.basic_preprocessing import (
@@ -88,4 +87,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for imputed data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/label_binarizer.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import LabelBinarizer as LabelBinarizerOperation
 
 from DashAI.back.converters.category.encoding import EncodingConverter
@@ -44,4 +43,6 @@ class LabelBinarizer(EncodingConverter, SklearnWrapper, LabelBinarizerOperation)
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Integer64 as the output type for binarized labels."""
+        import pyarrow as pa
+
         return Integer(arrow_type=pa.int64())

DashAI/back/converters/scikit_learn/label_encoder.py

@@ -1,8 +1,5 @@
 from typing import Union
 
-import pyarrow as pa
-from sklearn.preprocessing import LabelEncoder as LabelEncoderOperation
-
 from DashAI.back.converters.category.encoding import EncodingConverter
 from DashAI.back.converters.sklearn_wrapper import SklearnWrapper
 from DashAI.back.core.schema_fields.base_schema import BaseSchema
@@ -51,6 +48,8 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType:
         If the encoder has been fitted and has classes_, use them to create
         a proper categorical type.
         """
+        import pyarrow as pa
+
         if column_name and column_name in self.encoders:
             encoder = self.encoders[column_name]
             if hasattr(encoder, "classes_"):
@@ -63,6 +62,8 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType:
 
     def fit(self, x: DashAIDataset, y: Union[DashAIDataset, None] = None):
         """Fit label encoders to each column in the dataset."""
+        from sklearn.preprocessing import LabelEncoder as LabelEncoderOperation
+
         x_pandas = x.to_pandas()
 
         for col in x_pandas.columns:

DashAI/back/converters/scikit_learn/max_abs_scaler.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import MaxAbsScaler as MaxAbsScalerOperation
 
 from DashAI.back.converters.category.scaling_and_normalization import (
@@ -39,4 +38,6 @@ class MaxAbsScaler(
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for scaled data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/min_max_scaler.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import MinMaxScaler as MinMaxScalerOperation
 
 from DashAI.back.converters.category.scaling_and_normalization import (
@@ -74,4 +73,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for scaled data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/missing_indicator.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.impute import MissingIndicator as MissingIndicatorOperation
 
 from DashAI.back.converters.category.basic_preprocessing import (
@@ -35,4 +34,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Integer64 as the output type for binary indicators."""
+        import pyarrow as pa
+
         return Integer(arrow_type=pa.int64())

DashAI/back/converters/scikit_learn/normalizer.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import Normalizer as NormalizerOperation
 
 from DashAI.back.converters.category.scaling_and_normalization import (
@@ -45,4 +44,6 @@ class Normalizer(ScalingAndNormalizationConverter, SklearnWrapper, NormalizerOpe
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for normalized data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/nystroem.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.kernel_approximation import Nystroem as NystroemOperation
 
 from DashAI.back.api.utils import create_random_state, parse_string_to_dict
@@ -133,4 +132,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for transformed data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/one_hot_encoder.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import OneHotEncoder as OneHotEncoderOperation
 
 from DashAI.back.api.utils import cast_string_to_type, parse_string_to_list
@@ -118,4 +117,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Integer64 as the output type for one-hot encoded data."""
+        import pyarrow as pa
+
         return Integer(arrow_type=pa.int64())

DashAI/back/converters/scikit_learn/ordinal_encoder.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import OrdinalEncoder as OrdinalEncoderOperation
 
 from DashAI.back.api.utils import cast_string_to_type
@@ -116,6 +115,8 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType:
         Returns Categorical type with encoded values.
         After fitting, categories are encoded as integers.
         """
+        import pyarrow as pa
+
         # Return a placeholder categorical type
         # The actual categories will be set by sklearn_wrapper's transform method
         return Categorical(values=pa.array(["0", "1"]))

DashAI/back/converters/scikit_learn/pca.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.decomposition import PCA as PCAOPERATION
 
 from DashAI.back.api.utils import create_random_state
@@ -179,4 +178,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for PCA components."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/polynomial_features.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.preprocessing import PolynomialFeatures as PolynomialFeaturesOperation
 
 from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter
@@ -94,4 +93,6 @@ class PolynomialFeatures(
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for polynomial features."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/rbf_sampler.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.kernel_approximation import RBFSampler as RBFSamplerOperation
 
 from DashAI.back.api.utils import create_random_state
@@ -80,4 +79,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for transformed data."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/select_fdr.py

@@ -1,4 +1,3 @@
-import pyarrow as pa
 from sklearn.feature_selection import SelectFdr as SelectFdrOperation
 
 from DashAI.back.converters.category.feature_selection import FeatureSelectionConverter
@@ -45,4 +44,6 @@ def __init__(self, **kwargs):
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Returns Float64 as the output type for selected features."""
+        import pyarrow as pa
+
         return Float(arrow_type=pa.float64())