Checking dataset empty (microsoft#647)

you-n-g · web-flow · commit 3b11912724f9 · 2021-10-14T23:35:12.000+08:00
* Checking dataset empty

* add dataset checker
diff --git a/qlib/contrib/model/catboost_model.py b/qlib/contrib/model/catboost_model.py
@@ -38,6 +38,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
 
diff --git a/qlib/contrib/model/double_ensemble.py b/qlib/contrib/model/double_ensemble.py
@@ -64,6 +64,8 @@ def fit(self, dataset: DatasetH):
         df_train, df_valid = dataset.prepare(
             ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         x_train, y_train = df_train["feature"], df_train["label"]
         # initialize the sample weights
         N, F = x_train.shape
diff --git a/qlib/contrib/model/gbdt.py b/qlib/contrib/model/gbdt.py
@@ -25,6 +25,8 @@ def _prepare_data(self, dataset: DatasetH):
         df_train, df_valid = dataset.prepare(
             ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
 
@@ -83,6 +85,8 @@ def finetune(self, dataset: DatasetH, num_boost_round=10, verbose_eval=20):
         """
         # Based on existing model and finetune by train more rounds
         dtrain, _ = self._prepare_data(dataset)
+        if dtrain.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         self.model = lgb.train(
             self.params,
             dtrain,
diff --git a/qlib/contrib/model/highfreq_gdbt_model.py b/qlib/contrib/model/highfreq_gdbt_model.py
@@ -82,6 +82,8 @@ def _prepare_data(self, dataset: DatasetH):
         df_train, df_valid = dataset.prepare(
             ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_train["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/linear.py b/qlib/contrib/model/linear.py
@@ -51,6 +51,8 @@ def __init__(self, estimator="ols", alpha=0.0, fit_intercept=False):
 
     def fit(self, dataset: DatasetH):
         df_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        if df_train.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         X, y = df_train["feature"].values, np.squeeze(df_train["label"].values)
 
         if self.estimator in [self.OLS, self.RIDGE, self.LASSO]:
diff --git a/qlib/contrib/model/pytorch_alstm.py b/qlib/contrib/model/pytorch_alstm.py
@@ -224,6 +224,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_alstm_ts.py b/qlib/contrib/model/pytorch_alstm_ts.py
@@ -207,6 +207,8 @@ def fit(
     ):
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        if dl_train.empty or dl_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         dl_train.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
         dl_valid.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
diff --git a/qlib/contrib/model/pytorch_gats.py b/qlib/contrib/model/pytorch_gats.py
@@ -237,6 +237,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_gats_ts.py b/qlib/contrib/model/pytorch_gats_ts.py
@@ -245,6 +245,8 @@ def fit(
 
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        if dl_train.empty or dl_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         dl_train.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
         dl_valid.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
diff --git a/qlib/contrib/model/pytorch_gru.py b/qlib/contrib/model/pytorch_gru.py
@@ -224,6 +224,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_gru_ts.py b/qlib/contrib/model/pytorch_gru_ts.py
@@ -206,6 +206,8 @@ def fit(
     ):
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        if dl_train.empty or dl_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         dl_train.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
         dl_valid.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py
@@ -176,6 +176,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_localformer_ts.py b/qlib/contrib/model/pytorch_localformer_ts.py
@@ -153,6 +153,8 @@ def fit(
 
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        if dl_train.empty or dl_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         dl_train.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
         dl_valid.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
diff --git a/qlib/contrib/model/pytorch_lstm.py b/qlib/contrib/model/pytorch_lstm.py
@@ -219,6 +219,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_lstm_ts.py b/qlib/contrib/model/pytorch_lstm_ts.py
@@ -201,6 +201,8 @@ def fit(
     ):
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        if dl_train.empty or dl_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         dl_train.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
         dl_valid.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
diff --git a/qlib/contrib/model/pytorch_sfm.py b/qlib/contrib/model/pytorch_sfm.py
@@ -374,6 +374,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
 
diff --git a/qlib/contrib/model/pytorch_tabnet.py b/qlib/contrib/model/pytorch_tabnet.py
@@ -169,6 +169,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
         df_train.fillna(df_train.mean(), inplace=True)
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_tcts.py b/qlib/contrib/model/pytorch_tcts.py
@@ -255,6 +255,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_transformer.py b/qlib/contrib/model/pytorch_transformer.py
@@ -175,6 +175,8 @@ def fit(
             col_set=["feature", "label"],
             data_key=DataHandlerLP.DK_L,
         )
+        if df_train.empty or df_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
 
         x_train, y_train = df_train["feature"], df_train["label"]
         x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_transformer_ts.py b/qlib/contrib/model/pytorch_transformer_ts.py
@@ -151,6 +151,9 @@ def fit(
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
 
+        if dl_train.empty or dl_valid.empty:
+            raise ValueError("Empty data from dataset, please check your dataset config.")
+
         dl_train.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
         dl_valid.config(fillna_type="ffill+bfill")  # process nan brought by dataloader
 
diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py
@@ -385,6 +385,10 @@ def build_index(data: pd.DataFrame) -> Tuple[pd.DataFrame, dict]:
                     idx_map[real_idx] = (i, j)
         return idx_df, idx_map
 
+    @property
+    def empty(self):
+        return self.__len__() == 0
+
     def _get_indices(self, row: int, col: int) -> np.array:
         """
         get series indices of self.data_arr from the row, col indices of self.idx_df

Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,8 @@ def fit(`
`38`	`38`	`col_set=["feature", "label"],`
`39`	`39`	`data_key=DataHandlerLP.DK_L,`
`40`	`40`	`)`
	`41`	`+ if df_train.empty or df_valid.empty:`
	`42`	`+ raise ValueError("Empty data from dataset, please check your dataset config.")`
`41`	`43`	`x_train, y_train = df_train["feature"], df_train["label"]`
`42`	`44`	`x_valid, y_valid = df_valid["feature"], df_valid["label"]`
`43`	`45`
Original file line number	Diff line number	Diff line change
`@@ -64,6 +64,8 @@ def fit(self, dataset: DatasetH):`
`64`	`64`	`df_train, df_valid = dataset.prepare(`
`65`	`65`	`["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L`
`66`	`66`	`)`
	`67`	`+ if df_train.empty or df_valid.empty:`
	`68`	`+ raise ValueError("Empty data from dataset, please check your dataset config.")`
`67`	`69`	`x_train, y_train = df_train["feature"], df_train["label"]`
`68`	`70`	`# initialize the sample weights`
`69`	`71`	`N, F = x_train.shape`
Original file line number	Diff line number	Diff line change
`@@ -82,6 +82,8 @@ def _prepare_data(self, dataset: DatasetH):`
`82`	`82`	`df_train, df_valid = dataset.prepare(`
`83`	`83`	`["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L`
`84`	`84`	`)`
	`85`	`+ if df_train.empty or df_valid.empty:`
	`86`	`+ raise ValueError("Empty data from dataset, please check your dataset config.")`
`85`	`87`
`86`	`88`	`x_train, y_train = df_train["feature"], df_train["label"]`
`87`	`89`	`x_valid, y_valid = df_train["feature"], df_valid["label"]`
Original file line number	Diff line number	Diff line change
`@@ -224,6 +224,8 @@ def fit(`
`224`	`224`	`col_set=["feature", "label"],`
`225`	`225`	`data_key=DataHandlerLP.DK_L,`
`226`	`226`	`)`
	`227`	`+ if df_train.empty or df_valid.empty:`
	`228`	`+ raise ValueError("Empty data from dataset, please check your dataset config.")`
`227`	`229`
`228`	`230`	`x_train, y_train = df_train["feature"], df_train["label"]`
`229`	`231`	`x_valid, y_valid = df_valid["feature"], df_valid["label"]`
Original file line number	Diff line number	Diff line change
`@@ -237,6 +237,8 @@ def fit(`
`237`	`237`	`col_set=["feature", "label"],`
`238`	`238`	`data_key=DataHandlerLP.DK_L,`
`239`	`239`	`)`
	`240`	`+ if df_train.empty or df_valid.empty:`
	`241`	`+ raise ValueError("Empty data from dataset, please check your dataset config.")`
`240`	`242`
`241`	`243`	`x_train, y_train = df_train["feature"], df_train["label"]`
`242`	`244`	`x_valid, y_valid = df_valid["feature"], df_valid["label"]`