Prepare: Feature Cleaning

ChemML provides utilities to remove highly correlated and invariant features.

import numpy as np
import pandas as pd
from chemml.preprocessing import RemoveCorrFeatures, RemoveInvFeatures

df_corr = pd.DataFrame()
df_corr['feat_a'] = np.arange(0, 10)
df_corr['feat_b'] = np.arange(0, 10)
df_corr['feat_c'] = np.array([0, 1] * 5)
df_corr['feat_d'] = -np.arange(0, 10)
df_corr

	feat_a	feat_b	feat_c	feat_d
0	0	0	0	0
1	1	1	1	-1
2	2	2	0	-2
3	3	3	1	-3
4	4	4	0	-4
5	5	5	1	-5
6	6	6	0	-6
7	7	7	1	-7
8	8	8	0	-8
9	9	9	1	-9

Removing Highly Correlated Features

df_corr_clean = RemoveCorrFeatures(df_corr, correlation_threshold=0.9)
df_corr_clean

	feat_a	feat_c
0	0	0
1	1	1
2	2	0
3	3	1
4	4	0
5	5	1
6	6	0
7	7	1
8	8	0
9	9	1

df_inv = pd.DataFrame()
df_inv['feat_keep_1'] = np.array([0, 1] * 10)
df_inv['feat_const'] = np.ones(20)
df_inv['feat_binary_dominant'] = np.array([0] * 19 + [1])
df_inv['feat_nonbinary_dominant'] = np.array([5] * 18 + [2, 3])
df_inv['feat_low_variance'] = np.linspace(0.0, 0.0004, 20)
df_inv['feat_keep_2'] = np.arange(0, 20)
df_inv

	feat_keep_1	feat_const	feat_binary_dominant	feat_nonbinary_dominant	feat_low_variance	feat_keep_2
0	0	1.0	0	5	0.000000	0
1	1	1.0	0	5	0.000021	1
2	0	1.0	0	5	0.000042	2
3	1	1.0	0	5	0.000063	3
4	0	1.0	0	5	0.000084	4
5	1	1.0	0	5	0.000105	5
6	0	1.0	0	5	0.000126	6
7	1	1.0	0	5	0.000147	7
8	0	1.0	0	5	0.000168	8
9	1	1.0	0	5	0.000189	9
10	0	1.0	0	5	0.000211	10
11	1	1.0	0	5	0.000232	11
12	0	1.0	0	5	0.000253	12
13	1	1.0	0	5	0.000274	13
14	0	1.0	0	5	0.000295	14
15	1	1.0	0	5	0.000316	15
16	0	1.0	0	5	0.000337	16
17	1	1.0	0	5	0.000358	17
18	0	1.0	0	2	0.000379	18
19	1	1.0	1	3	0.000400	19

Removing Invariant and Low-Variance Features

df_inv_clean = RemoveInvFeatures(
    df_inv,
    sanitize_threshold=0.9,
    sanitize_nonbinary=True,
    use_variance_filtering=True,
    variance_threshold=0.01,
    keep_filtered_columns=False
)
df_inv_clean

	feat_keep_1	feat_keep_2
0	0	0
1	1	1
2	0	2
3	1	3
4	0	4
5	1	5
6	0	6
7	1	7
8	0	8
9	1	9
10	0	10
11	1	11
12	0	12
13	1	13
14	0	14
15	1	15
16	0	16
17	1	17
18	0	18
19	1	19

Keeping Removed Columns in a Separate DataFrame

df_inv_clean, df_removed = RemoveInvFeatures(
    df_inv,
    sanitize_threshold=0.9,
    sanitize_nonbinary=True,
    use_variance_filtering=True,
    variance_threshold=0.01,
    keep_filtered_columns=True
)
df_inv_clean

	feat_keep_1	feat_keep_2
0	0	0
1	1	1
2	0	2
3	1	3
4	0	4
5	1	5
6	0	6
7	1	7
8	0	8
9	1	9
10	0	10
11	1	11
12	0	12
13	1	13
14	0	14
15	1	15
16	0	16
17	1	17
18	0	18
19	1	19

df_removed

	feat_const	feat_binary_dominant	feat_nonbinary_dominant	feat_low_variance
0	1.0	0	5	0.000000
1	1.0	0	5	0.000021
2	1.0	0	5	0.000042
3	1.0	0	5	0.000063
4	1.0	0	5	0.000084
5	1.0	0	5	0.000105
6	1.0	0	5	0.000126
7	1.0	0	5	0.000147
8	1.0	0	5	0.000168
9	1.0	0	5	0.000189
10	1.0	0	5	0.000211
11	1.0	0	5	0.000232
12	1.0	0	5	0.000253
13	1.0	0	5	0.000274
14	1.0	0	5	0.000295
15	1.0	0	5	0.000316
16	1.0	0	5	0.000337
17	1.0	0	5	0.000358
18	1.0	0	2	0.000379
19	1.0	1	3	0.000400