feature_engineering.ipynb 5.3 KB

%pip install librosa
import pandas as pd
from pycaret.classification import *

data = pd.read_csv('data/data_all.csv')
print(data)
# data.style
                 timestamp   hum   temp  door  motion  illum  label  dayofweek
0      2021-08-30 00:00:00  78.0  246.0   0.0     0.0    0.0    0.0          0
1      2021-08-30 00:01:00  78.0  246.0   0.0     0.0    0.0    0.0          0
2      2021-08-30 00:02:00  78.0  246.0   0.0     0.0    0.0    0.0          0
3      2021-08-30 00:03:00  78.0  246.0   0.0     0.0    0.0    0.0          0
4      2021-08-30 00:04:00  78.0  246.0   0.0     0.0    0.0    0.0          0
...                    ...   ...    ...   ...     ...    ...    ...        ...
64795  2021-11-26 23:55:00  26.0  191.0   0.0     0.0    1.0    0.0          4
64796  2021-11-26 23:56:00  26.0  191.0   0.0     0.0    1.0    0.0          4
64797  2021-11-26 23:57:00  26.0  191.0   0.0     0.0    1.0    0.0          4
64798  2021-11-26 23:58:00  26.0  191.0   0.0     0.0    1.0    0.0          4
64799  2021-11-26 23:59:00  26.0  191.0   0.0     0.0    1.0    0.0          4

[64800 rows x 8 columns]
# Delta Feature $\Delta_t = \frac{\sum_{n=1}^{N} n(c_{t+n} - c_{t-n})}{2\sum_{n=1}^{N} n^2}$ $n=2, N=2$ $\Delta\Delta_t = \frac{\sum_{n=1}^{N} n(\Delta_{t+n} - \Delta_{t-n})}{2\sum_{n=1}^{N} n^2}$
import librosa as fe

data.timestamp = data.timestamp.apply(lambda x: pd.Timestamp(x))
data['month'] = data.timestamp.dt.month
data['day'] = data.timestamp.dt.day
data['hour'] = data.timestamp.dt.hour
data['minute'] = data.timestamp.dt.minute
data['dayofweek'] = data.timestamp.dt.dayofweek

def f_e(df, cn='hum'):   
    # delta feature 
    df[f'd_{cn}'] = fe.feature.delta(df[f'{cn}'])
    df[f'dd_{cn}'] = fe.feature.delta(df[f'd_{cn}'], order=2)
    # time expand
    for i in range(1,10):df[f'sh{i}_{cn}'] = df[f'{cn}'].shift(-i).fillna(0)
    return df
f_e(data, 'hum')
f_e(data, 'temp')
# out = data.drop('timestamp', axis=1)
# list(data.columns)

# nomal set : not engineering
out = data[['label', 'hum', 'temp', 'door', 'motion', 'illum',  'dayofweek', 'month', 'day', 'hour', 'minute']]
out.to_csv('data/feature_ori.csv', index=False)
# out.to_csv('data/feature_ori_to_aws.csv', index=False, header=False)
print(out.columns)

# eng set : delta feature
out = data[['label', 'hum', 'd_hum', 'dd_hum', 'temp', 'd_temp', 'dd_temp', 'door', 'motion', 'illum',  'dayofweek', 'month', 'day', 'hour', 'minute']]
out.to_csv('data/feature_delta.csv', index=False)
# out.to_csv('data/feature_delta_to_aws.csv', index=False, header=False)
print(out.columns)
Index(['label', 'hum', 'temp', 'door', 'motion', 'illum', 'dayofweek', 'month',
       'day', 'hour', 'minute'],
      dtype='object')
Index(['label', 'hum', 'd_hum', 'dd_hum', 'temp', 'd_temp', 'dd_temp', 'door',
       'motion', 'illum', 'dayofweek', 'month', 'day', 'hour', 'minute'],
      dtype='object')
data.columns
Index(['timestamp', 'hum', 'temp', 'door', 'motion', 'illum', 'label',
       'dayofweek', 'month', 'hour', 'd_hum', 'dd_hum', 'sh1_hum', 'sh2_hum',
       'sh3_hum', 'sh4_hum', 'sh5_hum', 'sh6_hum', 'sh7_hum', 'sh8_hum',
       'sh9_hum', 'd_temp', 'dd_temp', 'sh1_temp', 'sh2_temp', 'sh3_temp',
       'sh4_temp', 'sh5_temp', 'sh6_temp', 'sh7_temp', 'sh8_temp', 'sh9_temp'],
      dtype='object')