import awswrangler as wr
import pandas as pd
import numpy as np
import boto3
# from sagemaker import get_execution_role
import datetime
import string
import random
# role = get_execution_role()
def preproc_common(fn, id):
tmp = pd.read_csv(fn)
tmp['timestamp'] = pd.to_datetime(tmp.timestamp)
tmp.set_index('timestamp', inplace=True)
return tmp[tmp.device_id == id].copy()
def preproc_th(fn='temperature_humidity.csv', id='00158d00028d93d8'):
# Temperature, Humidity preprocessing
df_temp_hum = preproc_common(fn, id)
df_temp_hum = df_temp_hum.resample(rule='T').median()
return df_temp_hum[['device_id', 'hum', 'temp']].fillna(method='ffill').fillna(method='bfill')
def preproc_il(fn='illumination.csv', id='00158d0006c9d5ed'):
# illumination preprocessing
df_illumination = preproc_common(fn, id)
df_illumination.rename(columns={'evt':'illum'}, inplace=True)
df_illumination = df_illumination.resample(rule='T').median()
return df_illumination[['device_id', 'illum']].fillna(method='ffill').fillna(method='bfill')
def preproc_motion(fn='motion_door.csv', id='00158d0002d545b4'):
# motion, door preprocessing
df_motion = preproc_common(fn, id)
df_motion.rename(columns={'evt':'motion'}, inplace=True)
df_motion = df_motion.resample(rule='T').count()
return df_motion['device_id', 'motion'].fillna(value=0)
def preproc_door(fn='motion_door.csv', id='00158d0005bb96f3'):
# motion, door preprocessing
df_door = preproc_common(fn, id)
df_door.rename(columns={'evt':'door'}, inplace=True)
df_door = df_door.resample(rule='T').count()
return df_door['device_id', 'door'].fillna(value=0)
# def preproc_all():
# # [온습도, 문열림, 모션, 조도]
# rnd_data = {'00158d0006c9d545':'mr_temperature_humidity_0104.csv',
# '00158d0005bb968a':'mr_door_0104.csv',
# '00158d0002cab225':'mr_motion_0104.csv',
# '00158d0005a9998b':'mr_illumination_0104.csv'}
# m3_data = {'00158d00028d93d8':'m3_temperature_humidity_0104.csv',
# '00158d0005bb96f3':'m3_door_0104.csv',
# '00158d0002d545b4':'m3_motion_0104.csv',
# '00158d0006c9d5ed':'m3_illumination_0104.csv'}
# func = [preporc_il, preproc_th, preproc_motion, preproc_door]
df = preproc_th('mr_temperature_humidity_0104.csv', '00158d0006c9d545')
df = df.join(preproc_door('mr_door_0104.csv', '00158d0005bb968a'))
df = df.join(preproc_motion('mr_motion_0104.csv', '00158d0002cab225'))
df = df.join(preproc_il('mr_illumination_0104.csv', '00158d0005a9998b'))
df_rnd = df.copy()
df_rnd.to_csv('mr_0104.csv')
df = preproc_th('m3_temperature_humidity_0104.csv', '00158d00028d93d8')
df = df.join(preproc_door('m3_door_0104.csv', '00158d0005bb96f3'))
df = df.join(preproc_motion('m3_motion_0104.csv', '00158d0002d545b4'))
df = df.join(preproc_il('m3_illumination_0104.csv', '00158d0006c9d5ed'))
df_m3 = df.copy()
df_m3.to_csv('m3_0104.csv')
# read and merge label
df_l1 = pd.read_csv('20210830_080618A.csv')
df_l2 = pd.read_csv('20211025_080015A.csv')
df_label = pd.concat([df_l1, df_l2], ignore_index=True)
df_label.rename(columns={df_label.columns[1]:'timestamp', df_label.columns[2]:'label'}, inplace=True)
df_label['timestamp'] = pd.to_datetime(df_label.timestamp)
# filtering
df_label = df_label.loc[ (df_label['timestamp'] > '2021-01-01') ]
df_label.set_index('timestamp', inplace=True)
# resampling
df_label = df_label.resample(rule='T').max()
df_label = df_label[['label']]
df_label = df_merge.join(df_label).resample(rule='T').max()
df_label_8030_0916 = df_label.loc[ '2021-08-30 08:00': '2021-09-16 18:00'].fillna(value=0)
df_label_1025_1103 = df_label.loc[ '2021-10-25 08:00': '2021-11-03 18:00'].fillna(value=0)
df_label.to_csv('df_labeled.csv')
df_label_1025_1103.to_csv('df_label_1025_1103.csv')
df_label_8030_0916.to_csv('df_label_8030_0916.csv')
# read and merge label
# "20211105_193120A.mp4" 동영상 부터 Time Sink 조절 필요
# time stamp 가 실제 시간보다 약 12시간 22분 빠름
import glob
dfs = []
for elm in glob.glob('202111*.csv'):
print(elm)
dfs.append(pd.read_csv(elm))
df_label = pd.concat(dfs, ignore_index=True)
df_label.rename(columns={df_label.columns[1]:'timestamp', df_label.columns[2]:'label'}, inplace=True)
df_label.to_csv('dfs.csv')
20211103_095316A.csv 20211105_193120A.csv 20211109_203305A.csv 20211117_003420A.csv 20211124_001105A.csv