| 
															
																@@ -0,0 +1,201 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import json 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import urllib.parse 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import boto3 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+from csv import reader 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import pandas as pd 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import pymysql 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import numpy as np 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import awswrangler as wr 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+rds_host_write  = "database-ambt.cluster-cvnwxgmngsms.ap-northeast-2.rds.amazonaws.com" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+rds_host_read  = "database-ambt.cluster-ro-cvnwxgmngsms.ap-northeast-2.rds.amazonaws.com" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+name = "admin" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+password = "hdci12#$" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+db_name = "ambt" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+PORT=3306 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+def lambda_handler(event, context): 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    conn =  pymysql.connect(host=rds_host_read, user=name, passwd=password, port=PORT, database=db_name, charset='utf8') 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    #conn = pymysql.connect(host=rds_host, user=name, passwd=password, db=db_name, use_unicode=True, charset='utf8', connect_timeout=5) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    sql_statement = "SELECT * FROM `ambt.icos`.facility_type;" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    facility_type_id = pd.read_sql(sql=sql_statement, con=conn) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    sql_statement = "SELECT * FROM `ambt.icos`.facility_code;" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    material_code = pd.read_sql(sql=sql_statement, con=conn) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    sql_statement = "SELECT * FROM `ambt.icos`.control_value;" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    control_value = pd.read_sql(sql=sql_statement, con=conn) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    sql_statement = "SELECT * FROM `ambt.icos`.value_type;" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    value_type = pd.read_sql(sql=sql_statement, con=conn) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    sql_statement = "SELECT * FROM `ambt.icos`.ambt_anoicos_code;" 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    mappingTable = pd.read_sql(sql=sql_statement, con=conn) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    print(mappingTable.head()) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    for record in event['Records']: 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        #print("Received event: " + json.dumps(event, indent=2)) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        bucket = record['s3']['bucket']['name']  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        key = urllib.parse.unquote_plus(record['s3']['object']['key']) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('bucket : ', bucket) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('key : ', key) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        s3 = boto3.client('s3') 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        obj = s3.get_object(Bucket=bucket, Key=key) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        raw_data = pd.read_csv(obj['Body'], sep=',') 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        raw_data = raw_data.drop(['cnt'], axis=1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # raw_data = raw_data.drop(['Unnamed: 0'], axis=1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        raw_data = raw_data.dropna(axis=0)      # drop the row including null 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        raw_data = raw_data.reset_index(drop=True) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('raw', raw_data.columns) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print (raw_data[130:140]) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('length of raw data', len(raw_data)) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # --------------- Data pre-processing (considering data minimum properties) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # temperature, humidity (set point, ) - -15 ~ 100 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # operating status - 0/1 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # open ratio (SP) - 0 ~ 100 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # operating mode - 0 ~ 10 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        processed_data = raw_data.copy() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        null_sum = pd.DataFrame(np.zeros((1,len(raw_data.columns))), columns = raw_data.columns) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # control_value = 3(온도), 4(습도), 5(온도 설정값), 6(습도 설정값) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        target_row = mappingTable.loc[(mappingTable['site_id']==1) & (mappingTable['collect_status'] == 1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                 & ((mappingTable['control_value'] == 3) | (mappingTable['control_value'] == 4) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                  | (mappingTable['control_value'] == 5) | (mappingTable['control_value'] == 6))] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+               
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        for c_idx in range(raw_data[list(set(target_row['raw_tag']))].shape[1]): 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data[list(set(target_row['raw_tag']))[c_idx]] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = [r_value if r_value > -15 and r_value <= 100 else np.nan \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            for r_value in raw_data[list(set(target_row['raw_tag']))[c_idx]]] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            null_sum[list(set(target_row['raw_tag']))[c_idx]][0] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = processed_data[list(set(target_row['raw_tag']))[c_idx]].isnull().sum() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # control_value = 1(운전상태) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        target_row = mappingTable.loc[(mappingTable['site_id']==1) & (mappingTable['collect_status'] == 1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                 & (mappingTable['control_value'] == 1) ] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        for c_idx in range(raw_data[list(set(target_row['raw_tag']))].shape[1]): 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data[list(set(target_row['raw_tag']))[c_idx]] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = [r_value if r_value == 0 or r_value == 1 else np.nan \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            for r_value in raw_data[list(set(target_row['raw_tag']))[c_idx]]] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            null_sum[list(set(target_row['raw_tag']))[c_idx]][0] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = processed_data[list(set(target_row['raw_tag']))[c_idx]].isnull().sum()         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # control_value = 7(개도율), 8(개도율 설정값) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        target_row = mappingTable.loc[(mappingTable['site_id']==1) & (mappingTable['collect_status'] == 1)  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        & ((mappingTable['control_value'] == 7) | (mappingTable['control_value'] == 8)) ] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        for c_idx in range(raw_data[list(set(target_row['raw_tag']))].shape[1]): 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data[list(set(target_row['raw_tag']))[c_idx]] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = [r_value if r_value >= 0 or r_value <= 100 else np.nan \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            for r_value in raw_data[list(set(target_row['raw_tag']))[c_idx]]] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            null_sum[list(set(target_row['raw_tag']))[c_idx]][0] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = processed_data[list(set(target_row['raw_tag']))[c_idx]].isnull().sum()   
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # control_value = 30(운전모드) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        target_row = mappingTable.loc[(mappingTable['site_id']==1) & (mappingTable['collect_status'] == 1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                 & (mappingTable['control_value'] == 30) ] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        for c_idx in range(raw_data[list(set(target_row['raw_tag']))].shape[1]): 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data[list(set(target_row['raw_tag']))[c_idx]] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = [r_value if r_value >= 0 and r_value <= 10 else np.nan \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            for r_value in raw_data[list(set(target_row['raw_tag']))[c_idx]]] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+     
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            null_sum[list(set(target_row['raw_tag']))[c_idx]][0] \ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            = processed_data[list(set(target_row['raw_tag']))[c_idx]].isnull().sum()   
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('sum of null point:', null_sum.sum()) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # --------------- Fill the missing data and round off time index 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        time_resolution = 5 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        #data = raw_data 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        round_m_comp = 0 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        missing_idx = [] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        missing_date = [] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('len(processed_data))',len(processed_data)) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        for time_idx in range(len(processed_data)): 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            str_date = processed_data['time'][time_idx] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            date = str_date.split()[0] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            round_m = int(int(str_date.split()[1].split(':')[1])/time_resolution)*time_resolution 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            h, m, s = [str_date.split()[1].split(':')[0], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                       str(round_m) if round_m >= 10 else '0'+str(round_m), 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                       '00'] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+             
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            round_m_comp += time_resolution  # for compare time index 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            # ----- find the missing row ----- # 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            while round_m_comp <= int(h) * 60 + round_m: 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                round_m_comp_tmp = round_m_comp - time_resolution 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                round_h_tmp = int(round_m_comp_tmp/60) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                round_m_tmp = int(round_m_comp_tmp%60) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                h_tmp, m_tmp, s_tmp = [str(round_h_tmp) if round_h_tmp >= 10 else '0'+str(round_h_tmp), 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                       str(round_m_tmp) if round_m_tmp >= 10 else '0'+str(round_m_tmp), 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                       '00'] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                missing_date.append(date + ' ' + h_tmp + ':' + m_tmp + ':' + s_tmp) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                missing_idx.append(time_idx + len(missing_idx))   # save missing index considering append index 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                round_m_comp += time_resolution       
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            # ----- find the missing row ----- #         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data.at[time_idx,'time'] = date + ' ' + h + ':' + m + ':' + s # round off time index for rows of normal data  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('len', len(processed_data)) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('sum of missing row data:', len(missing_idx)) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ----- fill nan on the missing row ----- # 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        idx = 0 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        for miss_idx in missing_idx: 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            tmp_data = np.zeros((1,len(raw_data.columns))) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            tmp_data[:] = np.nan 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            reconstructed_data = pd.DataFrame(tmp_data, columns = raw_data.columns) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+             
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            idx_temp = miss_idx 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            temp1 = processed_data[processed_data.index < idx_temp].copy() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            temp2 = processed_data[processed_data.index >= idx_temp].copy() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+             
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            temp1 = temp1.append(reconstructed_data, ignore_index=True) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data = temp1.append(temp2, ignore_index=True) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+             
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            processed_data.at[idx_temp, 'time'] = missing_date[idx] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+            idx += 1 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ----- fill nan on the missing row ----- # 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ----- interpolation ----- # 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ControlValue = 3(온도), 4(습도) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        TemHum_row = mappingTable.loc[(mappingTable['site_id']==1) & (mappingTable['collect_status'] == 1)  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                 & ((mappingTable['control_value'] == 3) | (mappingTable['control_value'] == 4)) ] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        temp1 = processed_data[list(processed_data[list(set(TemHum_row['raw_tag']))])].fillna(method='pad').copy() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ControlValue = 1(운전상태(On/off)), 7(개도율), 8(개도율 설정값), 19(차압), 30(운전모드) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        StatusOpenRatio_row = mappingTable.loc[(mappingTable['site_id']==1) & (mappingTable['collect_status'] == 1)  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                          & ((mappingTable['control_value'] == 1) | (mappingTable['control_value'] == 7)  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                          | (mappingTable['control_value'] == 8) | (mappingTable['control_value'] == 19)  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                                          | (mappingTable['control_value'] == 30)) ] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        temp2 = processed_data[list(processed_data[list(set(StatusOpenRatio_row['raw_tag']))])].fillna(method='pad').copy() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        others_tag = [x for x in list(set(raw_data.columns)) if x not in list(set(TemHum_row['raw_tag']))] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        others_tag = [x for x in others_tag if x not in list(set(StatusOpenRatio_row['raw_tag']))] 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        #print('iik',list(set(raw_data.columns)).remove(list(set(TemHum_row['raw_tag'])) + list(set(StatusOpenRatio_row['raw_tag'])))) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        others = processed_data[others_tag].fillna(method='pad').copy() 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        processed_data = pd.concat([others, temp1, temp2], axis=1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        processed_data = processed_data.round(1) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ----- interpolation ----- # 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('processed', processed_data) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ----- load to S3 prep bucket ----- # 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        curated_file_key = key.replace(key.split('/')[-1], '') 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        target_path = 's3://hdci-ambt-anoicos-prep/{}'.format(curated_file_key) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        print('target_path', target_path) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        wr.s3.to_csv( 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                        df=processed_data, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                        path=target_path, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                        mode='overwrite', 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                        dataset=True 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+                        ) 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+         
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+        # ----- load to S3 prep bucket ----- # 
															 |