ambt-wallpad-whole-prep.ipynb 15 KB

import json
import base64
import boto3
import uuid
from datetime import datetime
from urllib import parse
import copy
def pre_proecssing(data) :
    global metering_data
    global temperature_data
    global energymeter_data
    global energymeter_light_data
    global energymeter_outlet_data
    try :
        if data['data']['function'] == 'metering' :
            metering_json['timestamp'] = data['timestamp']
            metering_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'
            metering_json['dong'] = data['dong']
            metering_json['ho'] = data['ho']
            metering_json['electric_instant'] = data['data']['value']['electric'][0]
            metering_json['electric_cumulative'] = data['data']['value']['electric'][1]
            metering_json['gas_instant'] = data['data']['value']['gas'][0]
            metering_json['gas_cumulative'] = data['data']['value']['gas'][1]
            metering_json['water_instant'] = data['data']['value']['water'][0]
            metering_json['water_cumulative'] = data['data']['value']['water'][1]
            metering_json['heating_instant'] = data['data']['value']['heating'][0]
            metering_json['heating_cumulative'] = data['data']['value']['heating'][1]
            metering_json['hotwater_instant'] = data['data']['value']['hotwater'][0]
            metering_json['hotwater_cumulative'] = data['data']['value']['hotwater'][1]
            metering_data += json.dumps(metering_json) + "\n"
        elif data['data']['function'] == 'current_temperture' :
            temperture_json['timestamp'] = data['timestamp']
            temperture_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'
            temperture_json['dong'] = data['dong']
            temperture_json['ho'] = data['ho']
            temperture_json['room1'] = data['data']['value']['room1'] if data['data']['value'].get('room1') != None else None
            temperture_json['room2'] = data['data']['value']['room2'] if data['data']['value'].get('room2') != None else None
            temperture_json['room3'] = data['data']['value']['room3'] if data['data']['value'].get('room3') != None else None
            temperture_json['room4'] = data['data']['value']['room4'] if data['data']['value'].get('room4') != None else None
            temperture_json['room5'] = data['data']['value']['room5'] if data['data']['value'].get('room5') != None else None
            temperture_json['room6'] = data['data']['value']['room6'] if data['data']['value'].get('room6') != None else None
            temperture_json['room7'] = data['data']['value']['room7'] if data['data']['value'].get('room7') != None else None
            temperture_json['room8'] = data['data']['value']['room8'] if data['data']['value'].get('room8') != None else None
            temperature_data += json.dumps(temperture_json) + "\n"
        elif data['data']['function'] == 'energy_meter' :
            energymeter_json['timestamp'] = data['timestamp']
            energymeter_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'
            energymeter_json['dong'] = data['dong']
            energymeter_json['ho'] = data['ho']
            energymeter_json['energy_meter_id'] = data['data']['value']['energyMeterId']
            energymeter_json['light_count'] = data['data']['value']['lightCount']
            energymeter_json['light_total_instant'] = data['data']['value']['lightPowerConsumption']
            energymeter_json['light_total_cumulative'] = data['data']['value']['lightElectricalEnergy']
            energymeter_data += json.dumps(energymeter_json) + "\n"
    
            for i in range(0, data['data']['value']['lightCount']) :
                energymeter_light_json['timestamp'] = data['timestamp']
                energymeter_light_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'
                energymeter_light_json['dong'] = data['dong']
                energymeter_light_json['ho'] = data['ho']
                energymeter_light_json['energy_meter_id'] = data['data']['value']['energyMeterId']
                energymeter_light_json['id'] = data['data']['value']['lightInfo'][i]['id']
                energymeter_light_json['state'] = data['data']['value']['lightInfo'][i]['state']
                energymeter_light_json['colortemperature'] = data['data']['value']['lightInfo'][i]['temperature']
                energymeter_light_json['dimming'] = data['data']['value']['lightInfo'][i]['dimming']
                energymeter_light_list.append(copy.deepcopy(energymeter_light_json))
                
            for i in range(0, data['data']['value']['outletCount']) :
                energymeter_outlet_json['timestamp'] = data['timestamp']
                energymeter_outlet_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'
                energymeter_outlet_json['dong'] = data['dong']
                energymeter_outlet_json['ho'] = data['ho']
                energymeter_outlet_json['energy_meter_id'] = data['data']['value']['energyMeterId']
                energymeter_outlet_json['id'] = data['data']['value']['outletInfo'][i]['id']
                energymeter_outlet_json['state'] = data['data']['value']['outletInfo'][i]['state']
                energymeter_outlet_json['mode'] = data['data']['value']['outletInfo'][i]['mode']
                energymeter_outlet_json['standby_instant'] = data['data']['value']['outletInfo'][i]['cutoffOutletPowerConsumption']
                energymeter_outlet_json['standby_cumulative'] = data['data']['value']['outletInfo'][i]['cutoffOutletElectricalEnergy']
                energymeter_outlet_json['constant_instant'] = data['data']['value']['outletInfo'][i]['constantOutletPowerConsumption']
                energymeter_outlet_json['constant_cumulative'] = data['data']['value']['outletInfo'][i]['constantOutletElectricalEnergy']
                energymeter_outlet_list.append(copy.deepcopy(energymeter_outlet_json))
    except KeyError as e:
        print (e)
metering_json = {}
metering_data = ""
temperture_json = {}
temperature_data = ""
energymeter_json = {}
energymeter_data = ""
energymeter_light_list = []
energymeter_light_json = {}
energymeter_light_data = ""
energymeter_outlet_json = {}
energymeter_outlet_list = []
energymeter_outlet_data = ""
s3=boto3.resource('s3')
bucket = s3.Bucket('hdci-ambt-wallpad-raw')
for obj in bucket.objects.all():
    if obj.key.split('/')[0] == 'dev_old_not_use' :
        break
    
    year = obj.key.split('-')[5]
    month = obj.key.split('-')[6]
    day = obj.key.split('-')[7]
    hour = obj.key.split('-')[8]
    minute = obj.key.split('-')[9]
    second = obj.key.split('-')[10]

    object = s3.Object('hdci-ambt-wallpad-raw',s3_key)
    bodystream = object.get()['Body'].read().decode("utf-8")
    try :
        for obj in bodystream.split('\n') :
            if obj != None :
                pre_proecssing(json.loads(obj))
    except ValueError as e:
        print (e)
    
    object = s3.Object('hdci-ambt-wallpad-prep','test/metering/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))
    object.put(Body=metering_data)
    object = s3.Object('hdci-ambt-wallpad-prep','test/temperature/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))
    object.put(Body=temperature_data)
    object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))
    object.put(Body=energymeter_data)
    object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/light/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))
    for obj in energymeter_light_list :
        energymeter_light_data += json.dumps(obj) + '\n'
    object.put(Body=energymeter_light_data)
    object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/outlet/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))
    for obj in energymeter_outlet_list :
        energymeter_outlet_data += json.dumps(obj) + '\n'
    object.put(Body=energymeter_outlet_data)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)
Expecting value: line 1 column 1 (char 0)