{ "cells": [ { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "import json\n", "import base64\n", "import boto3\n", "import uuid\n", "from datetime import datetime\n", "from urllib import parse\n", "import copy" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "def pre_proecssing(data) :\n", " global metering_data\n", " global temperature_data\n", " global energymeter_data\n", " global energymeter_light_data\n", " global energymeter_outlet_data\n", " try :\n", " if data['data']['function'] == 'metering' :\n", " metering_json['timestamp'] = str(datetime.strptime(data['timestamp'], \"%Y-%m-%dT%H:%M:%S\"))\n", " metering_json['site'] = data['site'] if data.get('site') != None else 'yeoju' if data['dong'] == '105' else 'sujigwanggyosan'\n", " metering_json['dong'] = data['dong']\n", " metering_json['ho'] = data['ho']\n", " metering_json['electric_instant'] = data['data']['value']['electric'][0]\n", " metering_json['electric_cumulative'] = data['data']['value']['electric'][1]\n", " metering_json['gas_instant'] = data['data']['value']['gas'][0]\n", " metering_json['gas_cumulative'] = data['data']['value']['gas'][1]\n", " metering_json['water_instant'] = data['data']['value']['water'][0]\n", " metering_json['water_cumulative'] = data['data']['value']['water'][1]\n", " metering_json['heating_instant'] = data['data']['value']['heating'][0]\n", " metering_json['heating_cumulative'] = data['data']['value']['heating'][1]\n", " metering_json['hotwater_instant'] = data['data']['value']['hotwater'][0]\n", " metering_json['hotwater_cumulative'] = data['data']['value']['hotwater'][1]\n", " metering_data += json.dumps(metering_json) + \"\\n\"\n", " \n", " elif data['data']['function'] == 'current_temperture' :\n", " temperture_json['timestamp'] = str(datetime.strptime(data['timestamp'], \"%Y-%m-%dT%H:%M:%S\"))\n", " temperture_json['site'] = data['site'] if data.get('site') != None else 'yeoju' if data['dong'] == '105' else 'sujigwanggyosan'\n", " temperture_json['dong'] = data['dong']\n", " temperture_json['ho'] = data['ho']\n", " temperture_json['room1'] = data['data']['value']['room1'] if data['data']['value'].get('room1') != None else None\n", " temperture_json['room2'] = data['data']['value']['room2'] if data['data']['value'].get('room2') != None else None\n", " temperture_json['room3'] = data['data']['value']['room3'] if data['data']['value'].get('room3') != None else None\n", " temperture_json['room4'] = data['data']['value']['room4'] if data['data']['value'].get('room4') != None else None\n", " temperture_json['room5'] = data['data']['value']['room5'] if data['data']['value'].get('room5') != None else None\n", " temperture_json['room6'] = data['data']['value']['room6'] if data['data']['value'].get('room6') != None else None\n", " temperture_json['room7'] = data['data']['value']['room7'] if data['data']['value'].get('room7') != None else None\n", " temperture_json['room8'] = data['data']['value']['room8'] if data['data']['value'].get('room8') != None else None\n", " temperature_data += json.dumps(temperture_json) + \"\\n\"\n", " elif data['data']['function'] == 'energy_meter' :\n", " energymeter_json['timestamp'] = str(datetime.strptime(data['timestamp'], \"%Y-%m-%dT%H:%M:%S\"))\n", " energymeter_json['site'] = data['site'] if data.get('site') != None else 'yeoju' if data['dong'] == '105' else 'sujigwanggyosan'\n", " energymeter_json['dong'] = data['dong']\n", " energymeter_json['ho'] = data['ho']\n", " energymeter_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n", " energymeter_json['light_count'] = data['data']['value']['lightCount']\n", " energymeter_json['light_total_instant'] = data['data']['value']['lightPowerConsumption']\n", " energymeter_json['light_total_cumulative'] = data['data']['value']['lightElectricalEnergy']\n", " energymeter_data += json.dumps(energymeter_json) + \"\\n\"\n", " \n", " for i in range(0, data['data']['value']['lightCount']) :\n", " energymeter_light_json['timestamp'] = str(datetime.strptime(data['timestamp'], \"%Y-%m-%dT%H:%M:%S\"))\n", " energymeter_light_json['site'] = data['site'] if data.get('site') != None else 'yeoju' if data['dong'] == '105' else 'sujigwanggyosan'\n", " energymeter_light_json['dong'] = data['dong']\n", " energymeter_light_json['ho'] = data['ho']\n", " energymeter_light_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n", " energymeter_light_json['id'] = data['data']['value']['lightInfo'][i]['id']\n", " energymeter_light_json['state'] = data['data']['value']['lightInfo'][i]['state']\n", " energymeter_light_json['colortemperature'] = data['data']['value']['lightInfo'][i]['temperature']\n", " energymeter_light_json['dimming'] = data['data']['value']['lightInfo'][i]['dimming']\n", " energymeter_light_list.append(copy.deepcopy(energymeter_light_json))\n", " \n", " for i in range(0, data['data']['value']['outletCount']) :\n", " energymeter_outlet_json['timestamp'] = str(datetime.strptime(data['timestamp'], \"%Y-%m-%dT%H:%M:%S\"))\n", " energymeter_outlet_json['site'] = data['site'] if data.get('site') != None else 'yeoju' if data['dong'] == '105' else 'sujigwanggyosan'\n", " energymeter_outlet_json['dong'] = data['dong']\n", " energymeter_outlet_json['ho'] = data['ho']\n", " energymeter_outlet_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n", " energymeter_outlet_json['id'] = data['data']['value']['outletInfo'][i]['id']\n", " energymeter_outlet_json['state'] = data['data']['value']['outletInfo'][i]['state']\n", " energymeter_outlet_json['mode'] = data['data']['value']['outletInfo'][i]['mode']\n", " energymeter_outlet_json['standby_instant'] = data['data']['value']['outletInfo'][i]['cutoffOutletPowerConsumption']\n", " energymeter_outlet_json['standby_cumulative'] = data['data']['value']['outletInfo'][i]['cutoffOutletElectricalEnergy']\n", " energymeter_outlet_json['constant_instant'] = data['data']['value']['outletInfo'][i]['constantOutletPowerConsumption']\n", " energymeter_outlet_json['constant_cumulative'] = data['data']['value']['outletInfo'][i]['constantOutletElectricalEnergy']\n", " energymeter_outlet_list.append(copy.deepcopy(energymeter_outlet_json))\n", " except KeyError as e:\n", " print (e)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "s3=boto3.resource('s3')\n", "bucket = s3.Bucket('hdci-ambt-wallpad-raw')\n", "for obj in bucket.objects.all():\n", " metering_json = {}\n", " metering_data = \"\"\n", " temperture_json = {}\n", " temperature_data = \"\"\n", " energymeter_json = {}\n", " energymeter_data = \"\"\n", " energymeter_light_list = []\n", " energymeter_light_json = {}\n", " energymeter_light_data = \"\"\n", " energymeter_outlet_json = {}\n", " energymeter_outlet_list = []\n", " energymeter_outlet_data = \"\"\n", " if obj.key.split('/')[0] != 'dev' :\n", " continue\n", " \n", " site = obj.key.split('/')[1]\n", " year = obj.key.split('-')[5]\n", " month = obj.key.split('-')[6]\n", " day = obj.key.split('-')[7]\n", " hour = obj.key.split('-')[8]\n", " minute = obj.key.split('-')[9]\n", " second = obj.key.split('-')[10]\n", " \n", " object = s3.Object('hdci-ambt-wallpad-raw', obj.key)\n", " bodystream = object.get()['Body'].read().decode(\"utf-8\")\n", " try :\n", " for i in range(0, len(bodystream.split('\\n'))-1):\n", " pre_proecssing(json.loads(bodystream.split('\\n')[i]))\n", " break\n", " except ValueError as e:\n", " print (e)\n", " \n", " object = s3.Object('hdci-ambt-wallpad-raw','dev-clean/metering/{}/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(site, year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n", " object.put(Body=metering_data)\n", " object = s3.Object('hdci-ambt-wallpad-raw','dev-clean/temperature/{}/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(site, year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n", " object.put(Body=temperature_data)\n", " object = s3.Object('hdci-ambt-wallpad-raw','dev-clean/energymeter/{}/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(site, year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n", " object.put(Body=energymeter_data)\n", " object = s3.Object('hdci-ambt-wallpad-raw','dev-clean/energymeter-light/{}/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(site, year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n", " for obj in energymeter_light_list :\n", " energymeter_light_data += json.dumps(obj) + '\\n'\n", " object.put(Body=energymeter_light_data)\n", " object = s3.Object('hdci-ambt-wallpad-raw','dev-clean/energymeter-outlet/{}/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(site, year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n", " for obj in energymeter_outlet_list :\n", " energymeter_outlet_data += json.dumps(obj) + '\\n'\n", " object.put(Body=energymeter_outlet_data)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.13" } }, "nbformat": 4, "nbformat_minor": 4 }