|
@@ -0,0 +1,282 @@
|
|
|
+{
|
|
|
+ "cells": [
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 944,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "import json\n",
|
|
|
+ "import base64\n",
|
|
|
+ "import boto3\n",
|
|
|
+ "import uuid\n",
|
|
|
+ "from datetime import datetime\n",
|
|
|
+ "from urllib import parse\n",
|
|
|
+ "import copy"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 945,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "def pre_proecssing(data) :\n",
|
|
|
+ " global metering_data\n",
|
|
|
+ " global temperature_data\n",
|
|
|
+ " global energymeter_data\n",
|
|
|
+ " global energymeter_light_data\n",
|
|
|
+ " global energymeter_outlet_data\n",
|
|
|
+ " try :\n",
|
|
|
+ " if data['data']['function'] == 'metering' :\n",
|
|
|
+ " metering_json['timestamp'] = data['timestamp']\n",
|
|
|
+ " metering_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
|
|
|
+ " metering_json['dong'] = data['dong']\n",
|
|
|
+ " metering_json['ho'] = data['ho']\n",
|
|
|
+ " metering_json['electric_instant'] = data['data']['value']['electric'][0]\n",
|
|
|
+ " metering_json['electric_cumulative'] = data['data']['value']['electric'][1]\n",
|
|
|
+ " metering_json['gas_instant'] = data['data']['value']['gas'][0]\n",
|
|
|
+ " metering_json['gas_cumulative'] = data['data']['value']['gas'][1]\n",
|
|
|
+ " metering_json['water_instant'] = data['data']['value']['water'][0]\n",
|
|
|
+ " metering_json['water_cumulative'] = data['data']['value']['water'][1]\n",
|
|
|
+ " metering_json['heating_instant'] = data['data']['value']['heating'][0]\n",
|
|
|
+ " metering_json['heating_cumulative'] = data['data']['value']['heating'][1]\n",
|
|
|
+ " metering_json['hotwater_instant'] = data['data']['value']['hotwater'][0]\n",
|
|
|
+ " metering_json['hotwater_cumulative'] = data['data']['value']['hotwater'][1]\n",
|
|
|
+ " metering_data += json.dumps(metering_json) + \"\\n\"\n",
|
|
|
+ " elif data['data']['function'] == 'current_temperture' :\n",
|
|
|
+ " temperture_json['timestamp'] = data['timestamp']\n",
|
|
|
+ " temperture_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
|
|
|
+ " temperture_json['dong'] = data['dong']\n",
|
|
|
+ " temperture_json['ho'] = data['ho']\n",
|
|
|
+ " temperture_json['room1'] = data['data']['value']['room1'] if data['data']['value'].get('room1') != None else None\n",
|
|
|
+ " temperture_json['room2'] = data['data']['value']['room2'] if data['data']['value'].get('room2') != None else None\n",
|
|
|
+ " temperture_json['room3'] = data['data']['value']['room3'] if data['data']['value'].get('room3') != None else None\n",
|
|
|
+ " temperture_json['room4'] = data['data']['value']['room4'] if data['data']['value'].get('room4') != None else None\n",
|
|
|
+ " temperture_json['room5'] = data['data']['value']['room5'] if data['data']['value'].get('room5') != None else None\n",
|
|
|
+ " temperture_json['room6'] = data['data']['value']['room6'] if data['data']['value'].get('room6') != None else None\n",
|
|
|
+ " temperture_json['room7'] = data['data']['value']['room7'] if data['data']['value'].get('room7') != None else None\n",
|
|
|
+ " temperture_json['room8'] = data['data']['value']['room8'] if data['data']['value'].get('room8') != None else None\n",
|
|
|
+ " temperature_data += json.dumps(temperture_json) + \"\\n\"\n",
|
|
|
+ " elif data['data']['function'] == 'energy_meter' :\n",
|
|
|
+ " energymeter_json['timestamp'] = data['timestamp']\n",
|
|
|
+ " energymeter_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
|
|
|
+ " energymeter_json['dong'] = data['dong']\n",
|
|
|
+ " energymeter_json['ho'] = data['ho']\n",
|
|
|
+ " energymeter_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n",
|
|
|
+ " energymeter_json['light_count'] = data['data']['value']['lightCount']\n",
|
|
|
+ " energymeter_json['light_total_instant'] = data['data']['value']['lightPowerConsumption']\n",
|
|
|
+ " energymeter_json['light_total_cumulative'] = data['data']['value']['lightElectricalEnergy']\n",
|
|
|
+ " energymeter_data += json.dumps(energymeter_json) + \"\\n\"\n",
|
|
|
+ " \n",
|
|
|
+ " for i in range(0, data['data']['value']['lightCount']) :\n",
|
|
|
+ " energymeter_light_json['timestamp'] = data['timestamp']\n",
|
|
|
+ " energymeter_light_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
|
|
|
+ " energymeter_light_json['dong'] = data['dong']\n",
|
|
|
+ " energymeter_light_json['ho'] = data['ho']\n",
|
|
|
+ " energymeter_light_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n",
|
|
|
+ " energymeter_light_json['id'] = data['data']['value']['lightInfo'][i]['id']\n",
|
|
|
+ " energymeter_light_json['state'] = data['data']['value']['lightInfo'][i]['state']\n",
|
|
|
+ " energymeter_light_json['colortemperature'] = data['data']['value']['lightInfo'][i]['temperature']\n",
|
|
|
+ " energymeter_light_json['dimming'] = data['data']['value']['lightInfo'][i]['dimming']\n",
|
|
|
+ " energymeter_light_list.append(copy.deepcopy(energymeter_light_json))\n",
|
|
|
+ " \n",
|
|
|
+ " for i in range(0, data['data']['value']['outletCount']) :\n",
|
|
|
+ " energymeter_outlet_json['timestamp'] = data['timestamp']\n",
|
|
|
+ " energymeter_outlet_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
|
|
|
+ " energymeter_outlet_json['dong'] = data['dong']\n",
|
|
|
+ " energymeter_outlet_json['ho'] = data['ho']\n",
|
|
|
+ " energymeter_outlet_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n",
|
|
|
+ " energymeter_outlet_json['id'] = data['data']['value']['outletInfo'][i]['id']\n",
|
|
|
+ " energymeter_outlet_json['state'] = data['data']['value']['outletInfo'][i]['state']\n",
|
|
|
+ " energymeter_outlet_json['mode'] = data['data']['value']['outletInfo'][i]['mode']\n",
|
|
|
+ " energymeter_outlet_json['standby_instant'] = data['data']['value']['outletInfo'][i]['cutoffOutletPowerConsumption']\n",
|
|
|
+ " energymeter_outlet_json['standby_cumulative'] = data['data']['value']['outletInfo'][i]['cutoffOutletElectricalEnergy']\n",
|
|
|
+ " energymeter_outlet_json['constant_instant'] = data['data']['value']['outletInfo'][i]['constantOutletPowerConsumption']\n",
|
|
|
+ " energymeter_outlet_json['constant_cumulative'] = data['data']['value']['outletInfo'][i]['constantOutletElectricalEnergy']\n",
|
|
|
+ " energymeter_outlet_list.append(copy.deepcopy(energymeter_outlet_json))\n",
|
|
|
+ " except KeyError as e:\n",
|
|
|
+ " print (e)"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 946,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "metering_json = {}\n",
|
|
|
+ "metering_data = \"\"\n",
|
|
|
+ "temperture_json = {}\n",
|
|
|
+ "temperature_data = \"\"\n",
|
|
|
+ "energymeter_json = {}\n",
|
|
|
+ "energymeter_data = \"\"\n",
|
|
|
+ "energymeter_light_list = []\n",
|
|
|
+ "energymeter_light_json = {}\n",
|
|
|
+ "energymeter_light_data = \"\"\n",
|
|
|
+ "energymeter_outlet_json = {}\n",
|
|
|
+ "energymeter_outlet_list = []\n",
|
|
|
+ "energymeter_outlet_data = \"\""
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "metadata": {
|
|
|
+ "scrolled": true
|
|
|
+ },
|
|
|
+ "outputs": [
|
|
|
+ {
|
|
|
+ "name": "stdout",
|
|
|
+ "output_type": "stream",
|
|
|
+ "text": [
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n",
|
|
|
+ "Expecting value: line 1 column 1 (char 0)\n"
|
|
|
+ ]
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "source": [
|
|
|
+ "s3=boto3.resource('s3')\n",
|
|
|
+ "bucket = s3.Bucket('hdci-ambt-wallpad-raw')\n",
|
|
|
+ "for obj in bucket.objects.all():\n",
|
|
|
+ " if obj.key.split('/')[0] == 'dev_old_not_use' :\n",
|
|
|
+ " break\n",
|
|
|
+ " \n",
|
|
|
+ " year = obj.key.split('-')[5]\n",
|
|
|
+ " month = obj.key.split('-')[6]\n",
|
|
|
+ " day = obj.key.split('-')[7]\n",
|
|
|
+ " hour = obj.key.split('-')[8]\n",
|
|
|
+ " minute = obj.key.split('-')[9]\n",
|
|
|
+ " second = obj.key.split('-')[10]\n",
|
|
|
+ "\n",
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-raw',s3_key)\n",
|
|
|
+ " bodystream = object.get()['Body'].read().decode(\"utf-8\")\n",
|
|
|
+ " try :\n",
|
|
|
+ " for obj in bodystream.split('\\n') :\n",
|
|
|
+ " if obj != None :\n",
|
|
|
+ " pre_proecssing(json.loads(obj))\n",
|
|
|
+ " except ValueError as e:\n",
|
|
|
+ " print (e)\n",
|
|
|
+ " \n",
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-prep','test/metering/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
|
|
|
+ " object.put(Body=metering_data)\n",
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-prep','test/temperature/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
|
|
|
+ " object.put(Body=temperature_data)\n",
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
|
|
|
+ " object.put(Body=energymeter_data)\n",
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/light/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
|
|
|
+ " for obj in energymeter_light_list :\n",
|
|
|
+ " energymeter_light_data += json.dumps(obj) + '\\n'\n",
|
|
|
+ " object.put(Body=energymeter_light_data)\n",
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/outlet/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
|
|
|
+ " for obj in energymeter_outlet_list :\n",
|
|
|
+ " energymeter_outlet_data += json.dumps(obj) + '\\n'\n",
|
|
|
+ " object.put(Body=energymeter_outlet_data)"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": []
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "metadata": {
|
|
|
+ "kernelspec": {
|
|
|
+ "display_name": "Python 3",
|
|
|
+ "language": "python",
|
|
|
+ "name": "python3"
|
|
|
+ },
|
|
|
+ "language_info": {
|
|
|
+ "codemirror_mode": {
|
|
|
+ "name": "ipython",
|
|
|
+ "version": 3
|
|
|
+ },
|
|
|
+ "file_extension": ".py",
|
|
|
+ "mimetype": "text/x-python",
|
|
|
+ "name": "python",
|
|
|
+ "nbconvert_exporter": "python",
|
|
|
+ "pygments_lexer": "ipython3",
|
|
|
+ "version": "3.8.3"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "nbformat": 4,
|
|
|
+ "nbformat_minor": 4
|
|
|
+}
|