Prechádzať zdrojové kódy

월패드저장시간부터 현재까지 데이터 전처리(검침, 온도, 에너지미터

LAPTOP-36F69PCF\HEEWON 3 rokov pred
rodič
commit
26de81ecb7

+ 282 - 0
ambt-wallpad-whole-prep/ambt-wallpad-whole-prep.ipynb

@@ -0,0 +1,282 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 944,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import base64\n",
+    "import boto3\n",
+    "import uuid\n",
+    "from datetime import datetime\n",
+    "from urllib import parse\n",
+    "import copy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 945,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pre_proecssing(data) :\n",
+    "    global metering_data\n",
+    "    global temperature_data\n",
+    "    global energymeter_data\n",
+    "    global energymeter_light_data\n",
+    "    global energymeter_outlet_data\n",
+    "    try :\n",
+    "        if data['data']['function'] == 'metering' :\n",
+    "            metering_json['timestamp'] = data['timestamp']\n",
+    "            metering_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
+    "            metering_json['dong'] = data['dong']\n",
+    "            metering_json['ho'] = data['ho']\n",
+    "            metering_json['electric_instant'] = data['data']['value']['electric'][0]\n",
+    "            metering_json['electric_cumulative'] = data['data']['value']['electric'][1]\n",
+    "            metering_json['gas_instant'] = data['data']['value']['gas'][0]\n",
+    "            metering_json['gas_cumulative'] = data['data']['value']['gas'][1]\n",
+    "            metering_json['water_instant'] = data['data']['value']['water'][0]\n",
+    "            metering_json['water_cumulative'] = data['data']['value']['water'][1]\n",
+    "            metering_json['heating_instant'] = data['data']['value']['heating'][0]\n",
+    "            metering_json['heating_cumulative'] = data['data']['value']['heating'][1]\n",
+    "            metering_json['hotwater_instant'] = data['data']['value']['hotwater'][0]\n",
+    "            metering_json['hotwater_cumulative'] = data['data']['value']['hotwater'][1]\n",
+    "            metering_data += json.dumps(metering_json) + \"\\n\"\n",
+    "        elif data['data']['function'] == 'current_temperture' :\n",
+    "            temperture_json['timestamp'] = data['timestamp']\n",
+    "            temperture_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
+    "            temperture_json['dong'] = data['dong']\n",
+    "            temperture_json['ho'] = data['ho']\n",
+    "            temperture_json['room1'] = data['data']['value']['room1'] if data['data']['value'].get('room1') != None else None\n",
+    "            temperture_json['room2'] = data['data']['value']['room2'] if data['data']['value'].get('room2') != None else None\n",
+    "            temperture_json['room3'] = data['data']['value']['room3'] if data['data']['value'].get('room3') != None else None\n",
+    "            temperture_json['room4'] = data['data']['value']['room4'] if data['data']['value'].get('room4') != None else None\n",
+    "            temperture_json['room5'] = data['data']['value']['room5'] if data['data']['value'].get('room5') != None else None\n",
+    "            temperture_json['room6'] = data['data']['value']['room6'] if data['data']['value'].get('room6') != None else None\n",
+    "            temperture_json['room7'] = data['data']['value']['room7'] if data['data']['value'].get('room7') != None else None\n",
+    "            temperture_json['room8'] = data['data']['value']['room8'] if data['data']['value'].get('room8') != None else None\n",
+    "            temperature_data += json.dumps(temperture_json) + \"\\n\"\n",
+    "        elif data['data']['function'] == 'energy_meter' :\n",
+    "            energymeter_json['timestamp'] = data['timestamp']\n",
+    "            energymeter_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
+    "            energymeter_json['dong'] = data['dong']\n",
+    "            energymeter_json['ho'] = data['ho']\n",
+    "            energymeter_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n",
+    "            energymeter_json['light_count'] = data['data']['value']['lightCount']\n",
+    "            energymeter_json['light_total_instant'] = data['data']['value']['lightPowerConsumption']\n",
+    "            energymeter_json['light_total_cumulative'] = data['data']['value']['lightElectricalEnergy']\n",
+    "            energymeter_data += json.dumps(energymeter_json) + \"\\n\"\n",
+    "    \n",
+    "            for i in range(0, data['data']['value']['lightCount']) :\n",
+    "                energymeter_light_json['timestamp'] = data['timestamp']\n",
+    "                energymeter_light_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
+    "                energymeter_light_json['dong'] = data['dong']\n",
+    "                energymeter_light_json['ho'] = data['ho']\n",
+    "                energymeter_light_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n",
+    "                energymeter_light_json['id'] = data['data']['value']['lightInfo'][i]['id']\n",
+    "                energymeter_light_json['state'] = data['data']['value']['lightInfo'][i]['state']\n",
+    "                energymeter_light_json['colortemperature'] = data['data']['value']['lightInfo'][i]['temperature']\n",
+    "                energymeter_light_json['dimming'] = data['data']['value']['lightInfo'][i]['dimming']\n",
+    "                energymeter_light_list.append(copy.deepcopy(energymeter_light_json))\n",
+    "                \n",
+    "            for i in range(0, data['data']['value']['outletCount']) :\n",
+    "                energymeter_outlet_json['timestamp'] = data['timestamp']\n",
+    "                energymeter_outlet_json['site'] = data['site'] if data['site'] != None else 'yeoju' if data['dong'] == 105 else 'sujigwanggyosan'\n",
+    "                energymeter_outlet_json['dong'] = data['dong']\n",
+    "                energymeter_outlet_json['ho'] = data['ho']\n",
+    "                energymeter_outlet_json['energy_meter_id'] = data['data']['value']['energyMeterId']\n",
+    "                energymeter_outlet_json['id'] = data['data']['value']['outletInfo'][i]['id']\n",
+    "                energymeter_outlet_json['state'] = data['data']['value']['outletInfo'][i]['state']\n",
+    "                energymeter_outlet_json['mode'] = data['data']['value']['outletInfo'][i]['mode']\n",
+    "                energymeter_outlet_json['standby_instant'] = data['data']['value']['outletInfo'][i]['cutoffOutletPowerConsumption']\n",
+    "                energymeter_outlet_json['standby_cumulative'] = data['data']['value']['outletInfo'][i]['cutoffOutletElectricalEnergy']\n",
+    "                energymeter_outlet_json['constant_instant'] = data['data']['value']['outletInfo'][i]['constantOutletPowerConsumption']\n",
+    "                energymeter_outlet_json['constant_cumulative'] = data['data']['value']['outletInfo'][i]['constantOutletElectricalEnergy']\n",
+    "                energymeter_outlet_list.append(copy.deepcopy(energymeter_outlet_json))\n",
+    "    except KeyError as e:\n",
+    "        print (e)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 946,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metering_json = {}\n",
+    "metering_data = \"\"\n",
+    "temperture_json = {}\n",
+    "temperature_data = \"\"\n",
+    "energymeter_json = {}\n",
+    "energymeter_data = \"\"\n",
+    "energymeter_light_list = []\n",
+    "energymeter_light_json = {}\n",
+    "energymeter_light_data = \"\"\n",
+    "energymeter_outlet_json = {}\n",
+    "energymeter_outlet_list = []\n",
+    "energymeter_outlet_data = \"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n",
+      "Expecting value: line 1 column 1 (char 0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "s3=boto3.resource('s3')\n",
+    "bucket = s3.Bucket('hdci-ambt-wallpad-raw')\n",
+    "for obj in bucket.objects.all():\n",
+    "    if obj.key.split('/')[0] == 'dev_old_not_use' :\n",
+    "        break\n",
+    "    \n",
+    "    year = obj.key.split('-')[5]\n",
+    "    month = obj.key.split('-')[6]\n",
+    "    day = obj.key.split('-')[7]\n",
+    "    hour = obj.key.split('-')[8]\n",
+    "    minute = obj.key.split('-')[9]\n",
+    "    second = obj.key.split('-')[10]\n",
+    "\n",
+    "    object = s3.Object('hdci-ambt-wallpad-raw',s3_key)\n",
+    "    bodystream = object.get()['Body'].read().decode(\"utf-8\")\n",
+    "    try :\n",
+    "        for obj in bodystream.split('\\n') :\n",
+    "            if obj != None :\n",
+    "                pre_proecssing(json.loads(obj))\n",
+    "    except ValueError as e:\n",
+    "        print (e)\n",
+    "    \n",
+    "    object = s3.Object('hdci-ambt-wallpad-prep','test/metering/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
+    "    object.put(Body=metering_data)\n",
+    "    object = s3.Object('hdci-ambt-wallpad-prep','test/temperature/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
+    "    object.put(Body=temperature_data)\n",
+    "    object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
+    "    object.put(Body=energymeter_data)\n",
+    "    object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/light/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
+    "    for obj in energymeter_light_list :\n",
+    "        energymeter_light_data += json.dumps(obj) + '\\n'\n",
+    "    object.put(Body=energymeter_light_data)\n",
+    "    object = s3.Object('hdci-ambt-wallpad-prep','test/energymeter/outlet/year={}/month={}/day={}/hour={}/ambt-wallpad-prep-dev-{}-{}-{}-{}-{}-{}-{}'.format(year, month, day, hour, year, month, day, hour, minute, second, uuid.uuid4()))\n",
+    "    for obj in energymeter_outlet_list :\n",
+    "        energymeter_outlet_data += json.dumps(obj) + '\\n'\n",
+    "    object.put(Body=energymeter_outlet_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}