|
@@ -0,0 +1,112 @@
|
|
|
|
+{
|
|
|
|
+ "cells": [
|
|
|
|
+ {
|
|
|
|
+ "cell_type": "code",
|
|
|
|
+ "execution_count": 13,
|
|
|
|
+ "id": "d8aaf170",
|
|
|
|
+ "metadata": {},
|
|
|
|
+ "outputs": [],
|
|
|
|
+ "source": [
|
|
|
|
+ "import boto3\n",
|
|
|
|
+ "import json\n",
|
|
|
|
+ "import uuid\n",
|
|
|
|
+ "from datetime import datetime"
|
|
|
|
+ ]
|
|
|
|
+ },
|
|
|
|
+ {
|
|
|
|
+ "cell_type": "code",
|
|
|
|
+ "execution_count": 14,
|
|
|
|
+ "id": "17335ed0",
|
|
|
|
+ "metadata": {},
|
|
|
|
+ "outputs": [],
|
|
|
|
+ "source": [
|
|
|
|
+ "site = {'yeoju':0, 'sujigwanggyosan':1, 'rnd_dev':2 }\n",
|
|
|
|
+ "s3 = boto3.resource('s3')"
|
|
|
|
+ ]
|
|
|
|
+ },
|
|
|
|
+ {
|
|
|
|
+ "cell_type": "code",
|
|
|
|
+ "execution_count": 15,
|
|
|
|
+ "id": "199da9f2",
|
|
|
|
+ "metadata": {},
|
|
|
|
+ "outputs": [],
|
|
|
|
+ "source": [
|
|
|
|
+ "s3BucketName = 'hdci-ambt-wallpad-raw'"
|
|
|
|
+ ]
|
|
|
|
+ },
|
|
|
|
+ {
|
|
|
|
+ "cell_type": "code",
|
|
|
|
+ "execution_count": 16,
|
|
|
|
+ "id": "07eab849",
|
|
|
|
+ "metadata": {},
|
|
|
|
+ "outputs": [],
|
|
|
|
+ "source": [
|
|
|
|
+ "bucket = s3.Bucket(s3BucketName)"
|
|
|
|
+ ]
|
|
|
|
+ },
|
|
|
|
+ {
|
|
|
|
+ "cell_type": "code",
|
|
|
|
+ "execution_count": 17,
|
|
|
|
+ "id": "752bff31",
|
|
|
|
+ "metadata": {},
|
|
|
|
+ "outputs": [],
|
|
|
|
+ "source": [
|
|
|
|
+ "jsondata = {}\n",
|
|
|
|
+ "for i in site :\n",
|
|
|
|
+ " jsondata[site[i]] = \"\""
|
|
|
|
+ ]
|
|
|
|
+ },
|
|
|
|
+ {
|
|
|
|
+ "cell_type": "code",
|
|
|
|
+ "execution_count": 18,
|
|
|
|
+ "id": "1eafb250",
|
|
|
|
+ "metadata": {},
|
|
|
|
+ "outputs": [],
|
|
|
|
+ "source": [
|
|
|
|
+ "for object in bucket.objects.filter(Prefix='dev/year=2021'):\n",
|
|
|
|
+ " key = object.key\n",
|
|
|
|
+ " if not key.endswith('/'):\n",
|
|
|
|
+ " object = s3.Object(s3BucketName,key)\n",
|
|
|
|
+ " bodystream = object.get()['Body'].read().decode(\"utf-8\")\n",
|
|
|
|
+ " try :\n",
|
|
|
|
+ " for payload in range(0, len(bodystream.split('\\n'))-1) :\n",
|
|
|
|
+ " jsonpayload = json.loads(bodystream.split('\\n')[payload])\n",
|
|
|
|
+ " year = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").year\n",
|
|
|
|
+ " month = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").month\n",
|
|
|
|
+ " day = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").day\n",
|
|
|
|
+ " hour = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").hour\n",
|
|
|
|
+ " minute = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").minute\n",
|
|
|
|
+ " second = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").second\n",
|
|
|
|
+ " jsonpayload['site'] = jsonpayload['site'] if jsonpayload.get('site') != None else 'yeoju' if jsonpayload['dong'] == '105' else 'sujigwanggyosan'\n",
|
|
|
|
+ " jsondata[site[jsonpayload['site']]] += json.dumps(jsonpayload) + '\\n' \n",
|
|
|
|
+ " for i in site:\n",
|
|
|
|
+ " if len(jsondata[site[i]]) != 0:\n",
|
|
|
|
+ " object = s3.Object('hdci-ambt-wallpad-raw','test3/{}/year={}/month={}/day={}/ambt-preden-lambda-consumer-dev-{}-{}-{}-{}-{}-{}-{}'.format(i, str(year), str(month), str(day), str(year), str(month), str(day), str(hour), str(minute), str(second), str(uuid.uuid4()))) \n",
|
|
|
|
+ " object.put(Body=jsondata.get(site[i]))\n",
|
|
|
|
+ " except ValueError as e:\n",
|
|
|
|
+ " print (e)"
|
|
|
|
+ ]
|
|
|
|
+ }
|
|
|
|
+ ],
|
|
|
|
+ "metadata": {
|
|
|
|
+ "kernelspec": {
|
|
|
|
+ "display_name": "conda_python3",
|
|
|
|
+ "language": "python",
|
|
|
|
+ "name": "conda_python3"
|
|
|
|
+ },
|
|
|
|
+ "language_info": {
|
|
|
|
+ "codemirror_mode": {
|
|
|
|
+ "name": "ipython",
|
|
|
|
+ "version": 3
|
|
|
|
+ },
|
|
|
|
+ "file_extension": ".py",
|
|
|
|
+ "mimetype": "text/x-python",
|
|
|
|
+ "name": "python",
|
|
|
|
+ "nbconvert_exporter": "python",
|
|
|
|
+ "pygments_lexer": "ipython3",
|
|
|
|
+ "version": "3.6.13"
|
|
|
|
+ }
|
|
|
|
+ },
|
|
|
|
+ "nbformat": 4,
|
|
|
|
+ "nbformat_minor": 5
|
|
|
|
+}
|