{ "cells": [ { "cell_type": "code", "execution_count": 22, "id": "cded2e4d", "metadata": {}, "outputs": [], "source": [ "import boto3\n", "import json\n", "import uuid\n", "from datetime import datetime" ] }, { "cell_type": "code", "execution_count": 23, "id": "9e8e58f3", "metadata": {}, "outputs": [], "source": [ "site = {'yeoju':0, 'sujigwanggyosan':1, 'rnd_dev':2 }\n", "s3 = boto3.resource('s3')" ] }, { "cell_type": "code", "execution_count": 24, "id": "2a781ebf", "metadata": {}, "outputs": [], "source": [ "s3BucketName = 'hdci-ambt-wallpad-raw'" ] }, { "cell_type": "code", "execution_count": 25, "id": "729af248", "metadata": {}, "outputs": [], "source": [ "bucket = s3.Bucket(s3BucketName)" ] }, { "cell_type": "code", "execution_count": 26, "id": "e35f6865", "metadata": {}, "outputs": [], "source": [ "for object in bucket.objects.filter(Prefix='type=test/'):\n", " jsondata = {}\n", " for i in site :\n", " jsondata[site[i]] = \"\"\n", " key = object.key\n", " if not key.endswith('/'):\n", " object = s3.Object(s3BucketName,key)\n", " bodystream = object.get()['Body'].read().decode(\"utf-8\")\n", " try :\n", " for payload in range(0, len(bodystream.split('\\n'))-1) :\n", " jsonpayload = json.loads(bodystream.split('\\n')[payload])\n", " year = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").year\n", " month = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").month\n", " day = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").day\n", " hour = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").hour\n", " minute = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").minute\n", " second = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").second\n", " jsonpayload['site'] = jsonpayload['site'] if jsonpayload.get('site') != None else 'yeoju' if jsonpayload['dong'] == '105' else 'sujigwanggyosan'\n", " jsondata[site[jsonpayload['site']]] += json.dumps(jsonpayload) + '\\n' \n", " for i in site:\n", " if len(jsondata[site[i]]) != 0:\n", " object = s3.Object('hdci-ambt-wallpad-raw','dev/{}/year={}/month={}/day={}/ambt-preden-lambda-consumer-dev-{}-{}-{}-{}-{}-{}-{}'.format(i, str(year), str(month), str(day), str(year), str(month), str(day), str(hour), str(minute), str(second), str(uuid.uuid4()))) \n", " object.put(Body=jsondata.get(site[i]))\n", " #s3.Object(s3BucketName,key).delete()\n", " except ValueError as e:\n", " print (e)" ] }, { "cell_type": "code", "execution_count": null, "id": "28127d8e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.13" } }, "nbformat": 4, "nbformat_minor": 5 }