Jelajahi Sumber

월패드 RAW 버킷 현장별로 path 구분하기 위한 노트북 추가

heewon 3 tahun lalu
induk
melakukan
be913b1693
1 mengubah file dengan 112 tambahan dan 0 penghapusan
  1. 112 0
      ambt-wallpad-site-transform/ambt-wallpad-site-transform.ipynb

+ 112 - 0
ambt-wallpad-site-transform/ambt-wallpad-site-transform.ipynb

@@ -0,0 +1,112 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "d8aaf170",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import boto3\n",
+    "import json\n",
+    "import uuid\n",
+    "from datetime import datetime"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "17335ed0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "site = {'yeoju':0, 'sujigwanggyosan':1, 'rnd_dev':2 }\n",
+    "s3 = boto3.resource('s3')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "199da9f2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s3BucketName = 'hdci-ambt-wallpad-raw'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "07eab849",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "bucket = s3.Bucket(s3BucketName)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "752bff31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "jsondata = {}\n",
+    "for i in site :\n",
+    "    jsondata[site[i]] = \"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "1eafb250",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for object in bucket.objects.filter(Prefix='dev/year=2021'):\n",
+    "    key = object.key\n",
+    "    if not key.endswith('/'):\n",
+    "        object = s3.Object(s3BucketName,key)\n",
+    "        bodystream = object.get()['Body'].read().decode(\"utf-8\")\n",
+    "        try :\n",
+    "            for payload in range(0, len(bodystream.split('\\n'))-1) :\n",
+    "                jsonpayload = json.loads(bodystream.split('\\n')[payload])\n",
+    "                year = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").year\n",
+    "                month = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").month\n",
+    "                day = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").day\n",
+    "                hour = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").hour\n",
+    "                minute = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").minute\n",
+    "                second = datetime.strptime(jsonpayload['timestamp'], \"%Y-%m-%dT%H:%M:%S\").second\n",
+    "                jsonpayload['site'] = jsonpayload['site'] if jsonpayload.get('site') != None else 'yeoju' if jsonpayload['dong'] == '105' else 'sujigwanggyosan'\n",
+    "                jsondata[site[jsonpayload['site']]] += json.dumps(jsonpayload) + '\\n'                \n",
+    "            for i in site:\n",
+    "                if len(jsondata[site[i]]) != 0:\n",
+    "                    object = s3.Object('hdci-ambt-wallpad-raw','test3/{}/year={}/month={}/day={}/ambt-preden-lambda-consumer-dev-{}-{}-{}-{}-{}-{}-{}'.format(i, str(year), str(month), str(day), str(year), str(month), str(day), str(hour), str(minute), str(second), str(uuid.uuid4())))                    \n",
+    "                    object.put(Body=jsondata.get(site[i]))\n",
+    "        except ValueError as e:\n",
+    "            print (e)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "conda_python3",
+   "language": "python",
+   "name": "conda_python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}