Parcourir la source

feat : icos 로컬파일 S3로 1회 전송하는 모듈 업로드

Gogs il y a 3 ans
Parent
commit
e45385eb46
1 fichiers modifiés avec 423 ajouts et 0 suppressions
  1. 423 0
      ambt-icos-raw-to-s3.ipynb

+ 423 - 0
ambt-icos-raw-to-s3.ipynb

@@ -0,0 +1,423 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "from datetime import datetime\n",
+    "import boto3\n",
+    "from io import StringIO\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\icontrols\\icos_proto_data\n"
+     ]
+    }
+   ],
+   "source": [
+    "os.chdir(\"C:/Users/icontrols/icos_proto_data\")\n",
+    "print(os.getcwd())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BUCKET_NAME = 'hdci-ambt-icos-raw'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def save_data(dt, data):\n",
+    "    csv_buffer = StringIO()\n",
+    "    data.to_csv(csv_buffer, encoding='cp949', index=False)\n",
+    "    \n",
+    "    s3 = boto3.resource(\"s3\")\n",
+    "\n",
+    "    bucket_name = BUCKET_NAME\n",
+    "    key = \"dev/site_name=ipark_tower/year={}/month={}/day={}/icos_{}.csv\".format(dt.strftime(\"%Y\"), dt.strftime(\"%m\"),\n",
+    "                                                                  dt.strftime(\"%d\"), dt.strftime(\"%Y%m%d\"))\n",
+    "    target_object = s3.Object(bucket_name, key)\n",
+    "    \n",
+    "    target_object.put(Body=csv_buffer.getvalue())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2020-10-21 00:00:00\n",
+      "2020-10-22 00:00:00\n",
+      "2020-10-23 00:00:00\n",
+      "2020-10-24 00:00:00\n",
+      "2020-10-25 00:00:00\n",
+      "2020-10-26 00:00:00\n",
+      "2020-10-27 00:00:00\n",
+      "2020-10-28 00:00:00\n",
+      "2020-10-29 00:00:00\n",
+      "2020-10-30 00:00:00\n",
+      "2020-10-31 00:00:00\n",
+      "2020-11-01 00:00:00\n",
+      "2020-11-02 00:00:00\n",
+      "2020-11-03 00:00:00\n",
+      "2020-11-04 00:00:00\n",
+      "2020-11-05 00:00:00\n",
+      "2020-11-06 00:00:00\n",
+      "2020-11-07 00:00:00\n",
+      "2020-11-08 00:00:00\n",
+      "2020-11-09 00:00:00\n",
+      "2020-11-10 00:00:00\n",
+      "2020-11-11 00:00:00\n",
+      "2020-11-12 00:00:00\n",
+      "2020-11-13 00:00:00\n",
+      "2020-11-14 00:00:00\n",
+      "2020-11-15 00:00:00\n",
+      "2020-11-16 00:00:00\n",
+      "2020-11-17 00:00:00\n",
+      "2020-11-18 00:00:00\n",
+      "2020-11-19 00:00:00\n",
+      "2020-11-20 00:00:00\n",
+      "2020-11-21 00:00:00\n",
+      "2020-11-22 00:00:00\n",
+      "2020-11-23 00:00:00\n",
+      "2020-11-24 00:00:00\n",
+      "2020-11-25 00:00:00\n",
+      "2020-11-26 00:00:00\n",
+      "2020-11-27 00:00:00\n",
+      "2020-11-28 00:00:00\n",
+      "2020-11-29 00:00:00\n",
+      "2020-11-30 00:00:00\n",
+      "2020-12-01 00:00:00\n",
+      "2020-12-02 00:00:00\n",
+      "2020-12-03 00:00:00\n",
+      "2020-12-04 00:00:00\n",
+      "2020-12-05 00:00:00\n",
+      "2020-12-06 00:00:00\n",
+      "2020-12-07 00:00:00\n",
+      "2020-12-08 00:00:00\n",
+      "2020-12-09 00:00:00\n",
+      "2020-12-10 00:00:00\n",
+      "2020-12-11 00:00:00\n",
+      "2020-12-12 00:00:00\n",
+      "2020-12-13 00:00:00\n",
+      "2020-12-14 00:00:00\n",
+      "2020-12-15 00:00:00\n",
+      "2020-12-16 00:00:00\n",
+      "2020-12-17 00:00:00\n",
+      "2020-12-18 00:00:00\n",
+      "2020-12-19 00:00:00\n",
+      "2020-12-20 00:00:00\n",
+      "2020-12-21 00:00:00\n",
+      "2020-12-22 00:00:00\n",
+      "2020-12-23 00:00:00\n",
+      "2020-12-24 00:00:00\n",
+      "2020-12-25 00:00:00\n",
+      "2020-12-26 00:00:00\n",
+      "2020-12-27 00:00:00\n",
+      "2020-12-28 00:00:00\n",
+      "2020-12-29 00:00:00\n",
+      "2020-12-30 00:00:00\n",
+      "2020-12-31 00:00:00\n",
+      "2021-01-01 00:00:00\n",
+      "2021-01-02 00:00:00\n",
+      "2021-01-03 00:00:00\n",
+      "2021-01-04 00:00:00\n",
+      "2021-01-05 00:00:00\n",
+      "2021-01-06 00:00:00\n",
+      "2021-01-07 00:00:00\n",
+      "2021-01-08 00:00:00\n",
+      "2021-01-09 00:00:00\n",
+      "2021-01-10 00:00:00\n",
+      "2021-01-11 00:00:00\n",
+      "2021-01-12 00:00:00\n",
+      "2021-01-13 00:00:00\n",
+      "2021-01-14 00:00:00\n",
+      "2021-01-15 00:00:00\n",
+      "2021-01-16 00:00:00\n",
+      "2021-01-17 00:00:00\n",
+      "2021-01-18 00:00:00\n",
+      "2021-01-19 00:00:00\n",
+      "2021-01-20 00:00:00\n",
+      "2021-01-21 00:00:00\n",
+      "2021-01-22 00:00:00\n",
+      "2021-01-23 00:00:00\n",
+      "2021-01-24 00:00:00\n",
+      "2021-01-25 00:00:00\n",
+      "2021-01-26 00:00:00\n",
+      "2021-01-27 00:00:00\n",
+      "2021-01-28 00:00:00\n",
+      "2021-01-29 00:00:00\n",
+      "2021-01-30 00:00:00\n",
+      "2021-01-31 00:00:00\n",
+      "2021-02-01 00:00:00\n",
+      "2021-02-02 00:00:00\n",
+      "2021-02-03 00:00:00\n",
+      "2021-02-04 00:00:00\n",
+      "2021-02-05 00:00:00\n",
+      "2021-02-06 00:00:00\n",
+      "2021-02-07 00:00:00\n",
+      "2021-02-08 00:00:00\n",
+      "2021-02-09 00:00:00\n",
+      "2021-02-10 00:00:00\n",
+      "2021-02-11 00:00:00\n",
+      "2021-02-12 00:00:00\n",
+      "2021-02-13 00:00:00\n",
+      "2021-02-14 00:00:00\n",
+      "2021-02-15 00:00:00\n",
+      "2021-02-16 00:00:00\n",
+      "2021-02-17 00:00:00\n",
+      "2021-02-18 00:00:00\n",
+      "2021-02-19 00:00:00\n",
+      "2021-02-20 00:00:00\n",
+      "2021-02-21 00:00:00\n",
+      "2021-02-22 00:00:00\n",
+      "2021-02-23 00:00:00\n",
+      "2021-02-24 00:00:00\n",
+      "2021-02-25 00:00:00\n",
+      "2021-02-26 00:00:00\n",
+      "2021-02-27 00:00:00\n",
+      "2021-02-28 00:00:00\n",
+      "2021-03-01 00:00:00\n",
+      "2021-03-02 00:00:00\n",
+      "2021-03-03 00:00:00\n",
+      "2021-03-04 00:00:00\n",
+      "2021-03-05 00:00:00\n",
+      "2021-03-06 00:00:00\n",
+      "2021-03-07 00:00:00\n",
+      "2021-03-08 00:00:00\n",
+      "2021-03-09 00:00:00\n",
+      "2021-03-10 00:00:00\n",
+      "2021-03-11 00:00:00\n",
+      "2021-03-12 00:00:00\n",
+      "2021-03-13 00:00:00\n",
+      "2021-03-14 00:00:00\n",
+      "2021-03-15 00:00:00\n",
+      "2021-03-16 00:00:00\n",
+      "2021-03-17 00:00:00\n",
+      "2021-03-18 00:00:00\n",
+      "2021-03-19 00:00:00\n",
+      "2021-03-20 00:00:00\n",
+      "2021-03-21 00:00:00\n",
+      "2021-03-22 00:00:00\n",
+      "2021-03-23 00:00:00\n",
+      "2021-03-24 00:00:00\n",
+      "2021-03-25 00:00:00\n",
+      "2021-03-26 00:00:00\n",
+      "2021-03-27 00:00:00\n",
+      "2021-03-28 00:00:00\n",
+      "2021-03-29 00:00:00\n",
+      "2021-03-30 00:00:00\n",
+      "2021-03-31 00:00:00\n",
+      "2021-04-01 00:00:00\n",
+      "2021-04-02 00:00:00\n",
+      "2021-04-03 00:00:00\n",
+      "2021-04-04 00:00:00\n",
+      "2021-04-05 00:00:00\n",
+      "2021-04-06 00:00:00\n",
+      "2021-04-07 00:00:00\n",
+      "2021-04-08 00:00:00\n",
+      "2021-04-09 00:00:00\n",
+      "2021-04-10 00:00:00\n",
+      "2021-04-11 00:00:00\n",
+      "2021-04-12 00:00:00\n",
+      "2021-04-13 00:00:00\n",
+      "2021-04-14 00:00:00\n",
+      "2021-04-15 00:00:00\n",
+      "2021-04-16 00:00:00\n",
+      "2021-04-17 00:00:00\n",
+      "2021-04-18 00:00:00\n",
+      "2021-04-19 00:00:00\n",
+      "2021-04-20 00:00:00\n",
+      "2021-04-21 00:00:00\n",
+      "2021-04-22 00:00:00\n",
+      "2021-04-23 00:00:00\n",
+      "2021-04-24 00:00:00\n",
+      "2021-04-25 00:00:00\n",
+      "2021-04-26 00:00:00\n",
+      "2021-04-27 00:00:00\n",
+      "2021-04-28 00:00:00\n",
+      "2021-04-29 00:00:00\n",
+      "2021-04-30 00:00:00\n",
+      "2021-05-01 00:00:00\n",
+      "2021-05-02 00:00:00\n",
+      "2021-05-03 00:00:00\n",
+      "2021-05-04 00:00:00\n",
+      "2021-05-05 00:00:00\n",
+      "2021-05-06 00:00:00\n",
+      "2021-05-07 00:00:00\n",
+      "2021-05-08 00:00:00\n",
+      "2021-05-09 00:00:00\n",
+      "2021-05-10 00:00:00\n",
+      "2021-05-11 00:00:00\n",
+      "2021-05-12 00:00:00\n",
+      "2021-05-13 00:00:00\n",
+      "2021-05-14 00:00:00\n",
+      "2021-05-15 00:00:00\n",
+      "2021-05-16 00:00:00\n",
+      "2021-05-17 00:00:00\n",
+      "2021-05-18 00:00:00\n",
+      "2021-05-19 00:00:00\n",
+      "2021-05-20 00:00:00\n",
+      "2021-05-21 00:00:00\n",
+      "2021-05-22 00:00:00\n",
+      "2021-05-23 00:00:00\n",
+      "2021-05-24 00:00:00\n",
+      "2021-05-25 00:00:00\n",
+      "2021-05-26 00:00:00\n",
+      "2021-05-27 00:00:00\n",
+      "2021-05-28 00:00:00\n",
+      "2021-05-29 00:00:00\n",
+      "2021-05-30 00:00:00\n",
+      "2021-05-31 00:00:00\n",
+      "2021-06-01 00:00:00\n",
+      "2021-06-02 00:00:00\n",
+      "2021-06-03 00:00:00\n",
+      "2021-06-04 00:00:00\n",
+      "2021-06-05 00:00:00\n",
+      "2021-06-06 00:00:00\n",
+      "2021-06-07 00:00:00\n",
+      "2021-06-08 00:00:00\n",
+      "2021-06-09 00:00:00\n",
+      "2021-06-10 00:00:00\n",
+      "2021-06-11 00:00:00\n",
+      "2021-06-12 00:00:00\n",
+      "2021-06-13 00:00:00\n",
+      "2021-06-14 00:00:00\n",
+      "2021-06-15 00:00:00\n",
+      "2021-06-16 00:00:00\n",
+      "2021-06-17 00:00:00\n",
+      "2021-06-18 00:00:00\n",
+      "2021-06-19 00:00:00\n",
+      "2021-06-20 00:00:00\n",
+      "2021-06-21 00:00:00\n"
+     ]
+    }
+   ],
+   "source": [
+    "for idx, txt in enumerate(os.listdir()):\n",
+    "    dt = datetime.strptime(os.listdir()[idx].split('.')[0], \"%Y%m%d\")\n",
+    "    df = pd.read_table(txt, encoding='cp949', index_col=False)\n",
+    "    df.rename(columns={\"시각\":\"time\"}, inplace = True)\n",
+    "    df['cnt'] = len(df)\n",
+    "    save_data(dt, df)\n",
+    "    print(dt)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "datetime.datetime(2021, 6, 21, 0, 0)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dt = datetime.strptime(os.listdir()[-1].split('.')[0], \"%Y%m%d\")\n",
+    "dt\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  },
+  "varInspector": {
+   "cols": {
+    "lenName": 16,
+    "lenType": 16,
+    "lenVar": 40
+   },
+   "kernels_config": {
+    "python": {
+     "delete_cmd_postfix": "",
+     "delete_cmd_prefix": "del ",
+     "library": "var_list.py",
+     "varRefreshCmd": "print(var_dic_list())"
+    },
+    "r": {
+     "delete_cmd_postfix": ") ",
+     "delete_cmd_prefix": "rm(",
+     "library": "var_list.r",
+     "varRefreshCmd": "cat(var_dic_list()) "
+    }
+   },
+   "types_to_exclude": [
+    "module",
+    "function",
+    "builtin_function_or_method",
+    "instance",
+    "_Feature"
+   ],
+   "window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}