|  | @@ -0,0 +1,265 @@
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | + "cells": [
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "!pip install pymysql"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "import sys\n",
 | 
	
		
			
				|  |  | +    "import pymysql\n",
 | 
	
		
			
				|  |  | +    "import json\n",
 | 
	
		
			
				|  |  | +    "import decimal\n",
 | 
	
		
			
				|  |  | +    "import boto3\n",
 | 
	
		
			
				|  |  | +    "import uuid\n",
 | 
	
		
			
				|  |  | +    "import logging\n",
 | 
	
		
			
				|  |  | +    "import os\n",
 | 
	
		
			
				|  |  | +    "import time\n",
 | 
	
		
			
				|  |  | +    "from datetime import datetime, timedelta, date\n",
 | 
	
		
			
				|  |  | +    "from dateutil.relativedelta import relativedelta\n",
 | 
	
		
			
				|  |  | +    "from botocore.exceptions import ClientError"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "class DecimalnDateTimeEncoder(json.JSONEncoder):\n",
 | 
	
		
			
				|  |  | +    "    def default(self, obj):\n",
 | 
	
		
			
				|  |  | +    "        if isinstance(obj, decimal.Decimal) or isinstance(obj, datetime):\n",
 | 
	
		
			
				|  |  | +    "            return str(obj)\n",
 | 
	
		
			
				|  |  | +    "        return json.JSONEncoder.default(self, obj)"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "def get_secret():\n",
 | 
	
		
			
				|  |  | +    "    secret_name = \"ambt-preden-gurigalmae-valleydb\"\n",
 | 
	
		
			
				|  |  | +    "    region_name = \"ap-northeast-2\"\n",
 | 
	
		
			
				|  |  | +    "\n",
 | 
	
		
			
				|  |  | +    "    # Create a Secrets Manager client\n",
 | 
	
		
			
				|  |  | +    "    session = boto3.session.Session()\n",
 | 
	
		
			
				|  |  | +    "    client = session.client(\n",
 | 
	
		
			
				|  |  | +    "        service_name='secretsmanager',\n",
 | 
	
		
			
				|  |  | +    "        region_name=region_name\n",
 | 
	
		
			
				|  |  | +    "    )\n",
 | 
	
		
			
				|  |  | +    "\n",
 | 
	
		
			
				|  |  | +    "    # In this sample we only handle the specific exceptions for the 'GetSecretValue' API.\n",
 | 
	
		
			
				|  |  | +    "    # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html\n",
 | 
	
		
			
				|  |  | +    "    # We rethrow the exception by default.\n",
 | 
	
		
			
				|  |  | +    "\n",
 | 
	
		
			
				|  |  | +    "    try:\n",
 | 
	
		
			
				|  |  | +    "        get_secret_value_response = client.get_secret_value(\n",
 | 
	
		
			
				|  |  | +    "            SecretId=secret_name\n",
 | 
	
		
			
				|  |  | +    "        )\n",
 | 
	
		
			
				|  |  | +    "    except ClientError as e:\n",
 | 
	
		
			
				|  |  | +    "        if e.response['Error']['Code'] == 'DecryptionFailureException':\n",
 | 
	
		
			
				|  |  | +    "            # Secrets Manager can't decrypt the protected secret text using the provided KMS key.\n",
 | 
	
		
			
				|  |  | +    "            # Deal with the exception here, and/or rethrow at your discretion.\n",
 | 
	
		
			
				|  |  | +    "            raise e\n",
 | 
	
		
			
				|  |  | +    "        elif e.response['Error']['Code'] == 'InternalServiceErrorException':\n",
 | 
	
		
			
				|  |  | +    "            # An error occurred on the server side.\n",
 | 
	
		
			
				|  |  | +    "            # Deal with the exception here, and/or rethrow at your discretion.\n",
 | 
	
		
			
				|  |  | +    "            raise e\n",
 | 
	
		
			
				|  |  | +    "        elif e.response['Error']['Code'] == 'InvalidParameterException':\n",
 | 
	
		
			
				|  |  | +    "            # You provided an invalid value for a parameter.\n",
 | 
	
		
			
				|  |  | +    "            # Deal with the exception here, and/or rethrow at your discretion.\n",
 | 
	
		
			
				|  |  | +    "            raise e\n",
 | 
	
		
			
				|  |  | +    "        elif e.response['Error']['Code'] == 'InvalidRequestException':\n",
 | 
	
		
			
				|  |  | +    "            # You provided a parameter value that is not valid for the current state of the resource.\n",
 | 
	
		
			
				|  |  | +    "            # Deal with the exception here, and/or rethrow at your discretion.\n",
 | 
	
		
			
				|  |  | +    "            raise e\n",
 | 
	
		
			
				|  |  | +    "        elif e.response['Error']['Code'] == 'ResourceNotFoundException':\n",
 | 
	
		
			
				|  |  | +    "            # We can't find the resource that you asked for.\n",
 | 
	
		
			
				|  |  | +    "            # Deal with the exception here, and/or rethrow at your discretion.\n",
 | 
	
		
			
				|  |  | +    "            raise e\n",
 | 
	
		
			
				|  |  | +    "    else:\n",
 | 
	
		
			
				|  |  | +    "        # Decrypts secret using the associated KMS CMK.\n",
 | 
	
		
			
				|  |  | +    "        # Depending on whether the secret is a string or binary, one of these fields will be populated.\n",
 | 
	
		
			
				|  |  | +    "        if 'SecretString' in get_secret_value_response:\n",
 | 
	
		
			
				|  |  | +    "            secret = get_secret_value_response['SecretString']\n",
 | 
	
		
			
				|  |  | +    "        return json.loads(secret)"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "os.environ['TZ'] = 'Asia/Seoul'\n",
 | 
	
		
			
				|  |  | +    "time.tzset()\n",
 | 
	
		
			
				|  |  | +    "\n",
 | 
	
		
			
				|  |  | +    "logger = logging.getLogger()\n",
 | 
	
		
			
				|  |  | +    "logger.setLevel(logging.INFO)"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "secret = get_secret()"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "try:\n",
 | 
	
		
			
				|  |  | +    "    conn = pymysql.connect(host=secret['host'], port=int(secret['port']), user=secret['username'], passwd=secret['password'], db=secret['dbname'], connect_timeout=5)\n",
 | 
	
		
			
				|  |  | +    "except:\n",
 | 
	
		
			
				|  |  | +    "    logger.error(\"ERROR: Unexpected error: Could not connect to MySql instance.\")\n",
 | 
	
		
			
				|  |  | +    "    sys.exit()\n",
 | 
	
		
			
				|  |  | +    "logger.info(\"SUCCESS: Connection to RDS mysql instance succeeded\")"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "s3=boto3.resource('s3')"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "sql = \"SELECT CREATED_AT FROM DAYENERGY ORDER BY CREATED_AT ASC LIMIT 1\"\n",
 | 
	
		
			
				|  |  | +    "with conn.cursor(pymysql.cursors.DictCursor) as cur:\n",
 | 
	
		
			
				|  |  | +    "    cur.execute(sql)\n",
 | 
	
		
			
				|  |  | +    "    logger.info(sql)\n",
 | 
	
		
			
				|  |  | +    "    row = cur.fetchone()"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "oldestdate = row['CREATED_AT']"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {
 | 
	
		
			
				|  |  | +    "scrolled": true
 | 
	
		
			
				|  |  | +   },
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "while oldestdate.date() != date.today() :\n",
 | 
	
		
			
				|  |  | +    "    sql = \"SELECT * FROM DAYENERGY WHERE ENERGY_YEAR={} AND ENERGY_MONTH={} AND ENERGY_DAY={}\".format(oldestdate.year, oldestdate.month, oldestdate.day)\n",
 | 
	
		
			
				|  |  | +    "    print (sql)\n",
 | 
	
		
			
				|  |  | +    "    with conn.cursor(pymysql.cursors.DictCursor) as cur:\n",
 | 
	
		
			
				|  |  | +    "        cur.execute(sql)\n",
 | 
	
		
			
				|  |  | +    "        rows = cur.fetchall()\n",
 | 
	
		
			
				|  |  | +    "    data = \"\"\n",
 | 
	
		
			
				|  |  | +    "    for row in rows:\n",
 | 
	
		
			
				|  |  | +    "        data += json.dumps(row, cls=DecimalnDateTimeEncoder) + '\\n'\n",
 | 
	
		
			
				|  |  | +    "        #logger.info(row)\n",
 | 
	
		
			
				|  |  | +    "    object = s3.Object('hdci-ambt-homenetserver-raw','dev/site_name=gurigalmae/table_name=dayenergy/year={}/month={}/ambt-preden-lambda-migration-dev-{}-{}-{}-{}'.format(oldestdate.year, oldestdate.month, oldestdate.year, oldestdate.month, oldestdate.day, str(uuid.uuid4())))\n",
 | 
	
		
			
				|  |  | +    "    object.put(Body=data)\n",
 | 
	
		
			
				|  |  | +    "    oldestdate += timedelta(days=1)"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "sql = \"SELECT CREATED_AT FROM MONTHENERGY ORDER BY CREATED_AT ASC LIMIT 1\"\n",
 | 
	
		
			
				|  |  | +    "with conn.cursor(pymysql.cursors.DictCursor) as cur:\n",
 | 
	
		
			
				|  |  | +    "    cur.execute(sql)\n",
 | 
	
		
			
				|  |  | +    "    #logger.info(sql)\n",
 | 
	
		
			
				|  |  | +    "    row = cur.fetchone()"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "oldestdate = row['CREATED_AT']"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {},
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "oldestdate.date()"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  {
 | 
	
		
			
				|  |  | +   "cell_type": "code",
 | 
	
		
			
				|  |  | +   "execution_count": null,
 | 
	
		
			
				|  |  | +   "metadata": {
 | 
	
		
			
				|  |  | +    "scrolled": true
 | 
	
		
			
				|  |  | +   },
 | 
	
		
			
				|  |  | +   "outputs": [],
 | 
	
		
			
				|  |  | +   "source": [
 | 
	
		
			
				|  |  | +    "while oldestdate.date().strftime('%Y%m') != date.today().strftime('%Y%m') :\n",
 | 
	
		
			
				|  |  | +    "    sql = \"SELECT * FROM MONTHENERGY WHERE ENERGY_YEAR={} AND ENERGY_MONTH={}\".format(oldestdate.year, oldestdate.month)\n",
 | 
	
		
			
				|  |  | +    "    print (sql)\n",
 | 
	
		
			
				|  |  | +    "    with conn.cursor(pymysql.cursors.DictCursor) as cur:\n",
 | 
	
		
			
				|  |  | +    "        cur.execute(sql)\n",
 | 
	
		
			
				|  |  | +    "        rows = cur.fetchall()\n",
 | 
	
		
			
				|  |  | +    "    data = \"\"\n",
 | 
	
		
			
				|  |  | +    "    for row in rows:\n",
 | 
	
		
			
				|  |  | +    "        data += json.dumps(row, cls=DecimalnDateTimeEncoder) + '\\n'\n",
 | 
	
		
			
				|  |  | +    "        #logger.info(row)\n",
 | 
	
		
			
				|  |  | +    "    object = s3.Object('hdci-ambt-homenetserver-raw','dev/site_name=gurigalmae/table_name=monthenergy/year={}/ambt-preden-lambda-migration-dev-{}-{}-{}'.format(oldestdate.year, oldestdate.year, oldestdate.month, str(uuid.uuid4())))\n",
 | 
	
		
			
				|  |  | +    "    object.put(Body=data)\n",
 | 
	
		
			
				|  |  | +    "    oldestdate += relativedelta(months=1)"
 | 
	
		
			
				|  |  | +   ]
 | 
	
		
			
				|  |  | +  }
 | 
	
		
			
				|  |  | + ],
 | 
	
		
			
				|  |  | + "metadata": {
 | 
	
		
			
				|  |  | +  "kernelspec": {
 | 
	
		
			
				|  |  | +   "display_name": "conda_python3",
 | 
	
		
			
				|  |  | +   "language": "python",
 | 
	
		
			
				|  |  | +   "name": "conda_python3"
 | 
	
		
			
				|  |  | +  },
 | 
	
		
			
				|  |  | +  "language_info": {
 | 
	
		
			
				|  |  | +   "codemirror_mode": {
 | 
	
		
			
				|  |  | +    "name": "ipython",
 | 
	
		
			
				|  |  | +    "version": 3
 | 
	
		
			
				|  |  | +   },
 | 
	
		
			
				|  |  | +   "file_extension": ".py",
 | 
	
		
			
				|  |  | +   "mimetype": "text/x-python",
 | 
	
		
			
				|  |  | +   "name": "python",
 | 
	
		
			
				|  |  | +   "nbconvert_exporter": "python",
 | 
	
		
			
				|  |  | +   "pygments_lexer": "ipython3",
 | 
	
		
			
				|  |  | +   "version": "3.6.13"
 | 
	
		
			
				|  |  | +  }
 | 
	
		
			
				|  |  | + },
 | 
	
		
			
				|  |  | + "nbformat": 4,
 | 
	
		
			
				|  |  | + "nbformat_minor": 4
 | 
	
		
			
				|  |  | +}
 |