ambt-wallpad-site-transform.ipynb 3.5 KB

import boto3
import json
import uuid
from datetime import datetime
site = {'yeoju':0, 'sujigwanggyosan':1, 'rnd_dev':2 }
s3 = boto3.resource('s3')
s3BucketName = 'hdci-ambt-wallpad-raw'
bucket = s3.Bucket(s3BucketName)
jsondata = {}
for i in site :
    jsondata[site[i]] = ""
for object in bucket.objects.filter(Prefix='dev/year=2021'):
    key = object.key
    if not key.endswith('/'):
        object = s3.Object(s3BucketName,key)
        bodystream = object.get()['Body'].read().decode("utf-8")
        try :
            for payload in range(0, len(bodystream.split('\n'))-1) :
                jsonpayload = json.loads(bodystream.split('\n')[payload])
                year = datetime.strptime(jsonpayload['timestamp'], "%Y-%m-%dT%H:%M:%S").year
                month = datetime.strptime(jsonpayload['timestamp'], "%Y-%m-%dT%H:%M:%S").month
                day = datetime.strptime(jsonpayload['timestamp'], "%Y-%m-%dT%H:%M:%S").day
                hour = datetime.strptime(jsonpayload['timestamp'], "%Y-%m-%dT%H:%M:%S").hour
                minute = datetime.strptime(jsonpayload['timestamp'], "%Y-%m-%dT%H:%M:%S").minute
                second = datetime.strptime(jsonpayload['timestamp'], "%Y-%m-%dT%H:%M:%S").second
                jsonpayload['site'] = jsonpayload['site'] if jsonpayload.get('site') != None else 'yeoju' if jsonpayload['dong'] == '105' else 'sujigwanggyosan'
                jsondata[site[jsonpayload['site']]] += json.dumps(jsonpayload) + '\n'                
            for i in site:
                if len(jsondata[site[i]]) != 0:
                    object = s3.Object('hdci-ambt-wallpad-raw','test3/{}/year={}/month={}/day={}/ambt-preden-lambda-consumer-dev-{}-{}-{}-{}-{}-{}-{}'.format(i, str(year), str(month), str(day), str(year), str(month), str(day), str(hour), str(minute), str(second), str(uuid.uuid4())))                    
                    object.put(Body=jsondata.get(site[i]))
        except ValueError as e:
            print (e)