How to extract the elements from csv to json in S3

Question

• I need to find the csv files from the folder
• List all the files inside the folder
• Convert files to json and save in the same bucket

Csv fil

Answer 1

You can check the following code:

from io import StringIO

import boto3
import pandas as pd

s3 = boto3.resource('s3')

def lambda_handler(event, context):
    
    s3 = boto3.resource('s3')
    
    input_bucket = 'bucket-with-csv-file-44244'
    
    my_bucket = s3.Bucket(input_bucket)
    
    for file in my_bucket.objects.all():
        
        if file.key.endswith(".csv"):
           
            csv_f = f"s3://{input_bucket}/{file.key}"
            
            print(csv_f)
            
            json_file = file.key.replace(".csv", ".json")
            
            print(json_file)
            
            json_buffer = StringIO()
            
            df = pd.read_csv(csv_f)
            
            df.to_json(json_buffer, orient='index')
            
            s3.Object(input_bucket, json_file).put(Body=json_buffer.getvalue())            

Your lambda layer will need to have:

fsspec
pandas
s3fs