Added 3 files(app.py, sales_data.csv, transformed_data.csv, runtime_log.txt, .github/workflows/etl_pipeline.yaml)

Author: SridharanA

.github/workflows/etl_pipeline.yaml

@@ -0,0 +1,31 @@
+name: ETL Pipeline Automation
+
+on:
+  push:
+    brances:
+      - main
+
+jobs:
+  etl_pipeline_job:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      
+      - name: Install Libraries or Dependencies
+        run: pip install streamlit pandas datetime
+      
+      - name: Run the python script
+        run: python app.py
+      
+      - name: Push back the changes to the files to the github repo
+        uses: mikeal/push-to-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          branch_name: 'main'

app.py

@@ -0,0 +1,69 @@
+import streamlit as st
+import pandas as pd
+import datetime
+
+# Function to load data
+def load_data():
+    data = pd.read_csv('sales_data.csv')
+    return data
+
+# Function to transform data
+def transform_data(df):
+    df['Date'] = pd.to_datetime(df['Date'])
+    df['Year'] = df['Date'].dt.year
+    df['Month'] = df['Date'].dt.month
+    df['Sales_to_Profit_Ratio'] = df['Sales'] / df['Profit']
+    df['Cumulative_Sales'] = df['Sales'].cumsum()
+    return df
+
+# Function to extract summary statistics
+def get_summary_statistics(df):
+    summary = df.describe()
+    return summary
+
+# Define the log file
+log_file = 'runtime_log.txt'
+
+def log_message(message):
+    with open(log_file, 'a') as f:
+        f.write(f'{datetime.datetime.now()} - {message}\n')
+
+# Main function to run the Streamlit app
+def main():
+    st.title("ETL Process for Sales Data")
+    
+    # Step 1: Extract
+    st.header("Step 1: Extract")
+    data = load_data()
+    print(data)
+    print(type(data))   # data - type(<class 'pandas.core.frame.DataFrame'>)
+    st.subheader("Raw Data")
+    st.table(data)
+    df_summary_statistics = get_summary_statistics(data)
+    print(df_summary_statistics)
+    
+    # Step 2: Transform
+    st.header("Step 2: Transform")
+    transformed_data = transform_data(data)
+    print(transformed_data)
+    print(type(transformed_data))   # transformed_data - type(<class 'pandas.core.frame.DataFrame)
+    st.subheader('Transformed Data')
+    st.table(transformed_data)
+    transformed_df_summary_statistics = get_summary_statistics(transformed_data)
+    print(transformed_df_summary_statistics)
+    
+    # Load data
+    st.header("Step 3: Load data")
+    transformed_data.to_csv('tranformed_data.csv', index=False)
+    
+    # Generate Run time log
+    log_message(f'ETL job completed and data saved to transformed_data.csv')
+    
+    # showing the contents in the file in streamlit app
+    with open(log_file, 'r') as f:
+        contents = f.read()
+    
+    st.header('Runtime Logs')
+    st.write(contents)
+    
+main()

runtime_log.txt

@@ -0,0 +1,4 @@
+2025-03-17 10:08:18.021671 - ETL job completed and data saved to transformed_data.csv
+2025-03-17 10:08:32.361232 - ETL job completed and data saved to transformed_data.csv
+2025-03-17 10:09:13.872924 - ETL job completed and data saved to transformed_data.csv
+2025-03-17 10:09:22.577862 - ETL job completed and data saved to transformed_data.csv

sales_data.csv

@@ -0,0 +1,6 @@
+Date,Product,Sales,Profit
+2023-01-01,Product A,100,20
+2023-01-02,Product B,150,30
+2023-01-03,Product A,200,40
+2023-01-04,Product C,250,50
+2023-01-05,Product B,300,60

tranformed_data.csv

@@ -0,0 +1,6 @@
+Date,Product,Sales,Profit,Year,Month,Sales_to_Profit_Ratio,Cumulative_Sales
+2023-01-01,Product A,100,20,2023,1,5.0,100
+2023-01-02,Product B,150,30,2023,1,5.0,250
+2023-01-03,Product A,200,40,2023,1,5.0,450
+2023-01-04,Product C,250,50,2023,1,5.0,700
+2023-01-05,Product B,300,60,2023,1,5.0,1000