co2_emissions_prediction.py

# -*- coding: utf-8 -*-
"""CO2 Emissions Prediction.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1eO-Sh4eqO8axdXtTZwtWQVBKttwuD-P9

## Topic: CO2 Emissions

### Importing Required Libraries
"""

# Essential 
import numpy as np 
import pandas as pd 

# Visualization
import matplotlib.pyplot as plt
import seaborn as sb

# encoding 
from sklearn.preprocessing import LabelEncoder

# Normalization
from sklearn.preprocessing import PolynomialFeatures

# Model Building 
import sklearn
from sklearn.model_selection import train_test_split


from sklearn.linear_model import LinearRegression as LR
from xgboost import XGBRegressor as XGB 
from sklearn.ensemble import AdaBoostRegressor as Ada
from sklearn.tree import DecisionTreeRegressor as DT
from sklearn.ensemble import RandomForestRegressor as RF
from sklearn.ensemble import GradientBoostingRegressor as GB

# Evaluation 
from sklearn.metrics import r2_score,mean_squared_error

# Cross Validation 
from sklearn.model_selection import GridSearchCV

#Customised function

def missing_cols(df):
    '''prints out columns with its amount of missing values with its %'''
    total = 0
    for col in df.columns:
        missing_vals = df[col].isnull().sum()
        pct = df[col].isna().mean() * 100
        total += missing_vals
        if missing_vals != 0:
          print('{} => {} [{}%]'.format(col, df[col].isnull().sum(), round(pct, 2)))
    
    if total == 0:
        print("no missing values")

"""### Uploading Training Data"""

df = pd.read_csv('train.csv')
df

"""### Uploading Testing Data"""

df2 = pd.read_csv('test.csv')
df2

"""## EDA

### EDA (Exploratory Data Analysis) is a data analysis technique by which we try to understand the given data and the relationship among those data and also try to find out features , insights and anomalies from the data using statistics and graphical representation.
"""

df = df.drop_duplicates()
df

df.describe()

df.nunique()

df['FUELTYPE'].value_counts()

sb.pairplot(df)

sb.heatmap(df.corr(),annot = True, cmap = 'magma')

sb.distplot(df['CO2EMISSIONS'], kde = False)

sb.distplot(df['CYLINDERS'], kde = False)

plt.figure(figsize = (14,7))
sb.countplot( x = 'MAKE', data = df)
plt.show()

#plt.grid()

plt.figure(figsize = (16,8))
plt.title("Box plot for CO2 Emissions")
sb.boxplot( y = 'CO2EMISSIONS', x = 'CYLINDERS', data = df)
plt.show()

plt.figure(figsize = (16,8))
plt.title("Scatter plot for CO2 Emissions")
plt.scatter(y = 'CO2EMISSIONS', x = 'CYLINDERS', data = df)

plt.figure(figsize = (16,8))
plt.title("Box plot for CO2 Emissions")
sb.boxplot( y = 'CO2EMISSIONS', x = 'FUELTYPE', data = df)
plt.show()

plt.figure(figsize = (16,8))
plt.title("Scatter plot for CO2 Emissions")
plt.scatter(y = 'CO2EMISSIONS', x = 'FUELCONSUMPTION_CITY', data = df)

plt.figure(figsize=(16,10))
ax = sb.scatterplot(x='FUELCONSUMPTION_CITY',y='CO2EMISSIONS',data=df, hue = 'FUELTYPE', palette='rainbow')
ax.set_title('Scatter plot of CO2 Emissions')

plt.figure(figsize=(16,10))
ax = sb.scatterplot(x='FUELCONSUMPTION_CITY',y='CO2EMISSIONS',data=df, hue = 'CYLINDERS', palette='rainbow')
ax.set_title('Scatter plot of CO2 Emissions')

plt.figure(figsize=(16,10))
ax = sb.scatterplot(x='FUELCONSUMPTION_HWY',y='CO2EMISSIONS',data=df, hue = 'FUELTYPE', palette='rainbow')
ax.set_title('Scatter plot of CO2 Emissions')

plt.figure(figsize=(16,10))
ax = sb.scatterplot(x='FUELCONSUMPTION_COMB',y='CO2EMISSIONS',data=df, hue = 'FUELTYPE', palette='rainbow')
ax.set_title('Scatter plot of CO2 Emissions')

plt.figure(figsize=(16,10))
ax = sb.scatterplot(x='FUELCONSUMPTION_COMB_MPG',y='CO2EMISSIONS',data=df, hue = 'FUELTYPE', palette='rainbow')
ax.set_title('Scatter plot of CO2 Emissions')

"""### Preprocessing on Training Data"""

missing_cols(df)

df.nunique()

df.info()

objList = df.select_dtypes(include = "object").columns
print (objList)

#Label Encoding for object to numeric conversion
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()

for feat in objList:
    df[feat] = le.fit_transform(df[feat].astype(str))

print (df.info())

df

"""### Preprocessing on Testing Data"""

objList = df2.select_dtypes(include = "object").columns
print (objList)

#Label Encoding for object to numeric conversion
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()

for feat in objList:
    df2[feat] = le.fit_transform(df2[feat].astype(str))

print (df2.info())

"""### xVar & yVar"""

xVar = df.drop(['CO2EMISSIONS'], axis = 1)
xVar

yVar = df['CO2EMISSIONS']
yVar

"""### Train-test Split"""

#data split

X_train, X_test, y_train, y_test = train_test_split(xVar, yVar, test_size=0.25, random_state = 0)
print (X_train.shape, y_train.shape)
print (X_test.shape, y_test.shape)

#test data 

X_test = df2
X_test

#test data

X_train = xVar
y_train = yVar
print (X_train.shape, y_train.shape)
print (X_test.shape)

"""## Scaling and Normalization

### Reference: 
https://machinelearningmastery.com/polynomial-features-transforms-for-machine-learning/
"""

## Scaling ## Not performed for this dataset 

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

# transform data
X_train = scaler.fit_transform(X_train)
X_test = scaler.fit_transform(X_test)

## PolynomialFeatures improves accuracy of Linear regression but not necessarily of decision tree based algorithms 

quad = PolynomialFeatures (degree = 2)

X_train = quad.fit_transform(X_train)
X_test = quad.fit_transform(X_test)

"""## Simple Linear Regression

### Reference:
https://www.youtube.com/watch?app=desktop&v=U7D1h5bbpcs&feature=youtu.be
"""

import statsmodels.api as sm

X_train = sm.add_constant(X_train)
X_test = sm.add_constant(X_test)


model = sm.OLS(y_train, X_train)
model = model.fit()
print(model.summary2())

predictions = model.predict(X_test)
df_results = pd.DataFrame({'Actual': y_test, 'Predicted': predictions})

RMSE = np.sqrt(mean_squared_error(y_test, predictions))
r2 = r2_score(y_test, predictions)
print(RMSE, r2)

# Plot the actual vs predicted results
plt.figure(figsize=(16,10))
sb.lmplot(x='Actual', y='Predicted', data=df_results, fit_reg=False)

#Plot the diagonal line
d_line= np.arange(df_results.min().min(), df_results.max().max())
plt.plot(d_line, d_line, color='red', linestyle='--')
plt.show()

"""## Different Algorithms """

# Implementing models

# Linear Regression
linear = LR()
lr = linear.fit(X_train, y_train)
lr.score(X_test,y_test)

# Adaboost Regressor

ada = Ada(random_state=0)

ada_ = ada.fit(X_train, y_train)

ada_.score(X_test,y_test)

# XGB Regressor

xgbregressor = XGB(random_state=0)

xgb_ = xgbregressor.fit(X_train, y_train)

xgb_.score(X_test,y_test)

# Decision Tree Regressor

dtr = DT(criterion="mse",
        max_depth=6,
        max_features="auto",
        random_state=0)

dtr_ = dtr.fit(X_train, y_train)

dtr_.score(X_test,y_test)

# Random Tree Regressor

rfr = RF(n_estimators=100,
        criterion="mse",
        max_depth=6,
        max_features="auto",
        random_state=0)

rfr_ = rfr.fit(X_train, y_train)

rfr_.score(X_test,y_test)

"""## How to check if model is overfitting or not """

forest = RF(n_estimators = 500,
            criterion = 'mse',
            random_state = 0,
            n_jobs = -1)
forest.fit(X_train,y_train)
forest_train_pred = forest.predict(X_train)
forest_test_pred = forest.predict(X_test)

print('MSE train data: %.3f, MSE test data: %.3f' % (
mean_squared_error(y_train,forest_train_pred),
mean_squared_error(y_test,forest_test_pred)))

print('R2 train data: %.3f, R2 test data: %.3f' % (
r2_score(y_train,forest_train_pred),
r2_score(y_test,forest_test_pred)))

"""### Validating Models"""

lr_pred = lr.predict(X_test)
ada_pred = ada_.predict(X_test)
xgb_pred = xgb_.predict(X_test)
dtr_pred = dtr_.predict(X_test)
rfr_pred = rfr_.predict(X_test)

# Plot the actual vs predicted results
plt.figure(figsize=(16,16))
df_results = pd.DataFrame({'Actual': y_test, 'Predicted': lr_pred})

sb.lmplot(x='Actual', y='Predicted', data=df_results, fit_reg=False)

d_line= np.arange(df_results.min().min(), df_results.max().max())
plt.plot(d_line, d_line, color='red', linestyle='--')
plt.show()

# Plot the actual vs predicted results
plt.figure(figsize=(16,16))
df_results = pd.DataFrame({'Actual': y_test, 'Predicted': xgb_pred})

sb.lmplot(x='Actual', y='Predicted', data=df_results, fit_reg=False)

d_line= np.arange(df_results.min().min(), df_results.max().max())
plt.plot(d_line, d_line, color='red', linestyle='--')
plt.show()

from sklearn.metrics import mean_squared_error

linear_mse = mean_squared_error(y_test, lr_pred)
ada_mse = mean_squared_error(y_test, ada_pred)
xgb_mse = mean_squared_error(y_test, xgb_pred)
dtr_mse = mean_squared_error(y_test, dtr_pred)
rfr_mse = mean_squared_error(y_test, rfr_pred)

print(pow(linear_mse, 0.5))
print(pow(ada_mse, 0.5))
print(pow(xgb_mse, 0.5))
print(pow(dtr_mse, 0.5))
print(pow(rfr_mse, 0.5))

"""## Let's try to Improve the Model

### XGBoost Model
"""

## Note: It may take a few minutes. 

xgb1 = XGBR()
parameters = {'nthread':[4], 
              'objective':['reg:squarederror'],
              'learning_rate': [0.01, 0.03, 0.1],
              'max_depth': [5, 6, 7],
              'min_child_weight': [1, 5],
              'subsample': [0.6, 0.8, 1.0],
              'colsample_bytree': [0.6, 0.8, 1.0],
              'n_estimators': [500],
              'gamma': [0.5, 1, 1.5, 2, 5]}

xgb_grid = GridSearchCV(xgb1,
                        parameters,
                        cv = 3,
                        n_jobs = 5,
                        scoring='mean_squared_error',
                        verbose=True)

xgb_grid.fit(X_train,
         y_train)

print(xgb_grid.best_score_)
print(xgb_grid.best_params_)

xgboost = XGB(learning_rate=0.04,
                       nthread = 4,
                       n_estimators=500,
                       max_depth=5,
                       min_child_weight=1,
                       gamma=1.5,
                       objective='reg:squarederror',
                       subsample= 0.8,
                       random_state=42)

xgb = xgboost.fit(X_train, y_train)

"""### Prediction """

prediction = xgb.predict(X_test)
prediction

"""### RMSE Calculation for training data"""

from sklearn.metrics import mean_squared_error
RMSE = np.sqrt(mean_squared_error(y_test, prediction))
print(RMSE)

# Plot the actual vs predicted results
plt.figure(figsize=(16,16))
df_results = pd.DataFrame({'Actual': y_test, 'Predicted': prediction})

sb.lmplot(x='Actual', y='Predicted', data=df_results, fit_reg=False)

d_line= np.arange(df_results.min().min(), df_results.max().max())
plt.plot(d_line, d_line, color='red', linestyle='--')
plt.show()

"""## Decision Tree Regressor

### Convert into csv
"""

from pandas import DataFrame
df3 = DataFrame(prediction,columns=['CO2EMISSIONS'], index = False)

df3.to_csv('submission.csv')

"""# End of code"""