Predicted the popularity of songs

Keywords: Predicative modeling, Random Forest, XGBoost

Github Code

Loading Data

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm_notebook, tnrange
import os
from sklearn.preprocessing import LabelEncoder

#os.chdir("/content/drive/My Drive/Chartbusters/ChartbustersParticipantsData")
%matplotlib inline
train = pd.read_csv("Data_Train.csv")
test = pd.read_csv("Data_Test.csv")
print(train.shape)
print(test.shape)
(78458, 11)
(19615, 10)
print(train.info())
print(test.info())
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 78458 entries, 0 to 78457
Data columns (total 11 columns):
Unique_ID     78458 non-null int64
Name          78458 non-null object
Genre         78458 non-null object
Country       78458 non-null object
Song_Name     78457 non-null object
Timestamp     78458 non-null object
Views         78458 non-null int64
Comments      78458 non-null int64
Likes         78458 non-null object
Popularity    78458 non-null object
Followers     78458 non-null int64
dtypes: int64(4), object(7)
memory usage: 6.6+ MB
None
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 19615 entries, 0 to 19614
Data columns (total 10 columns):
Unique_ID     19615 non-null int64
Name          19615 non-null object
Genre         19615 non-null object
Country       19615 non-null object
Song_Name     19615 non-null object
Timestamp     19615 non-null object
Comments      19615 non-null int64
Likes         19615 non-null object
Popularity    19615 non-null object
Followers     19615 non-null int64
dtypes: int64(3), object(7)
memory usage: 1.5+ MB
None

Data Cleaning

train.Likes = (train.Likes.replace(r'[KM]+$', '', regex=True).astype(float) * \
   ....:           train.Likes.str.extract(r'[\d\.]+([KM]+)', expand=False)
   ....:             .fillna(1)
   ....:             .replace(['K','M'], [10**3, 10**6]).astype(int))

train.Popularity = (train.Popularity.replace(r'[KM]+$', '', regex=True).astype(float) * \
   ....:           train.Popularity.str.extract(r'[\d\.]+([KM]+)', expand=False)
   ....:             .fillna(1)
   ....:             .replace(['K','M'], [10**3, 10**6]).astype(int))

test.Likes = (test.Likes.replace(r'[KM]+$', '', regex=True).astype(float) * \
   ....:           test.Likes.str.extract(r'[\d\.]+([KM]+)', expand=False)
   ....:             .fillna(1)
   ....:             .replace(['K','M'], [10**3, 10**6]).astype(int))

test.Popularity = (test.Popularity.replace(r'[KM]+$', '', regex=True).astype(float) * \
   ....:           test.Popularity.str.extract(r'[\d\.]+([KM]+)', expand=False)
   ....:             .fillna(1)
   ....:             .replace(['K','M'], [10**3, 10**6]).astype(int))
# training Data 
train['Likes'] = train['Likes'].astype(int)
train['Popularity'] = train['Popularity'].astype(int)
train['Name'] = train['Name'].astype(str)
train['Genre'] = train['Genre'].astype(str)
train['Country'] = train['Country'].astype(str)

# testing Data
test['Likes'] = test['Likes'].astype(int)
test['Popularity'] = test['Popularity'].astype(int)
test['Name'] = test['Name'].astype(str)
test['Genre'] = test['Genre'].astype(str)
test['Country'] = test['Country'].astype(str)
train['Timestamp'] = pd.to_datetime(train['Timestamp'])
test['Timestamp'] = pd.to_datetime(test['Timestamp'])
# Converting into Datetime format both training and testing data
#train['Timestamp'] = pd.to_datetime(train['Timestamp'])
#test['Timestamp'] = pd.to_datetime(test['Timestamp'])

# Add columns to training and testing data
train['year'] = pd.DatetimeIndex(train['Timestamp']).year
train['month'] = pd.DatetimeIndex(train['Timestamp']).month
train['day'] = pd.DatetimeIndex(train['Timestamp']).day
train['hour'] = pd.DatetimeIndex(train['Timestamp']).hour

test['year'] = pd.DatetimeIndex(test['Timestamp']).year
test['month'] = pd.DatetimeIndex(test['Timestamp']).month
test['day'] = pd.DatetimeIndex(test['Timestamp']).day
test['hour'] = pd.DatetimeIndex(test['Timestamp']).hour
target = "Views"
target_value = train[target]
train.drop(["Views", "Unique_ID", "Song_Name", "Timestamp", "Name"], axis=1, inplace=True)
test.drop(["Unique_ID", "Song_Name", "Timestamp", "Name"], axis=1, inplace=True)
print(train.head())
print(test.head())
      Genre Country  Comments  Likes  Popularity  Followers  year  month  day  \
0  danceedm      AU         4    499          97     119563  2018      3   30   
1  danceedm      AU        17     49          17       2141  2016      6   20   
2  danceedm      AU        11    312          91      22248  2015      8    5   
3    rbsoul      AU         2   2400          76     393655  2017      8    6   
4  danceedm      AU        81   3031         699     201030  2016      9   17   

   hour  
0    15  
1     5  
2    17  
3    23  
4    20  
       Genre Country  Comments  Likes  Popularity  Followers  year  month  \
0  all-music      AU        26   5428         118       2568  2019      2   
1   danceedm      AU        18    738         195     155645  2015      9   
2   danceedm      AU        18    386          96      29446  2013      9   
3  all-music      AU         1    388          19     799410  2007      8   
4       rock      AU         0     79           1          2  2018     10   

   day  hour  
0    1     9  
1    3    17  
2   18    16  
3   14     0  
4    8    16  
target_value.head()
0     14017
1      1918
2      9668
3    113036
4    110024
Name: Views, dtype: int64
target_value = pd.DataFrame(target_value, columns=['Views'])
print("Total number of unique entries are {}".format(train['year'].nunique()))
print(train['year'].unique())
Total number of unique entries are 31
[2018 2016 2015 2017 2012 2014 2011 2013 2009 2019 2005 2004 2010 1994
 1991 2002 2007 2001 2003 2008 2006 1999 2000 1996 1973 1995 1992 1990
 1997 1979 1986]
print(train['Country'].nunique())
print(train['Country'].unique())
train.drop(["Country"], axis=1, inplace=True)
test.drop(["Country"], axis=1, inplace=True)
1
['AU']
print(train['Genre'].nunique())
print(train['Genre'].unique())
21
['danceedm' 'rbsoul' 'all-music' 'trap' 'latin' 'electronic' 'deephouse'
 'hiphoprap' 'classical' 'ambient' 'pop' 'rock' 'reggaeton' 'country'
 'disco' 'alternativerock' 'dubstep' 'drumbass' 'metal'
 'folksingersongwriter' 'indie']
print(train.head())
print(test.head())
      Genre  Comments  Likes  Popularity  Followers  year  month  day  hour
0  danceedm         4    499          97     119563  2018      3   30    15
1  danceedm        17     49          17       2141  2016      6   20     5
2  danceedm        11    312          91      22248  2015      8    5    17
3    rbsoul         2   2400          76     393655  2017      8    6    23
4  danceedm        81   3031         699     201030  2016      9   17    20
       Genre  Comments  Likes  Popularity  Followers  year  month  day  hour
0  all-music        26   5428         118       2568  2019      2    1     9
1   danceedm        18    738         195     155645  2015      9    3    17
2   danceedm        18    386          96      29446  2013      9   18    16
3  all-music         1    388          19     799410  2007      8   14     0
4       rock         0     79           1          2  2018     10    8    16
train.drop(['day', 'hour'], axis=1, inplace=True)
test.drop(['day', 'hour'], axis=1, inplace=True)
print(train.head())
print(test.head())
print(target_value.head())
      Genre  Comments  Likes  Popularity  Followers  year  month
0  danceedm         4    499          97     119563  2018      3
1  danceedm        17     49          17       2141  2016      6
2  danceedm        11    312          91      22248  2015      8
3    rbsoul         2   2400          76     393655  2017      8
4  danceedm        81   3031         699     201030  2016      9
       Genre  Comments  Likes  Popularity  Followers  year  month
0  all-music        26   5428         118       2568  2019      2
1   danceedm        18    738         195     155645  2015      9
2   danceedm        18    386          96      29446  2013      9
3  all-music         1    388          19     799410  2007      8
4       rock         0     79           1          2  2018     10
    Views
0   14017
1    1918
2    9668
3  113036
4  110024

Data Preprocession

Preprocessing only for non-tree based algorithm

#fig, axs = plt.subplots(ncols=2, nrows=2)
sns.distplot(train['Comments'], hist=False, rug=True)
sns.distplot(train['Likes'], hist=False, rug=True)
sns.distplot(train['Popularity'], hist=False, rug=True)
sns.distplot(train['Followers'], hist=False, rug=True)
<matplotlib.axes._subplots.AxesSubplot at 0x1c100721128>

png

import matplotlib.pyplot as pl
def distribution(data, transformed = False):
    """
    Visualization code for displaying skewed distributions of features
    """
    
    # Create figure
    fig = pl.figure(figsize = (11,5));

    # Skewed feature plotting
    for i, feature in enumerate(['Comments','Likes', 'Popularity', 'Followers']):
        ax = fig.add_subplot(1, 4, i+1)
        ax.hist(train[feature], bins = 25, color = '#00A0A0')
        ax.set_title("'%s' Feature Dist"%(feature), fontsize = 14)
        ax.set_xlabel("Value")
        ax.set_ylabel("Number of Records")
        ax.set_ylim((0, 2000))
        ax.set_yticks([0, 500, 1000, 1500, 2000])
        ax.set_yticklabels([0, 500, 1000, 1500, ">2000"])

    # Plot aesthetics
    if transformed:
        fig.suptitle("Log-transformed Distributions of Continuous Census Data Features", \
            fontsize = 16, y = 1.03)
    else:
        fig.suptitle("Skewed Distributions of Continuous Census Data Features", \
            fontsize = 16, y = 1.03)

    fig.tight_layout()
    fig.show()
distribution(train)
C:\Users\shaki\Anaconda3\envs\tensorflow_env\lib\site-packages\ipykernel_launcher.py:30: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.

png

skewed = ['Comments', 'Likes', 'Popularity', 'Followers']
train[skewed] = train[skewed].apply(lambda x: np.log(x + 1))
test[skewed] = test[skewed].apply(lambda x: np.log(x+ 1 ))
# Visualize the new log distributions
distribution(train, transformed = True)
C:\Users\shaki\Anaconda3\envs\tensorflow_env\lib\site-packages\ipykernel_launcher.py:30: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.

png

import matplotlib.pyplot as pl
def distribution(data, transformed = False):
    """
    Visualization code for displaying skewed distributions of features
    """
    
    # Create figure
    fig = pl.figure(figsize = (11,5));

    # Skewed feature plotting
    for i, feature in enumerate(['Views']):
        ax = fig.add_subplot(1, 2, i+1)
        ax.hist(target_value[feature], bins = 25, color = '#00A0A0')
        ax.set_title("'%s' Feature Dist"%(feature), fontsize = 14)
        ax.set_xlabel("Value")
        ax.set_ylabel("Number of Records")
        ax.set_ylim((0, 2000))
        ax.set_yticks([0, 500, 1000, 1500, 2000])
        ax.set_yticklabels([0, 500, 1000, 1500, ">2000"])

    # Plot aesthetics
    if transformed:
        fig.suptitle("Log-transformed Distributions of Continuous Census Data Features", \
            fontsize = 16, y = 1.03)
    else:
        fig.suptitle("Skewed Distributions of Continuous Census Data Features", \
            fontsize = 16, y = 1.03)

    fig.tight_layout()
    fig.show()
distribution(target_value)
C:\Users\shaki\Anaconda3\envs\tensorflow_env\lib\site-packages\ipykernel_launcher.py:30: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.

png

skewed = ['Views']
target_value[skewed] = target_value[skewed].apply(lambda x: np.log(x + 1))

distribution(target_value, transformed=True)
C:\Users\shaki\Anaconda3\envs\tensorflow_env\lib\site-packages\ipykernel_launcher.py:30: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.

png

print(train.head())
print(test.head())
print(target_value.head())
      Genre  Comments     Likes  Popularity  Followers  year  month
0  danceedm  1.609438  6.214608    4.584967  11.691607  2018      3
1  danceedm  2.890372  3.912023    2.890372   7.669495  2016      6
2  danceedm  2.484907  5.746203    4.521789  10.010052  2015      8
3    rbsoul  1.098612  7.783641    4.343805  12.883233  2017      8
4  danceedm  4.406719  8.016978    6.551080  12.211214  2016      9
       Genre  Comments     Likes  Popularity  Followers  year  month
0  all-music  3.295837  8.599510    4.779123   7.851272  2019      2
1   danceedm  2.944439  6.605298    5.278115  11.955339  2015      9
2   danceedm  2.944439  5.958425    4.574711  10.290347  2013      9
3  all-music  0.693147  5.963579    2.995732  13.591630  2007      8
4       rock  0.000000  4.382027    0.693147   1.098612  2018     10
       Views
0   9.548097
1   7.559559
2   9.176680
3  11.635470
4  11.608463

Preprocessing for tree based algorithm

label_encoding = LabelEncoder()
#catg = ['Genre', 'year', 'month']
train['Genre'] = label_encoding.fit_transform(train['Genre'])
#train['year'] = label_encoding.fit_transform(train['year'])
#train['month'] = label_encoding.fit_transform(train['month'])

test['Genre'] = label_encoding.fit_transform(test['Genre'])
#test['year'] = label_encoding.fit_transform(test['year'])
#test['month'] = label_encoding.fit_transform(test['month'])
#train['Genre'] = label_encoding.transform(train['Genre'])
print(train.head(n=3))
print(test.head(n=3))
   Genre  Comments     Likes  Popularity  Followers  year  month
0      5  1.609438  6.214608    4.584967  11.691607  2018      3
1      5  2.890372  3.912023    2.890372   7.669495  2016      6
2      5  2.484907  5.746203    4.521789  10.010052  2015      8
   Genre  Comments     Likes  Popularity  Followers  year  month
0      0  3.295837  8.599510    4.779123   7.851272  2019      2
1      5  2.944439  6.605298    5.278115  11.955339  2015      9
2      5  2.944439  5.958425    4.574711  10.290347  2013      9
from sklearn.model_selection import train_test_split

# Split the 'features' and 'income' data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(train, 
                                                    target_value, 
                                                    test_size = 0.2, 
                                                    random_state = 123)

# Show the results of the split
print("Training set has {} samples.".format(X_train.shape[0]))
print("Testing set has {} samples.".format(X_test.shape[0]))
Training set has 62766 samples.
Testing set has 15692 samples.

Applying Different algorithm

sub_3_bs model

import xgboost as xgb
xg_reg = xgb.XGBRegressor(colsample_bytree=1, max_depth=5, min_child_weight=1, n_estimators=400)
xg_reg.fit(X_train,y_train)
preds = xg_reg.predict(X_test)
preds.shape
[22:59:03] WARNING: C:/Jenkins/workspace/xgboost-win64_release_0.90/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.





(15692,)
import sklearn
rmse = np.sqrt(sklearn.metrics.mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))

y_out = xg_reg.predict(test)
y_out = np.reshape(y_out, (-1,1))
RMSE: 653199.524651

sub_12 best model

import xgboost as xgb
xg_reg = xgb.XGBRegressor(colsample_bytree=1, max_depth=5, min_child_weight=1, n_estimators=500)
xg_reg.fit(X_train,y_train)
preds = xg_reg.predict(X_test)
preds.shape
[09:31:21] WARNING: C:/Jenkins/workspace/xgboost-win64_release_0.90/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.





(15692,)
import sklearn
rmse = np.sqrt(sklearn.metrics.mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))

y_out = xg_reg.predict(test)
y_out = np.reshape(y_out, (-1,1))
RMSE: 652720.452490
y_out = pd.DataFrame(data=y_out, columns=['Views'])
t1 = pd.read_csv('Data_Test.csv')
fin = pd.concat([t1['Unique_ID'], y_out['Views']], axis=1, names=['Unique_ID', 'Views'])
fin.to_csv("result.csv")
res = pd.read_csv("result.csv", index_col=['Unique_ID'])
res.drop(['Unnamed: 0'], axis=1, inplace=True)
res.to_csv("result.csv")

sub_11 2 best model

import xgboost as xgb
xg_reg = xgb.XGBRegressor(colsample_bytree=1, max_depth=5)
xg_reg.fit(X_train,y_train)
preds = xg_reg.predict(X_test)
preds.shape
[09:39:23] WARNING: C:/Jenkins/workspace/xgboost-win64_release_0.90/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.





(15692,)
import sklearn
rmse = np.sqrt(sklearn.metrics.mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))

y_out = xg_reg.predict(test)
y_out = np.reshape(y_out, (-1,1))
RMSE: 687064.042208

RandomForest Algorithm

from sklearn.ensemble import RandomForestRegressor
model = RandomForestRegressor(n_estimators=200, random_state=0)
model.fit(X_train, y_train)
C:\Users\shaki\Anaconda3\envs\tensorflow_env\lib\site-packages\ipykernel_launcher.py:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  





RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,
                      max_features='auto', max_leaf_nodes=None,
                      min_impurity_decrease=0.0, min_impurity_split=None,
                      min_samples_leaf=1, min_samples_split=2,
                      min_weight_fraction_leaf=0.0, n_estimators=200,
                      n_jobs=None, oob_score=False, random_state=0, verbose=0,
                      warm_start=False)
preds = model.predict(X_test)
predictions = model.predict(test)
import sklearn
rmse = np.sqrt(sklearn.metrics.mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))
RMSE: 768164.961457
y_out = pd.DataFrame(data=predictions, columns=['Views'])

GradientBoostingAlgorithm

from sklearn.ensemble import GradientBoostingRegressor
grb=GradientBoostingRegressor(learning_rate=0.1,n_estimators=200, max_depth=5
                              ,subsample=0.8,
                              verbose=True,random_state=10)
grb.fit(X_train, y_train)
C:\Users\shaki\Anaconda3\envs\tensorflow_env\lib\site-packages\sklearn\ensemble\gradient_boosting.py:1450: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)


      Iter       Train Loss      OOB Improve   Remaining Time 
         1 11476674610981.4082 3840863089772.7930           20.09s
         2 9866672255542.0566 2406720072630.9277           19.69s
         3 8330430346923.1592 1738174885540.3906           19.16s
         4 6993864977081.2383 1214466522607.9082           18.32s
         5 5469402885722.8359 1416490844786.0049           17.89s
         6 4705869042506.9512 793548231609.7095           17.81s
         7 3931982905909.7578 557592720648.4448           17.47s
         8 3185043505855.1875 514952954257.7817           17.22s
         9 2494029947183.8403 562861864815.6255           17.37s
        10 2081111433765.3962 413652236944.8169           17.13s
        20 446882786325.4976 36216233868.0272           16.02s
        30 188417538675.2888  5458222840.5702           15.05s
        40 137242149572.8594    37777520.4012           14.10s
        50 113432768082.3768  -313756691.5271           12.90s
        60 101288279593.5180  -234674436.8158           11.86s
        70 89934317959.5893    69824332.4267           11.27s
        80 79050263288.7053   102179840.4725           10.36s
        90 68929127333.5310  -186081434.6905            9.44s
       100 61611051065.0692  -347528503.8614            8.55s
       200 29355345339.6046   -11435975.8146            0.00s





GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
                          learning_rate=0.1, loss='ls', max_depth=5,
                          max_features=None, max_leaf_nodes=None,
                          min_impurity_decrease=0.0, min_impurity_split=None,
                          min_samples_leaf=1, min_samples_split=2,
                          min_weight_fraction_leaf=0.0, n_estimators=200,
                          n_iter_no_change=None, presort='auto',
                          random_state=10, subsample=0.8, tol=0.0001,
                          validation_fraction=0.1, verbose=True,
                          warm_start=False)
preds = grb.predict(X_test)
preds.shape
(15692,)
import sklearn
rmse = np.sqrt(sklearn.metrics.mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))

y_out = xg_reg.predict(test)
y_out = np.reshape(y_out, (-1,1))
RMSE: 694455.207721

Submission

y_out = pd.DataFrame(data=y_out, columns=['Views'])
t1 = pd.read_csv('Data_Test.csv')
fin = pd.concat([t1['Unique_ID'], y_out['Views']], axis=1, names=['Unique_ID', 'Views'])
fin.to_csv("result.csv")
res = pd.read_csv("result.csv", index_col=['Unique_ID'])
res.drop(['Unnamed: 0'], axis=1, inplace=True)
res.to_csv("result.csv")