How to do node classification and choose important features

import networkx as nx
import numpy as np
import pandas as pd
import os

import stellargraph as sg
from stellargraph.mapper import GraphSAGENodeGenerator
from stellargraph.layer import GraphSAGE

from tensorflow.keras import layers, optimizers, losses, metrics, Model
from sklearn import model_selection
from sklearn.utils import class_weight

big_df = pd.read_csv("/data/big_df_2.csv")
big_df.drop(['Unnamed: 0'], axis=1, inplace=True)
big_df['Count'] = np.abs(big_df['count']-big_df['count'].mean())/big_df['count'].std()

big_df.head()

big_df_ = big_df[big_df['depart'] == 1]

ddf = pd.read_csv("data/ddf_2.csv")
ddf.drop(['Unnamed: 0'], axis=1, inplace=True)
ddf.head()

big_G = nx.Graph()
for i in ddf.index:
    big_G.add_edge(ddf.iloc[i]['IndividualId'], ddf.iloc[i]['FatherIndividualId'])

list_nodes = list(big_df['node'])

big_G = big_G.subgraph(list_nodes)

list_nodes = list(big_G.nodes())
big_df = big_df[big_df['node'].isin(list_nodes)]

big_df = big_df.set_index('node')

train_data, test_data = model_selection.train_test_split(big_df, train_size=0.8, test_size=None, stratify=big_df['conflict'])

train_data.head()

feature_names = ['eigen', 'Count']

node_features = big_df[feature_names]

node_features.head()

node_features.describe()

node_features.info()

G = sg.StellarGraph(big_G, node_features=node_features)

print(G.info())

import matplotlib.pyplot as plt
%matplotlib inline

options = {
    'node_color': 'blue',
    'node_size': 100,
    'width': 1,
    
}
plt.figure(figsize = (18,18))
nx.draw(big_G, with_labels = False, **options)
plt.show()

from collections import Counter
Counter(train_data['conflict'])

import missingno as msno

msno.bar(train_data)

print(nx.info(big_G))

batch_size = 80; num_samples = [20, 15]

generator = GraphSAGENodeGenerator(G, batch_size, num_samples)

train_gen = generator.flow(train_data.index, np.array(train_data[["conflict"]]))

graphsage_model = GraphSAGE(
    layer_sizes=[32, 32],
    generator=train_gen,
    bias=True,
    dropout=0.1,
    #aggregator=sg.layer.graphsage.MeanAggregator
)

from tensorflow import keras

#from keras import layers, optimizers, losses, metrics, Model

x_inp, x_out = graphsage_model.build()

prediction = keras.layers.Dense(units=np.array(train_data[["conflict"]]).shape[1], activation="sigmoid")(x_out)

model = keras.Model(inputs=x_inp, outputs=prediction)

model.compile(
    optimizer=keras.optimizers.Adam(lr=0.1),
    loss=keras.losses.binary_crossentropy,
    metrics=["acc"]
)

test_gen = generator.flow(test_data.index, np.array(test_data[["conflict"]]))

history = model.fit_generator(
    train_gen,
    steps_per_epoch=len(big_df) // batch_size,
    epochs=25,
    validation_data=test_gen,
    verbose=2,
    shuffle=False
)

import matplotlib.pyplot as plt
%matplotlib inline

def plot_history(history):
    metrics = sorted(history.history.keys())
    metrics = metrics[:len(metrics)//2]
    for m in metrics:
        # summarize history for metric m
        plt.plot(history.history[m])
        plt.plot(history.history['val_' + m])
        plt.title(m)
        plt.ylabel(m)
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='best')
        plt.show()

plot_history(history)

import multiprocessing
num_workers = multiprocessing.cpu_count()//2

train_metrics = model.evaluate_generator(train_gen, use_multiprocessing=True, workers=num_workers, verbose=1)
test_metrics = model.evaluate_generator(test_gen, use_multiprocessing=True, workers=num_workers, verbose=1)

print("\nTrain Set Metrics of the trained model:")
for name, val in zip(model.metrics_names, train_metrics):
    print("\t{}: {:0.4f}".format(name, val))

print("\nTest Set Metrics of the trained model:")
for name, val in zip(model.metrics_names, test_metrics):
    print("\t{}: {:0.4f}".format(name, val))

y_true = np.array(test_data[["conflict"]])
y_pred = model.predict_generator(test_gen)

from sklearn.metrics import classification_report

print(classification_report(np.around(y_pred), y_true))

test_metrics = model.evaluate_generator(test_gen)
print("\nTest Set Metrics:")
for name, val in zip(model.metrics_names, test_metrics):
    print("\t{}: {:0.4f}".format(name, val))

from sklearn.metrics import precision_recall_fscore_support

precision_recall_fscore_support(np.array(test_data['depart']), np.around(model.predict_generator(generator.flow(test_data[feature_names].index))), average='macro')

precision_recall_fscore_support(np.array(test_data['depart']), np.around(model.predict_generator(generator.flow(test_data[feature_names].index))), average='micro')

precision_recall_fscore_support(np.array(test_data['depart']), np.around(model.predict_generator(generator.flow(test_data[feature_names].index))), average='weighted')

num_tests = 1 # the number of times to generate predictions
all_test_predictions = [model.predict_generator(test_gen, verbose=True) for _ in np.arange(num_tests)]

from sklearn.calibration import calibration_curve

calibration_data = [calibration_curve(y_prob=test_predictions, 
                                      y_true=np.array(test_data['depart']), 
                                      n_bins=10, 
                                      normalize=True) for test_predictions in all_test_predictions]

from stellargraph import expected_calibration_error, plot_reliability_diagram

for fraction_of_positives, mean_predicted_value in calibration_data:
    ece_pre_calibration = expected_calibration_error(prediction_probabilities=all_test_predictions[0], 
                                                     accuracy=fraction_of_positives, 
                                                     confidence=mean_predicted_value)
    print('ECE: (before calibration) {:.4f}'.format(ece_pre_calibration))

plot_reliability_diagram(calibration_data, 
                         np.array(all_test_predictions[0]), 
                         ece=[ece_pre_calibration])

use_platt = False  # True for Platt scaling or False for Isotonic Regression
num_tests = 1
score_model = Model(inputs=x_inp, outputs=prediction)
if use_platt:
    all_test_score_predictions = [score_model.predict_generator(test_gen, verbose=True) for _ in np.arange(num_tests)]
    all_test_probabilistic_predictions = [model.predict_generator(test_gen, verbose=True) for _ in np.arange(num_tests)]
else:
    all_test_probabilistic_predictions = [model.predict_generator(test_gen, verbose=True) for _ in np.arange(num_tests)]

# These are the uncalibrated prediction probabilities. 
if use_platt:
    test_predictions = np.mean(np.array(all_test_score_predictions), axis=0)
    test_predictions.shape
else:
    test_predictions = np.mean(np.array(all_test_probabilistic_predictions), axis=0)
    test_predictions.shape

from stellargraph import IsotonicCalibration, TemperatureCalibration

if use_platt:
    # for binary classification this class performs Platt Scaling
    lr = TemperatureCalibration()  
else:
    lr = IsotonicCalibration()

lr.fit(test_predictions, np.array(test_data['depart']))

lr_test_predictions = lr.predict(test_predictions)

calibration_data = [calibration_curve(y_prob=lr_test_predictions, 
                                      y_true=np.array(test_data['depart']), 
                                      n_bins=10, 
                                      normalize=True)] 

for fraction_of_positives, mean_predicted_value in calibration_data:
    ece_post_calibration = expected_calibration_error(prediction_probabilities=lr_test_predictions, 
                                                      accuracy=fraction_of_positives, 
                                                      confidence=mean_predicted_value)
    print('ECE (after calibration): {:.4f}'.format(ece_post_calibration))

plot_reliability_diagram(calibration_data, 
                         lr_test_predictions, 
                         ece=[ece_post_calibration])

from sklearn.metrics import accuracy_score

y_pred = np.zeros(len(test_predictions))
if use_platt:
    # the true predictions are the probabilistic outputs
    test_predictions = np.mean(np.array(all_test_probabilistic_predictions), axis=0)
y_pred[test_predictions.reshape(-1)>0.5] = 1
print('Accuracy of model before calibration: {:.2f}'.format(accuracy_score(y_pred=y_pred, 
                                                                           y_true=np.array(test_data['depart']))))

y_pred = np.zeros(len(lr_test_predictions))
y_pred[lr_test_predictions[:,0]>0.5] = 1
print('Accuracy for model after calibration: {:.2f}'.format(accuracy_score(y_pred=y_pred, y_true=np.array(test_data['depart']))))

big_df = pd.read_csv("data/big_df_orig.csv")
big_df.drop(['Unnamed: 0'], axis=1, inplace=True)
big_df['Count'] = (big_df['Count']-big_df['Count'].mean())/big_df['Count'].std()

big_df.columns

X_train, X_test, y_train, y_test = model_selection.train_test_split(big_df[['between', 'Count', 'deg', 'close', 'eigen']], big_df['depart'], test_size = 0.2)
n_train, _ = X_train.shape
print(X_train.shape, X_test.shape)

n, d = big_df[['between', 'Count', 'deg', 'close', 'eigen']].shape

import matplotlib.pyplot as plt
%matplotlib inline

big_df[big_df['depart'] == 1]['between'].describe()

big_df[big_df['depart'] == 0]['between'].describe()

import seaborn as sns

#Create Correlation df
corr = X_train.corr()
#Plot figsize
fig, ax = plt.subplots(figsize=(10, 10))
#Generate Color Map
colormap = sns.diverging_palette(220, 10, as_cmap=True)
#Generate Heat Map, allow annotations and place floats in map
sns.heatmap(corr, cmap=colormap, annot=True, fmt=".2f")
#Apply xticks
plt.xticks(range(len(corr.columns)), corr.columns);
#Apply yticks
plt.yticks(range(len(corr.columns)), corr.columns)
#show plot
plt.show()

sns.pairplot(X_train)
plt.show()

df = pd.DataFrame(X_train)
COV = df.cov()
plt.matshow(COV)
plt.show()

U, s, V = np.linalg.svd(COV, full_matrices = True)

fig = plt.figure()
plt.plot(s, 'r-o')
#plt.axvline(60, c='r')
plt.title("Valeurs singulières de la matrice de corrélation")
plt.show()

s < 0.01

len(s)

s

from sklearn.linear_model import LinearRegression, Ridge, LassoCV

regr0 = LinearRegression()
regr0.fit(X_train, y_train)

fig = plt.figure()
plt.plot(regr0.coef_, 'r-o')
plt.title("Valeurs des coefficients OLS")
plt.show()

regr0.coef_

X_train_reduce = np.dot(X_train, U[:, :2])
X_test_reduce = np.dot(X_test, U[:, :2])

regr1 = LinearRegression()
regr1.fit(X_train_reduce, y_train)

fig = plt.figure()
plt.plot(regr1.coef_, 'r-o')
plt.title("Valeurs des coefficients de PCA_before_OLS (sans intercept)")
plt.show()

for y in (regr0.intercept_, regr1.intercept_, y_train.mean(axis=0)):
    print("%.3f" % y)

print("\nLes deux intercepts sont-ils égaux: %s"
      % np.isclose(regr0.intercept_, regr1.intercept_))

from sklearn import preprocessing

X_train_reduce2 = preprocessing.scale(X_train_reduce)

regrtest = LinearRegression()
regrtest.fit(X_train_reduce2, y_train)

print("On vérifie l'égalité. Cette égalité est: %s."
      % np.isclose(regrtest.intercept_, np.mean(y_train)))

R20 = regr0.score(X_test, y_test) 
R21 = regr1.score(X_test_reduce, y_test)

def MSE(y_pred, y_true):
    return np.mean((y_pred - y_true) ** 2)

pred_error0 = MSE(regr0.predict(X_test), y_test)
pred_error1 = MSE(regr1.predict(X_test_reduce), y_test)

print("Le R2 de OLS:            %.3f" % R20)
print("Le R2 de PCA before OLS: %.3f\n" % R21)
print("Le rique de prédiction de OLS calculé sur l'échantillon test:            %.2f" % pred_error0)
print("Le rique de prédiction de PCA before OLS calculé sur l'échantillon test: %.2f" % pred_error1)

eps0 = regr0.predict(X_test) - y_test
eps1 = regr1.predict(X_test_reduce) - y_test

plt.figure()
plt.plot(eps0.values, label="ols")
plt.plot(eps1.values, label="pca_before_ols")
plt.legend(loc=1)
plt.title("Visualisation des résidus")
plt.show()

eps = pd.DataFrame(np.c_[eps0, eps1], columns=['OLS', 'PCA_before_OLS'])
eps.hist(bins=20)
plt.show()

resids = y_train

test = np.zeros((d, d))
pval_mem = np.zeros(d)
pval = np.zeros((d, d))
var_sel = []
var_remain = ['between', 'Count', 'deg', 'close', 'eigen'] #list(range(d))
in_test = []
 
regr = LinearRegression()

from scipy.stats import norm

d

for k in range(d): 
    resids_mem = np.zeros((d, n_train))
    j = 0
    for i in var_remain:
        xtmp = np.array(list(X_train[i])) #X_train[:, [i]]
        xtmp = xtmp.reshape(-1, 1)
        regr.fit(xtmp, resids)
        
        # calcul de (x'x)
        xx = np.sum(X_train[i] ** 2) #xx = np.sum(X_train[:, i] ** 2)
        resids_mem[j, :] = regr.predict(xtmp) - resids
        sigma2_tmp = np.sum(resids_mem[j, :] ** 2) / xx
        test[k, j] = np.sqrt(n) * np.abs(regr.coef_) / (np.sqrt(sigma2_tmp))
        pval[k, j] = 2 * (1 - norm.cdf(test[k, j]))
        j = j + 1

    # separe en deux vecteurs la listes des variables séléctionnées et les autres
    best_var = np.argmax(test[k, :])
    var_sel.append(best_var)
    resids = resids_mem[best_var, :]
    pval_mem[k] = pval[k, best_var]
    var_remain = np.setdiff1d(var_remain, var_sel)

print("Voici l'ordre dans lequel les variables sont sélectionées par la méthode forward :\n\n%s" % var_sel)

fig = plt.figure()

for k in range(3):
    
    plt.subplot(311 + k)
    
    if k == 0:
        plt.title("values of the t-stat at each step")

    plt.plot(np.arange(d), test[k, :], '-o', label="step %s" % k)
    plt.plot(var_sel[k], test[k, var_sel[k]], 'r-o')
    plt.legend(loc=1)
    
    if k == 2:
        plt.xlabel("features")

plt.show()

fig2 = plt.figure()

for k in range(3): 
    plt.plot(np.arange(d), pval_mem, 'o')
    plt.plot([-0.5, 10], [.1, .1], color="b")
    plt.axis(xmin=-.5, xmax=10, ymin=-.03)

plt.title("Graph des p-valeurs")
plt.xlabel("steps")
plt.show()

var_sel_a = np.array(var_sel)
#print(var_sel_a,var_sel)
var_sel_def = var_sel_a[pval_mem < 0.001]

print("Il y a donc %s variables selectionnées.\nLes voici: %s" % (len(var_sel_def), var_sel_def))

X_train.columns

X_train_sel = X_train[['between', 'Count']]
X_test_sel = X_test[['between', 'Count']]

regr2 = LinearRegression()
regr2.fit(X_train_sel, y_train)

print(regr2.coef_)
print(regr2.intercept_)

pred_error_forward = MSE(regr2.predict(X_test_sel), y_test)

print("As a reminder, let us give the prediction scores obtained previouslyt.\n")

for method, error in zip(["ols           ", "pca_before_ols", "forward       "],
                         [pred_error0, pred_error1, pred_error_forward]):
    print(method + " : %.2f" % error)

np.random.seed(2)
perm = np.random.permutation(range(n_train))
q = n_train / 4.
split = np.array([0, 1, 2, 3, 4]) * q
split = split.astype(int)

for fold in range(4):
    print("The fold %s contains:\n%s\n\n" % (fold, perm[split[fold]: split[fold + 1]]))

lasso = LassoCV()
lasso.fit(X_train, y_train)

# The estimator chose automatically its lambda:
lasso.alpha_

pred_error_lasso = MSE(lasso.predict(X_test), y_test)

print("As a reminder, let's give the scores obtained previously.\n")

for method, error in zip(["ols           ", "pca_before_ols", "forward       ",
                          "lasso         "],
                         [pred_error0, pred_error1, pred_error_forward,
                           pred_error_lasso]):
    print(method + " : %.2f" % error)

import pandas as pd
import numpy as np
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2

big_df.head()

big_df_ = big_df
big_df_[['deg', 'close', 'between', 'eigen', 'Count']] = big_df_[['deg', 'close', 'between', 'eigen', 'Count']]*1000

big_df_.head()

big_df.columns

X = big_df[['deg', 'close', 'between', 'eigen', 'Count']]  #independent columns
y = big_df[['depart']]    #target column i.e price range
from sklearn.ensemble import ExtraTreesClassifier
import matplotlib.pyplot as plt
model = ExtraTreesClassifier()
model.fit(X,y)
print(model.feature_importances_) #use inbuilt class feature_importances of tree based classifiers
#plot graph of feature importances for better visualization
feat_importances = pd.Series(model.feature_importances_, index=X.columns)
feat_importances.nlargest(5).plot(kind='barh')
plt.show()

X = big_df_[['deg', 'close', 'between', 'eigen', 'Count']]  #independent columns
y = big_df_[['depart']]    #target column i.e price range
from sklearn.ensemble import ExtraTreesClassifier
import matplotlib.pyplot as plt
model = ExtraTreesClassifier()
model.fit(X,y)
print(model.feature_importances_) #use inbuilt class feature_importances of tree based classifiers
#plot graph of feature importances for better visualization
feat_importances = pd.Series(model.feature_importances_, index=X.columns)
feat_importances.nlargest(5).plot(kind='barh')
plt.show()

import statsmodels.api as sm
from sklearn.feature_selection import RFE
from sklearn.linear_model import RidgeCV, LassoCV, Ridge, Lasso

big_df.columns

#Adding constant column of ones, mandatory for sm.OLS model
X_1 = sm.add_constant(X)
#Fitting sm.OLS model
model = sm.OLS(y,X_1).fit()
model.pvalues

#Backward Elimination
cols = list(X.columns)
pmax = 1
while (len(cols)>0):
    p= []
    X_1 = X[cols]
    X_1 = sm.add_constant(X_1)
    model = sm.OLS(y,X_1).fit()
    p = pd.Series(model.pvalues.values[1:],index = cols)      
    pmax = max(p)
    feature_with_p_max = p.idxmax()
    if(pmax>0.05):
        cols.remove(feature_with_p_max)
    else:
        break
selected_features_BE = cols

print(selected_features_BE)

model = LinearRegression()
#Initializing RFE model
rfe = RFE(model, 7)
#Transforming data using RFE
X_rfe = rfe.fit_transform(X,y)  
#Fitting the data to model
model.fit(X_rfe,y)
print(rfe.support_)
print(rfe.ranking_)

#no of features
nof_list=np.arange(1,13)            
high_score=0
#Variable to store the optimum features
nof=0           
score_list =[]
for n in range(len(nof_list)):
    X_train, X_test, y_train, y_test = model_selection.train_test_split(X,y, test_size = 0.2, random_state = 0)
    model = LinearRegression()
    rfe = RFE(model,nof_list[n])
    X_train_rfe = rfe.fit_transform(X_train,y_train)
    X_test_rfe = rfe.transform(X_test)
    model.fit(X_train_rfe,y_train)
    score = model.score(X_test_rfe,y_test)
    score_list.append(score)
    if(score>high_score):
        high_score = score
        nof = nof_list[n]
print("Optimum number of features: %d" %nof)
print("Score with %d features: %f" % (nof, high_score))

cols = list(X.columns)
model = LinearRegression()
#Initializing RFE model
rfe = RFE(model, 1)             
#Transforming data using RFE
X_rfe = rfe.fit_transform(X,y)  
#Fitting the data to model
model.fit(X_rfe,y)              
temp = pd.Series(rfe.support_,index = cols)
selected_features_rfe = temp[temp==True].index
print(selected_features_rfe)

reg = LassoCV()
reg.fit(X, y)
print("Best alpha using built-in LassoCV: %f" % reg.alpha_)
print("Best score using built-in LassoCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)

print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " +  str(sum(coef == 0)) + " variables")

imp_coef = coef.sort_values()
import matplotlib
matplotlib.rcParams['figure.figsize'] = (8.0, 10.0)
imp_coef.plot(kind = "barh")
plt.title("Feature importance using Lasso Model")

from feature_selector import FeatureSelector

fs = FeatureSelector(data = X, labels = y)

fs.identify_missing(missing_threshold = 0.6)
fs.missing_stats.head()

fs.identify_collinear(correlation_threshold = 0.5)

# Pass in the appropriate parameters
fs.identify_zero_importance(task = 'classification', 
                            eval_metric = 'auc', 
                            n_iterations = 10, 
                             early_stopping = True)
# list of zero importance features
zero_importance_features = fs.ops['zero_importance']

# plot the feature importances
fs.plot_feature_importances(threshold = 0.99, plot_n = 12)

fs.identify_low_importance(cumulative_importance = 0.99)

fs.identify_single_unique()

fs.plot_unique()

train_removed = fs.remove(methods = 'all')

train_removed.head()

train_removed_all = fs.remove(methods = 'all', keep_one_hot=False)

train_removed_all.head()

fs.identify_all(selection_params = {'missing_threshold': 0.6,    
                                    'correlation_threshold': 0.98, 
                                    'task': 'classification',    
                                    'eval_metric': 'auc', 
                                    'cumulative_importance': 0.99})

nodes_with_conflict = list(ddf[ddf['conflict'] == 0]['IndividualId'])

nodes_without_conflict = list(ddf[ddf['conflict'] == 1]['IndividualId'])

set(nodes_with_conflict) == set(nodes_without_conflict)

big_df_with_conflict = big_df[big_df['node'].isin(nodes_with_conflict)]

big_df_without_conflict = big_df[big_df['node'].isin(nodes_without_conflict)]

from pandarallel import pandarallel
pandarallel.initialize()
def inv(x):
    if x == 1:
        return 0
    elif x == 0:
        return 1
big_df['depart'] = big_df['depart'].parallel_apply(inv)

X = big_df_with_conflict[['close', 'between', 'eigen']]  #independent columns
y = big_df_with_conflict[['depart']]    #target column i.e price range
from sklearn.ensemble import ExtraTreesClassifier
import matplotlib.pyplot as plt
model = ExtraTreesClassifier()
model.fit(X,y)
print(model.feature_importances_) #use inbuilt class feature_importances of tree based classifiers
#plot graph of feature importances for better visualization
feat_importances = pd.Series(model.feature_importances_, index=X.columns)
feat_importances.nlargest(5).plot(kind='barh')
plt.show()

X = big_df_without_conflict[['close', 'between', 'eigen']]  #independent columns
y = big_df_without_conflict[['depart']]    #target column i.e price range
from sklearn.ensemble import ExtraTreesClassifier
import matplotlib.pyplot as plt
model = ExtraTreesClassifier()
model.fit(X,y)
print(model.feature_importances_) #use inbuilt class feature_importances of tree based classifiers
#plot graph of feature importances for better visualization
feat_importances = pd.Series(model.feature_importances_, index=X.columns)
feat_importances.nlargest(5).plot(kind='barh')
plt.show()

len(big_df_with_conflict), len(big_df_without_conflict)

fs = FeatureSelector(data = X, labels = y)

fs.identify_collinear(correlation_threshold = 0.5)

# Pass in the appropriate parameters
fs.identify_zero_importance(task = 'classification', 
                            eval_metric = 'auc', 
                            n_iterations = 10, 
                             early_stopping = True)
# list of zero importance features
zero_importance_features = fs.ops['zero_importance']

fs.identify_low_importance(cumulative_importance = 0.99)

#train_removed_all = fs.remove(methods = 'all', keep_one_hot=False)
train_removed_all = fs.remove(methods = 'all')