import os
import numpy as np
import pandas as pd
pd.set_option('display.max_row', 500)
pd.set_option('display.max_columns', 30)
from pandas import DataFrame

import warnings
warnings.filterwarnings("ignore")

train = pd.read_csv("train.csv")
print(train.shape)
train.head()

import missingno

missingno.matrix(train)

train.isnull().sum()

train.loc[train.isnull()['Age'],'Age'] = train['Age'].mean()

train.isnull().sum()

train2 = train.drop(columns=['Cabin']).dropna(axis=0)

train2.isnull().sum()

pivot1 = pd.pivot_table(train2, index='Sex', columns='Survived', values='Age', aggfunc='mean', margins=True)
pivot1

pivot1.unstack().reset_index()

pivot1.stack().reset_index()

pd.pivot_table(train2, index=['Sex','Parch'], columns='Survived', values='Age', aggfunc='median')

pd.pivot_table(train2, index=['Sex','Parch'], columns=['Survived','SibSp'], values=['Age','Fare'], aggfunc='median')

f_train = pd.read_csv("https://rfriend.tistory.com/attachment/cfile21.uf@2577A64555C56ECF0CF0C8.csv",
           encoding='cp949')
del f_train['company']
f_train.head()

from sklearn.preprocessing import StandardScaler

standardScaler = StandardScaler()

standardScaled = standardScaler.fit_transform(f_train)

scaled_data = DataFrame(standardScaled, columns=f_train.columns)
scaled_data.head()

scaled_data.describe().iloc[0:3]

from sklearn.preprocessing import MinMaxScaler

min_max_scaler = MinMaxScaler()

MinMax_train = min_max_scaler.fit_transform(f_train)

MinMax_train = DataFrame(MinMax_train, columns=f_train.columns)
MinMax_train.head()

MinMax_train.describe().T[['min','max']]

from sklearn.decomposition import PCA

pca = PCA()

pca.fit(scaled_data )

score = pca.transform(scaled_data)

pca.components_

ratio = pca.explained_variance_ratio_
ratio

import matplotlib.pyplot as plt

ax = plt.plot(['PC1','PC2','PC3','PC4','PC5'], ratio)
plt.show()

pca = PCA(n_components=2)

pca_data = pca.fit_transform(scaled_data )

DataFrame(pca_data, columns=['PC1','PC2']).head()

from factor_analyzer import FactorAnalyzer
df = pd.read_csv("https://vincentarelbundock.github.io/Rdatasets/csv/psych/bfi.csv",
                index_col=0)
df.drop(['gender','education','age'],axis=1,inplace=True)
df.dropna(inplace=True)
df.head(2)

from factor_analyzer.factor_analyzer import calculate_bartlett_sphericity

calculate_bartlett_sphericity(df)

from factor_analyzer.factor_analyzer import calculate_kmo

kmo_all, kmo_model = calculate_kmo(df)

kmo_model

fa = FactorAnalyzer(n_factors = df.shape[1], rotation=None)
fa.fit(df)

ev, v = fa.get_eigenvalues()
ev

plt.scatter(range(1,df.shape[1]+1),ev)
plt.plot(range(1,df.shape[1]+1),ev)

import seaborn as sns

fa = FactorAnalyzer(n_factors = 6, rotation=None)
fa.fit(df)

efa_result = pd.DataFrame(fa.loadings_, index = df.columns)

plt.figure(figsize=(6,10))
sns.heatmap(efa_result, annot=True, cmap='Blues', fmt='.2f');

fa = FactorAnalyzer(n_factors = 5, rotation='varimax') #최대 우도 방법
fa.fit(df)

df1 = pd.DataFrame(fa.get_factor_variance())

df1.index = ['SS Loading','Proportion Var','Cumulative Var'] # ss loading = 각 factor의 설명력
df1

def CronbachAlpha(item):
    item = np.asarray(item)
    itemvars = item.var(axis=0, ddof=1)
    tscores = item.sum(axis=1)
    nitems = item.shape[1]
    return (nitems/(nitems-1)) * (1 - (itemvars.sum() / tscores.var(ddof=1) ))

factors = ['A','C','E','N','O']
items_dict = {}
for factor in factors:
    items_dict[factor] = [x for x in df.columns if x[0] == factor]
    
for key, value in items_dict.items():
    print(key)
    print(CronbachAlpha(df[value]))
    print()

from sklearn.cluster import KMeans
import numpy as np
X = np.array([[1, 2,3], [1, 4,2], [1, 0,0],
              [10, 2,5], [10, 4,7], [1, 0,3]])
X = DataFrame(X)
kmeans = KMeans(n_clusters=3, random_state=0).fit(X)
kmeans.labels_

kmeans.predict([[0, 0,3], [12, 3,7]])

kmeans.cluster_centers_

from collections import Counter

url = 'https://raw.githubusercontent.com/jbrownlee/Datasets/master/glass.csv'
df = pd.read_csv(url, header=None)
data = df.values
X, y = data[:, :-1], data[:, -1]

X[0]

from imblearn.under_sampling import RandomUnderSampler
X_resampled, y_resampled = RandomUnderSampler(random_state=0).fit_resample(X, y)
Counter(y_resampled)

from imblearn.over_sampling import RandomOverSampler
X_resampled, y_resampled = RandomOverSampler(random_state=0).fit_resample(X, y)
Counter(y_resampled)

from imblearn.over_sampling import SMOTE
X_resampled, y_resampled = SMOTE(random_state=0).fit_resample(X, y)
Counter(y_resampled)