import os
import numpy as np
import pandas as pd
pd.set_option('display.max_row', 500)
pd.set_option('display.max_columns', 30)
from pandas import DataFrame

import warnings
warnings.filterwarnings("ignore")

from scipy import stats

from scipy.stats import chisquare #교차분석
xo, xe = [324, 78, 261], [371, 80, 212]
xc = DataFrame([xo, xe], columns=['Item A', 'Item B', 'Item C'], 
               index=['Obs', 'Exp'])
print(xc)

result = chisquare(xo, f_exp=xe)
result

xf, xm = [269, 83, 215], [155, 57, 181]
x = DataFrame([xf, xm], columns=['Item 1', 'Item 2', 'Item 3'], 
              index=['Female', 'Male'])
print(x)

stats.chi2_contingency(x)

from statsmodels.formula.api import ols
from statsmodels.stats.anova import anova_lm

import urllib #데이터 불러오기 용도
url = 'https://raw.githubusercontent.com/thomas-haslwanter/statsintro_python/master/ipynb/Data/data_altman/altman_910.txt'
data = np.genfromtxt(urllib.request.urlopen(url), delimiter=',')

# Sort them into groups, according to column 1
group1, group2, group3 = data[data[:,1]==1,0], data[data[:,1]==2,0], data[data[:,1]==3,0]

stats.f_oneway(group1, group2, group3)

df = pd.DataFrame(data, columns=['value', 'treatment'])    

# the "C" indicates categorical data
model = ols('value ~ C(treatment)', df).fit()
print(anova_lm(model))

inFile = 'altman_12_6.txt'
url_base = 'https://raw.githubusercontent.com/thomas-haslwanter/statsintro_python/master/ipynb/Data/data_altman/'
url = url_base + inFile
data = np.genfromtxt(urllib.request.urlopen(url), delimiter=',')

# Bring them in dataframe-format
df = pd.DataFrame(data, columns=['head_size', 'fetus', 'observer'])

formula = 'head_size ~ C(fetus) + C(observer) + C(fetus):C(observer)'
lm = ols(formula, df).fit()
print(anova_lm(lm))

import matplotlib.pyplot as plt
import seaborn as sns

##########데이터 로드
df = pd.DataFrame([
        ['A01', 2, 1, 60, 139, 'country', 0, 'fail'],
        ['A02', 3, 2, 80, 148, 'country', 0, 'fail'],
        ['A03', 3, 4, 50, 149, 'country', 0, 'fail'],
        ['A04', 5, 5, 40, 151, 'country', 0, 'pass'],
        ['A05', 7, 5, 35, 154, 'city', 0, 'pass'],
        ['A06', 2, 5, 45, 149, 'country', 0, 'fail'],
        ['A07',8, 9, 40, 155, 'city', 1, 'pass'],
        ['A08', 9, 10, 70, 155, 'city', 3, 'pass'],
        ['A09', 6, 12, 55, 154, 'city', 0, 'pass'],
        ['A10', 9, 2, 40, 156, 'city', 1, 'pass'],
        ['A11', 6, 10, 60, 153, 'city', 0, 'pass'],
        ['A12', 2, 4, 75, 151, 'country', 0, 'fail']
    ], columns=['ID', 'hour', 'attendance', 'weight', 'iq', 'region', 'library', 'pass'])

##########데이터 분석

corr = df.corr(method='pearson')
print(corr)

sns.heatmap(corr, cmap='viridis')
plt.show();

sns.set(style="white")
mask=np.zeros_like(corr,dtype=np.bool)
mask[np.triu_indices_from(mask)]=True

f,ax=plt.subplots(figsize=(12,9))
cmap=sns.diverging_palette(200,10,as_cmap=True)
sns.heatmap(corr,mask=mask,cmap=cmap,center=0,square=True,
            linewidths=0.5,cbar_kws={"shrink":1},annot=True);
#annot는 히트맵 안에 숫자 생성해주는 것
#글자 잘림은 subplots 사이즈 등 조절하면 쉽게 조절 가능

from mlxtend.preprocessing import TransactionEncoder
from mlxtend.frequent_patterns import apriori, association_rules

df1 = pd.read_csv("chipotle.tsv",error_bad_lines=False, sep='\t')

df_tmp=df1[['order_id','item_name']]
df_tmp_arr=[[]for i in range(1835)]
num=0
for i in df_tmp['item_name'] :
    df_tmp_arr[df_tmp['order_id'][num]].append(i)
    num+=1
    
df_tmp_arr.pop(0)
num=0
for i in df_tmp_arr :
    df_tmp_arr[num] = list(set(df_tmp_arr[num]))
    num+=1
df_tmp_arr[0]

te = TransactionEncoder()
#주어진 코드에서 fit 함수를 통해 dataset은 고유한 라벨을 갖게 되고 , 
#transform함수를 통해서 파이썬 리스트를 one-hot 인코딩 된 numPy 배열로 변환합니다.
te_ary = te.fit(df_tmp_arr).transform(df_tmp_arr) 
df = pd.DataFrame(te_ary, columns=te.columns_)

frequent_itemsets = apriori(df, min_support=0.05, use_colnames=True)
frequent_itemsets

association_rules(frequent_itemsets, metric="lift", min_threshold=1)

association_rules(frequent_itemsets, metric="confidence", min_threshold=0.1)

dat_M = [117, 108, 105, 89, 101, 93, 96, 108, 108, 94, 93, 112, 92, 91, 100, 96, 120, 86, 96, 95]
dat_F = [121, 101, 102, 114, 103, 105, 101, 131, 96, 109, 109, 113, 115, 94, 108, 96, 110, 112, 120, 100]

stats.ttest_1samp(dat_M, 105) #일표본 t 검정

print(stats.f_oneway(dat_M, dat_F)) # -> 검증결과 pvalue 0.01로 분산 다름 확인 완료

#독립표본 t검정
stats.ttest_ind(dat_M, dat_F, equal_var=False) #마지막 옵션은 분산 동일 여부

stats.ttest_rel(dat_M, dat_F)