# 분석에 필요한 라이브러리 불러오기
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

%matplotlib inline
import seaborn as sns
import warnings

warnings.filterwarnings("ignore")

df = pd.read_csv("./input/diabetic_data.csv", na_values="?", low_memory=False)
# 결측치는 ?로 표시되어 있습니다.
df.tail()

df.describe().transpose()  # Transpose를 통해 좀더 보기 쉽게 합니다

f, ([ax0, ax1], [ax2, ax3]) = plt.subplots(
    nrows=2, ncols=2, sharey=False, figsize=(9, 9)
)
f = sns.distplot(df.time_in_hospital, kde=False, ax=ax0)
f = sns.distplot(df.num_lab_procedures, kde=True, ax=ax1)
f = sns.distplot(df.num_medications, kde=True, ax=ax2)
f = sns.distplot(df.number_diagnoses, kde=False, ax=ax3)

# 피봇 테이블 만들기
df_race = pd.pivot_table(
    df,
    index=["race", "gender"],
    columns=["readmitted"],
    values=["encounter_id"],
    aggfunc=lambda x: len(x.unique()),
    margins=True,
)
# 재입원 비율을 계산해 행을 추가합니다.
df_race["encounter_id", "readmit_ratio"] = (
    1 - df_race["encounter_id", "NO"] / df_race["encounter_id", "All"]
)
df_race

# f = df_race['encounter_id','readmit_ratio'].groupby('race').mean().plot(kind='barh')
g = df_race["encounter_id", "readmit_ratio"].reset_index()
g = g.groupby("race").mean().plot(kind="barh", legend=False)

df_null = df.isnull().apply(sum)
df_null[df_null > 0]

df["age"].replace(
    {
        "[0-10)": 5,
        "[10-20)": 15,
        "[20-30)": 25,
        "[30-40)": 35,
        "[40-50)": 45,
        "[50-60)": 55,
        "[60-70)": 65,
        "[70-80)": 75,
        "[80-90)": 85,
        "[90-100)": 95,
    },
    inplace=True,
)

df["max_glu_serum"].replace(
    {"None": 0, "Norm": 100, ">200": 200, ">300": 300}, inplace=True
)

df["A1Cresult"].replace({"None": 0, "Norm": 10, ">7": 20, ">8": 30}, inplace=True)

df["change"].replace({"No": 0, "Ch": 1}, inplace=True)

df["diabetesMed"].replace({"No": 0, "Yes": 1}, inplace=True)

medications = [
    "metformin",
    "repaglinide",
    "nateglinide",
    "chlorpropamide",
    "glimepiride",
    "acetohexamide",
    "glipizide",
    "glyburide",
    "tolbutamide",
    "pioglitazone",
    "rosiglitazone",
    "acarbose",
    "miglitol",
    "troglitazone",
    "tolazamide",
    "examide",
    "citoglipton",
    "insulin",
    "glyburide-metformin",
    "glipizide-metformin",
    "glimepiride-pioglitazone",
    "metformin-rosiglitazone",
    "metformin-pioglitazone",
]
for i in medications:
    df[i].replace({"No": -1, "Down": 0, "Steady": 1, "Up": 2}, inplace=True)

df["race"].replace(
    {
        "AfricanAmerican": 2,
        "Asian": 0,
        "Caucasian": 2,
        "Hispanic": 1,
        "Other": 1,
        None: 3,
    },
    inplace=True,
)
df["race"].astype("int64")

categoricals = [
    "gender",
    "payer_code",
    "medical_specialty",
    "diag_1",
    "diag_2",
    "diag_3",
]
for j in categoricals:
    df[j] = pd.Categorical(df[j]).codes  #  integer 값으로 바꿔 줍니다.
    df[j].astype("int64")

df["readmitted"].replace({"NO": 0, ">30": 1, "<30": 1}, inplace=True)
# 쓸모 없어 보이는 데이터 삭제
del_columns = ["weight", "payer_code", "medical_specialty"]  # 삭제할 데이터
df.drop(del_columns, inplace=True, axis=1)

df.head()

# 나이
f = sns.countplot(df.age, palette="Blues_d")

f = sns.countplot(x="age", hue="readmitted", data=df)

df_age = pd.pivot_table(
    df,
    index=["age"],
    columns=["readmitted"],
    values=["encounter_id"],
    aggfunc=lambda x: len(x.unique()),
    margins=True,
)
df_age["encounter_id", "readmitted_ratio"] = (
    df_age["encounter_id", 1] / df_age["encounter_id", "All"]
)
df_age

f = df_age["encounter_id", "readmitted_ratio"].plot(kind="barh", xlim=(0.1, 0.5))

conditions = [
    (df["age"] <= 5),
    (df["age"] >= 15) & (df["age"] < 25),
    (df["age"] >= 25) & (df["age"] < 65),
    (df["age"] >= 65) & (df["age"] < 95),
    (df["age"] >= 95),
]
choices = [0, 1, 2, 3, 4]
df["age"] = np.select(conditions, choices)

df_diag = pd.pivot_table(
    df,
    index=["number_diagnoses"],
    columns=["readmitted"],
    values=["encounter_id"],
    aggfunc=lambda x: len(x.unique()),
    margins=True,
)
df_diag["encounter_id", "readmitted_ratio"] = (
    df_diag["encounter_id", 1] / df_diag["encounter_id", "All"]
)
df_diag

conditions = [
    (df["number_diagnoses"] <= 1),
    (df["number_diagnoses"] >= 2) & (df["number_diagnoses"] < 6),
    (df["number_diagnoses"] >= 6) & (df["number_diagnoses"] < 9),
    (df["number_diagnoses"] >= 9),
]
choices = [0, 1, 2, 3]
df["number_diagnoses"] = np.select(conditions, choices)

from sklearn.model_selection import train_test_split

y = df.readmitted  # label
X = df.drop(
    ["encounter_id", "patient_nbr", "readmitted"], axis=1
)  # label과 고유값을 제거
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.2)
print(train_X.shape, train_y.shape)
print(test_X.shape, test_y.shape)

from xgboost import XGBClassifier

model = XGBClassifier(
    max_depth=6,
    learning_rate=0.1,
    n_estimators=1000,
    objective="binary:logistic",
    n_jobs=32,
    gamma=0,
    min_child_weight=1,
    subsample=0.8,
    colsample_bytree=0.8,
    scale_pos_weight=1,
    seed=1,
)
eval_set = [(train_X, train_y), (test_X, test_y)]
model.fit(
    train_X,
    train_y,
    eval_metric=["error", "logloss"],
    eval_set=eval_set,
    verbose=False,
    early_stopping_rounds=20,
)
results = model.evals_result()
# print(results)

from sklearn.metrics import accuracy_score

pred_y = model.predict(test_X)
print(" Accuracy : {} %".format(accuracy_score(test_y, pred_y) * 100))

epochs = len(results["validation_0"]["error"])  # retrieve performance metrics
x_axis = range(0, epochs)
# plot log loss
f, (ax0, ax1) = plt.subplots(nrows=1, ncols=2, sharey=False, figsize=(10, 4))
ax0.plot(x_axis, results["validation_0"]["logloss"], label="Train")
ax0.plot(x_axis, results["validation_1"]["logloss"], label="Test")
ax0.set_title("XGBoost Log Loss")
ax0.set_ylabel("Log loss")
ax0.legend()
ax1.plot(x_axis, results["validation_0"]["error"], label="Train")
ax1.plot(x_axis, results["validation_1"]["error"], label="Test")
ax1.set_ylabel("Classification Error")
ax1.set_title("XGBoost Classification Error")
ax1.legend()

from xgboost import plot_importance

f, ax = plt.subplots(figsize=(10, 8))
plot_importance(model, ax=ax)

from sklearn.model_selection import GridSearchCV

param_test1 = {"max_depth": range(3, 10, 1), "min_child_weight": range(1, 6, 1)}
gsearch1 = GridSearchCV(
    estimator=XGBClassifier(
        learning_rate=0.1,
        n_estimators=160,
        max_depth=5,
        min_child_weight=1,
        gamma=0,
        subsample=0.8,
        colsample_bytree=0.8,
        objective="binary:logistic",
        scale_pos_weight=1,
        seed=27,
    ),
    param_grid=param_test1,
    scoring="accuracy",
    n_jobs=32,
    iid=False,
    cv=5,
)
gsearch1.fit(train_X, train_y)
gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_

param_test3 = {"gamma": [i / 10.0 for i in range(0, 5)]}
gsearch3 = GridSearchCV(
    estimator=XGBClassifier(
        learning_rate=0.1,
        n_estimators=160,
        max_depth=7,
        min_child_weight=5,
        gamma=0,
        subsample=0.8,
        colsample_bytree=0.8,
        objective="binary:logistic",
        scale_pos_weight=1,
        seed=27,
    ),
    param_grid=param_test3,
    scoring="accuracy",
    n_jobs=32,
    iid=False,
    cv=5,
)
gsearch3.fit(train_X, train_y)
gsearch3.grid_scores_, gsearch3.best_params_, gsearch3.best_score_

param_test4 = {
    "subsample": [i / 10.0 for i in range(6, 10)],
    "colsample_bytree": [i / 10.0 for i in range(6, 10)],
}
gsearch4 = GridSearchCV(
    estimator=XGBClassifier(
        learning_rate=0.1,
        n_estimators=160,
        max_depth=7,
        min_child_weight=5,
        gamma=0,
        subsample=0.8,
        colsample_bytree=0.8,
        objective="binary:logistic",
        scale_pos_weight=1,
        seed=27,
    ),
    param_grid=param_test4,
    scoring="accuracy",
    n_jobs=32,
    iid=False,
    cv=5,
)
gsearch4.fit(train_X, train_y)
gsearch4.grid_scores_, gsearch4.best_params_, gsearch4.best_score_

param_test6 = {"reg_alpha": [1e-5, 1e-2, 0.1, 1, 100]}
gsearch6 = GridSearchCV(
    estimator=XGBClassifier(
        learning_rate=0.1,
        n_estimators=160,
        max_depth=7,
        min_child_weight=5,
        gamma=0,
        subsample=0.8,
        colsample_bytree=0.8,
        objective="binary:logistic",
        scale_pos_weight=1,
        seed=27,
    ),
    param_grid=param_test6,
    scoring="accuracy",
    n_jobs=32,
    iid=False,
    cv=5,
)
gsearch6.fit(train_X, train_y)
gsearch6.grid_scores_, gsearch6.best_params_, gsearch6.best_score_

model2 = XGBClassifier(
    max_depth=7,
    learning_rate=0.01,
    n_estimators=5000,
    objective="binary:logistic",
    n_jobs=32,
    reg_alpha=1,
    gamma=0,
    min_child_weight=5,
    subsample=0.8,
    colsample_bytree=0.8,
    scale_pos_weight=1,
    seed=27,
)
eval_set = [(train_X, train_y), (test_X, test_y)]
model2.fit(
    train_X,
    train_y,
    eval_metric=["error", "logloss"],
    eval_set=eval_set,
    verbose=False,
    early_stopping_rounds=50,
)
results = model2.evals_result()

pred_y = model2.predict(test_X)
print(" Accuracy : {} %".format(accuracy_score(test_y, pred_y) * 100))

epochs = len(results["validation_0"]["error"])  # retrieve performance metrics
x_axis = range(0, epochs)
# plot log loss
f, (ax0, ax1) = plt.subplots(nrows=1, ncols=2, sharey=False, figsize=(10, 4))
ax0.plot(x_axis, results["validation_0"]["logloss"], label="Train")
ax0.plot(x_axis, results["validation_1"]["logloss"], label="Test")
ax0.set_title("XGBoost Log Loss")
ax0.set_ylabel("Log loss")
ax0.legend()
ax1.plot(x_axis, results["validation_0"]["error"], label="Train")
ax1.plot(x_axis, results["validation_1"]["error"], label="Test")
ax1.set_ylabel("Classification Error")
ax1.set_title("XGBoost Classification Error")
ax1.legend()

f, ax = plt.subplots(figsize=(10, 8))
plot_importance(model, ax=ax)

#  필요한 라이브러리를 불러옵니다.
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

%matplotlib inline
import warnings  # Suppress Deprecation and Incorrect Usage Warnings

warnings.filterwarnings("ignore")

question = pd.read_csv("../input/schema.csv")
response = pd.read_csv("../input/multipleChoiceResponses.csv", encoding="ISO-8859-1")
# question.shape
question.tail()

response.tail()

print("전체 응답자의 수: ", response.shape[0])
print("전체 국가의 수: ", response["Country"].nunique())
print(
    "가장 많이 응답한 국가: ",
    response["Country"].value_counts().index[0],
    "with",
    response["Country"].value_counts().values[0],
    "respondents",
)
print(
    "가장 어린 응답자의 나이: ",
    response["Age"].min(),
    " 가장 나이 많은 응답자의 나이 : ",
    response["Age"].max(),
)

import missingno as msno  # https://github.com/ResidentMario/missingno

msno.matrix(response)

plt.subplots()
sns.countplot(
    y=response["GenderSelect"], order=response["GenderSelect"].value_counts().index
)
plt.show()

resp_coun = (
    response["Country"].value_counts()[:15].to_frame()
)  # 응답률로 정렬하고 상위 15개만 분리합니다.
sns.barplot(resp_coun["Country"], resp_coun.index)
plt.title("Top 15 Countries by number of respondents")
plt.xlabel("")  # x레이블을 지웁니다.

korean = response[response["Country"] == "South Korea"]  # 국적이 한국인 데이터만
print("한국인 응답자 수:  " + str(korean.shape[0]))

sns.distplot(response["Age"].dropna())  # 무응답 데이터 제거
print(response[response["Age"] > 0]["Age"].mean())  # 평균 나이

sns.distplot(korean["Age"].dropna())
print(korean[korean["Age"] > 0]["Age"].mean())  # 평균 나이

major_df = pd.DataFrame(
    response["MajorSelect"].value_counts()
)  # value_counts 를 사용하면 그룹화된 데이터의 카운트 값을 보여준다.
major_df["ratio"] = pd.DataFrame(
    response["MajorSelect"].value_counts(normalize=True)
)  # 해당 데이터가 전체 데이터에서 어느 정도의 비율을 차지하는지 알 수 있다.
major_df.head(10)  # 상위 10개의 전공

major_df["ratio"].plot(kind="barh")  # pandas를 이용한 간단한 시각화

sns.countplot(y="Tenure", data=response, order=response["Tenure"].value_counts().index)

response["CompensationAmount"] = response["CompensationAmount"].str.replace(",", "")
response["CompensationAmount"] = response["CompensationAmount"].str.replace("-", "")
rates = pd.read_csv("../input/conversionRates.csv")
rates.drop("Unnamed: 0", axis=1, inplace=True)
salary = response[
    [
        "CompensationAmount",
        "CompensationCurrency",
        "GenderSelect",
        "Country",
        "CurrentJobTitleSelect",
    ]
].dropna()
salary = salary.merge(
    rates, left_on="CompensationCurrency", right_on="originCountry", how="left"
)
salary["Salary"] = pd.to_numeric(salary["CompensationAmount"]) * salary["exchangeRate"]
print("최고 연봉($)", salary["Salary"].dropna().astype(int).max())
print("최적 연봉($)", salary["Salary"].dropna().astype(int).min())
print("중위 연봉($)", salary["Salary"].dropna().astype(int).median())

plt.subplots()
salary = salary[
    (salary["Salary"] < 300000) & (salary["Salary"] > 1000)
]  # 현실적인 연봉 값만 선택
sns.distplot(salary["Salary"]).set(xlim=(0, None))
plt.axvline(salary["Salary"].median(), linestyle="dashed")  # 중위값
plt.title("Salary Distribution")

plt.subplots(figsize=(8, 12))
sal_coun = (
    salary.groupby("Country")["Salary"]
    .median()
    .sort_values(ascending=False)[:30]
    .to_frame()
)
sns.barplot("Salary", sal_coun.index, data=sal_coun, palette="Spectral")
plt.axvline(salary["Salary"].median(), linestyle="dashed")
plt.title("Highest Salary Paying Countries")

f, ax = plt.subplots(figsize=(8, 12))
tool = response["MLToolNextYearSelect"].str.split(",")
tool_nxt = []
for i in tool.dropna():
    tool_nxt.extend(i)
pd.Series(tool_nxt).value_counts()[:15].sort_values(ascending=True).plot.barh(width=0.9)
ax.set_title("ML Tool Next Year")

plt.subplots(figsize=(6, 8))
learn = response["LearningPlatformSelect"].str.split(",")
platform = []
for i in learn.dropna():
    platform.extend(i)
pd.Series(platform).value_counts()[:15].sort_values(ascending=True).plot.barh(width=0.9)
plt.title("Best Platforms to Learn", size=15)

plt.subplots()
challenge = response["WorkChallengesSelect"].str.split(",")
challenges = []
for i in challenge.dropna():
    challenges.extend(i)
plt1 = (
    pd.Series(challenges).value_counts().sort_values(ascending=False).to_frame()[:5]
)  # 상위 5개만
sns.barplot(plt1[0], plt1.index)
plt.title("Challenges in Data Science")
plt.xlabel("")  # x레이블을 지웁니다.

qc = question.loc[question["Column"].str.contains("JobFactor")]
job_factors = [x for x in response.columns if x.find("JobFactor") != -1]
jfdf = {}
for feature in job_factors:
    a = response[feature].value_counts()
    a = a / a.sum()
    jfdf[feature[len("JobFactor") :]] = a
jfdf = pd.DataFrame(jfdf).transpose()
plt.figure(figsize=(8, 12))
sns.heatmap(
    jfdf.sort_values("Very Important", ascending=False), annot=True, cmap="YlGnBu"
)