파이썬으로 하는 탐색적 데이터 분석

%matplotlib inline

from pathlib import Path

import pandas as pd
import numpy as np
from scipy.stats import trim_mean
from statsmodels import robust
import wquantiles

import seaborn as sns
import matplotlib.pylab as plt

try:
    import common

    DATA = common.dataDirectory()
except ImportError:
    DATA = Path().resolve() / "data"

AIRLINE_STATS_CSV = DATA / "airline_stats.csv"
KC_TAX_CSV = DATA / "kc_tax.csv.gz"
LC_LOANS_CSV = DATA / "lc_loans.csv"
AIRPORT_DELAYS_CSV = DATA / "dfw_airline.csv"
SP500_DATA_CSV = DATA / "sp500_data.csv.gz"
SP500_SECTORS_CSV = DATA / "sp500_sectors.csv"
STATE_CSV = DATA / "state.csv"

state = pd.read_csv(STATE_CSV)
state.head(8)

state = pd.read_csv(STATE_CSV)
print(state["Population"].mean())

print(trim_mean(state["Population"], 0.1))

print(state["Population"].median())

print(np.average(state["Murder.Rate"], weights=state["Population"]))

print(wquantiles.median(state["Murder.Rate"], weights=state["Population"]))

state.head(8)

print(state["Population"].std())

print(robust.scale.mad(state["Population"]))
# print(abs(state['Population'] - state['Population'].median()).median() / 0.6744897501960817)

print(state["Murder.Rate"].quantile([0.05, 0.25, 0.5, 0.75, 0.95]))

# Table 1.4
percentages = [0.05, 0.25, 0.5, 0.75, 0.95]
df = pd.DataFrame(state["Murder.Rate"].quantile(percentages))
df.index = [f"{p * 100}%" for p in percentages]
print(df.transpose())

ax = (state["Population"] / 1_000_000).plot.box(figsize=(3, 4))
ax.set_ylabel("Population (millions)")

plt.tight_layout()
plt.show()

binnedPopulation = pd.cut(state["Population"], 10)
print(binnedPopulation.value_counts())

각각의 주가 어느 구간에 속하는지 좀 더 보기좋게 만들어봅니다.

# Table 1.5
binnedPopulation.name = "binnedPopulation"
df = pd.concat([state, binnedPopulation], axis=1)
df = df.sort_values(by="Population")

groups = []
for group, subset in df.groupby(by="binnedPopulation"):
    groups.append(
        {
            "BinRange": group,
            "Count": len(subset),
            "States": ",".join(subset.Abbreviation),
        }
    )
print(pd.DataFrame(groups))

ax = (state["Population"] / 1_000_000).plot.hist(figsize=(4, 4))
ax.set_xlabel("Population (millions)")

plt.tight_layout()
plt.show()

ax = state["Murder.Rate"].plot.hist(
    density=True, xlim=[0, 12], bins=range(1, 12), figsize=(4, 4)
)
state["Murder.Rate"].plot.density(ax=ax)
ax.set_xlabel("Murder Rate (per 100,000)")

plt.tight_layout()
plt.show()

# Table 1-6
dfw = pd.read_csv(AIRPORT_DELAYS_CSV)
print(100 * dfw / dfw.values.sum())

ax = dfw.transpose().plot.bar(figsize=(4, 4), legend=False)
ax.set_xlabel("Cause of delay")
ax.set_ylabel("Count")

plt.tight_layout()
plt.show()

sp500_sym = pd.read_csv(SP500_SECTORS_CSV)
sp500_px = pd.read_csv(SP500_DATA_CSV, index_col=0)

etfs = sp500_px.loc[
    sp500_px.index > "2012-07-01", sp500_sym[sp500_sym["sector"] == "etf"]["symbol"]
]
print(etfs.head())

fig, ax = plt.subplots(figsize=(5, 4))
ax = sns.heatmap(
    etfs.corr(),
    vmin=-1,
    vmax=1,
    cmap=sns.diverging_palette(20, 220, as_cmap=True),
    ax=ax,
)

plt.tight_layout()
plt.show()

ax = telecom.plot.scatter(x="T", y="VZ", figsize=(4, 4), marker="$\u25EF$", alpha=0.5)
# $\u25EF$는 속이 빈 원을 뜻한다.
ax.set_xlabel("ATT (T)")
ax.set_ylabel("Verizon (VZ)")
ax.axhline(0, color="grey", lw=1)
print(ax.axvline(0, color="grey", lw=1))

kc_tax = pd.read_csv(KC_TAX_CSV)
kc_tax0 = kc_tax.loc[
    (kc_tax.TaxAssessedValue < 750000)
    & (kc_tax.SqFtTotLiving > 100)
    & (kc_tax.SqFtTotLiving < 3500),
    :,
]
print(kc_tax0.shape)

ax = kc_tax0.plot.hexbin(
    x="SqFtTotLiving", y="TaxAssessedValue", gridsize=30, sharex=False, figsize=(5, 4)
)
ax.set_xlabel("Finished Square Feet")
ax.set_ylabel("Tax Assessed Value")

plt.tight_layout()
plt.show()

fig, ax = plt.subplots(figsize=(4, 4))
sns.kdeplot(data=kc_tax0.sample(10000), x="SqFtTotLiving", y="TaxAssessedValue", ax=ax)
ax.set_xlabel("Finished Square Feet")
ax.set_ylabel("Tax Assessed Value")

plt.tight_layout()
plt.show()

lc_loans = pd.read_csv(LC_LOANS_CSV)

df = crosstab.copy().loc["A":"G", :]
df.loc[:, "Charged Off":"Late"] = df.loc[:, "Charged Off":"Late"].div(df["All"], axis=0)
df["All"] = df["All"] / sum(df["All"])
perc_crosstab = df
print(perc_crosstab)

airline_stats = pd.read_csv(AIRLINE_STATS_CSV)
airline_stats.head()
ax = airline_stats.boxplot(by="airline", column="pct_carrier_delay", figsize=(5, 5))
ax.set_xlabel("")
ax.set_ylabel("Daily % of Delayed Flights")
plt.suptitle("")

plt.tight_layout()
plt.show()

fig, ax = plt.subplots(figsize=(5, 5))
sns.violinplot(
    data=airline_stats,
    x="airline",
    y="pct_carrier_delay",
    ax=ax,
    inner="quartile",
    color="white",
)
ax.set_xlabel("")
ax.set_ylabel("Daily % of Delayed Flights")

plt.tight_layout()
plt.show()

zip_codes = [98188, 98105, 98108, 98126]
kc_tax_zip = kc_tax0.loc[kc_tax0.ZipCode.isin(zip_codes), :]
kc_tax_zip


def hexbin(x, y, color, **kwargs):
    cmap = sns.light_palette(color, as_cmap=True)
    plt.hexbin(x, y, gridsize=25, cmap=cmap, **kwargs)


g = sns.FacetGrid(kc_tax_zip, col="ZipCode", col_wrap=2)
g.map(hexbin, "SqFtTotLiving", "TaxAssessedValue", extent=[0, 3500, 0, 700000])
g.set_axis_labels("Finished Square Feet", "Tax Assessed Value")
g.set_titles("Zip code {col_name:.0f}")

plt.tight_layout()
plt.show()