import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_csv('vehicles.csv')

pd.set_option('display.max_columns', None)

df.head()

df.info()

df.isna().sum()

df.describe()

df.columns

df.drop(['Unnamed: 0', 'id', 'url', 'region_url', 'VIN',
         'image_url', 'description', 'state', 'lat', 'long',
         'posting_date'], axis=1, inplace=True)

df['age'] = 2021 - df['year']
df.drop('year', axis=1, inplace=True)

len(df['manufacturer'].value_counts())

plt.figure(figsize=(8,10))
sns.countplot(data=df.fillna('n/a'), y='manufacturer', order=df.fillna('n/a')['manufacturer'].value_counts().index)

len(df['model'].value_counts())

# for model, num in zip(df['model'].value_counts().index df['model'].value_counts()):
# print(model, num)

len(df['condition'].value_counts())

len(df['cylinders'].value_counts())

sns.countplot(data=df.fillna('n/a'), y='transmission', order=df.fillna('n/a')['transmission'].value_counts().index)

sns.countplot(data=df.fillna('n/a'), y='drive', order=df.fillna('n/a')['drive'].value_counts().index)

sns.countplot(data=df.fillna('n/a'), y='size', order=df.fillna('n/a')['size'].value_counts().index)

sns.countplot(data=df.fillna('n/a'), y='type', order=df.fillna('n/a')['type'].value_counts().index)

sns.countplot(data=df.fillna('n/a'), y='paint_color', order=df.fillna('n/a')['paint_color'].value_counts().index)

df.columns

plt.figure(figsize=(8,2))
sns.rugplot(data=df, x='odometer', height=1)

sns.histplot(data=df, x='age', bins=20, kde=True)

sns.boxplot(data=df.fillna('n/a'), x='manufacturer', y='price')

df.columns

col= 'manufacturer'
counts = df[col].fillna('others').value_counts()
plt.grid()
plt.plot(range(len(counts)), counts)

# 10 정도가 적당해보임.

# 상위 10개에 포함되지 않는 분류의 경우 others로 취급
n_categorical = 10 
counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in counts.index[n_categorical:] else 'others')

df['manufacturer'].value_counts()

col= 'model'
counts = df[col].fillna('others').value_counts()
plt.grid()
plt.plot(range(len(counts)), counts)

col= 'model'
counts = df[col].fillna('others').value_counts()
plt.grid()
plt.plot(range(len(counts[:20])), counts[:20])

# manufacturer 와 달리 분류가 엄청 많아서 lambda 실행으로 느리게 작동할 수 있음.
n_categorical = 10 
#counts.index[n_categorical:]
# df[col] = df[col].apply(lambda s : s if str(s) not in counts.index[n_categorical:] else 'others')
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

df['model'].value_counts()

col = 'condition'
counts = df[col].fillna('others').value_counts()
n_categorical = 3 
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

col = 'cylinders'
counts = df[col].fillna('others').value_counts()
n_categorical = 4 
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

col = 'fuel'
# counts = df[col].fillna('others').value_counts()
# counts.index

n_categorical = 2
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

df.drop('title_status', axis=1, inplace=True)

col = 'transmission'
counts = df[col].fillna('others').value_counts()
n_categorical = 3
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

col = 'drive'
df[col].fillna('others', inplace=True)

col = 'size'
counts = df[col].fillna('others').value_counts()
n_categorical = 2
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

col = 'type'
counts = df[col].fillna('others').value_counts()
n_categorical = 8
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

df.loc[df[col] == 'other', col] = 'others'

# other이란 분류도 있어서 others에 편입.

col = 'paint_color'
counts = df[col].fillna('others').value_counts()
n_categorical = 7
others = counts.index[n_categorical:]
df[col] = df[col].apply(lambda s : s if str(s) not in others else 'others')

# age는 양호, odometer와 price 조정 필요.
p1 = df['price'].quantile(0.99)
p2 = df['price'].quantile(0.1)
print(p1, p2)

# 가격 = 상위 1% , 하위 10%를 제거하여 아웃라이어 제거.

df = df[(p1 > df['price']) & (df['price'] > p2)]

o1 = df['odometer'].quantile(0.99)
o2 = df['odometer'].quantile(0.1)
print(o1, o2)

# 주행거리 = 상위, 하위 1% 제거하여 아웃라이어 제거.

df = df[(o1 > df['odometer']) & (df['odometer'] > o2)]

df.describe()

# 아웃라이어 제거 하여 수치형 데이터 개괄

plt.figure(figsize=(10,8))
sns.boxplot(data=df, x='manufacturer', y='price')

plt.figure(figsize=(10,8))
sns.boxplot(data=df, x='model', y='price')

sns.heatmap(df.corr(), annot=True, cmap='YlOrRd')

from sklearn.preprocessing import StandardScaler

X_num = df[['odometer', 'age']]

scaler = StandardScaler()
scaler.fit(X_num)
X_scaled = scaler.transform(X_num)
X_scaled = pd.DataFrame(X_scaled, index=X_num.index, columns=X_num.columns)

# 범주형 데이터 one-hot 벡터로
X_cat = df.drop(['price', 'odometer', 'age'], axis=1)
X_cat = pd.get_dummies(X_cat)

X = pd.concat([X_scaled, X_cat], axis=1)
y = df['price']

X.head()

X.isna().sum()

X['age'].mean()

X['age'].fillna(0.0, inplace=True)

# 평균값이 0에 가깝고 따라서 nan값을 0으로 채워준다.

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1)

from xgboost import XGBRegressor

model_reg = XGBRegressor()
model_reg.fit(X_train, y_train)

from sklearn.metrics import mean_absolute_error, mean_squared_error
from math import sqrt

pred = model_reg.predict(X_test)
print(mean_absolute_error(y_test, pred))
print(sqrt(mean_squared_error(y_test, pred)))

plt.scatter(x=y_test, y=pred, alpha=0.005)
plt.plot([0,60000], [0,60000], 'r-')

err = (pred - y_test) / y_test * 100
sns.histplot(err)
plt.xlabel('error(%)')
plt.xlim(-100, 100)

# 오차율에 관한 히스토그램

sns.histplot(x=y_test, y=pred)
plt.plot([0,60000], [0,60000], 'r-')