Complete Code

# %%
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('seaborn')

# read training data
df_train = pd.read_csv('train.csv')

# specify features and target (remove unuseful features)
df_train = df_train[['Survived', 'Sex', 'Pclass', 'SibSp', 'Fare', 'Embarked']]

# data preprocessing
df_train.dropna(inplace=True)  # drop missing values
gender_dummies = pd.get_dummies(df_train.Sex)  # one-hot encoding for Sex
embarked_dummies = pd.get_dummies(df_train.Embarked)  # one-hot encoding for Embarked
df_num = df_train[['Survived', 'Pclass', 'SibSp', 'Fare']]  # choose numerical features
df_train_processed = pd.concat([df_num, gender_dummies, embarked_dummies], axis=1)  # concatenate all dataframes into one
df_train_processed.head()

# %%
X = df_train_processed.drop('Survived', axis=1)
y = df_train_processed['Survived']

# train a DT model by using all default settings
# note the default criterion='gini', which can be changed to criterion='entropy'
from sklearn.tree import DecisionTreeClassifier
tree_clf = DecisionTreeClassifier()
tree_clf.fit(X, y)

# %%
# passenger 1 who bought a class 3 ticket at $8.5 with no siblings / spouses, Jack ? 
# passenger 2 who bought a first class ticket at $88 with no siblings / spouses, Rose ?

p = {
    'Pclass': [3, 1], # [] needed, even for one row
    'SibSp': [0, 0],
    'Fare': [7, 85],
    'female': [0, 1],
    'male': [1, 0],
    'C': [0, 0],
    'Q': [0, 0], 
    'S': [1, 1],
}

df_p = pd.DataFrame(p)

# %%
p_pred = tree_clf.predict(df_p)

# %%
df_test = pd.read_csv('test.csv')

# %%
# process test dataset for prediction
# NOTE: we CANNOT remove any rows from the test data for Kaggle prediction scoring

# read testing data
df_test = pd.read_csv('test.csv')

# select the same features as in training data X + PassengerId which we will need for submission
df_test = df_test[['PassengerId', 'Sex', 'Pclass', 'SibSp', 'Fare', 'Embarked']].copy()

# check missing values
df_test.isnull().sum()

# %%
# we have to impute the missing values for Kaggle prediction scoring
# we just fill the missing fare with the mean
fare_mean = df_test.Fare.mean()
df_test.Fare.fillna(fare_mean, inplace=True)

# %%
# sanity check for missing values
df_test.isnull().sum()

# %%
gender_dummies_test = pd.get_dummies(df_test.Sex)  # one-hot encoding for Sex
embarked_dummies_test = pd.get_dummies(df_test.Embarked)  # one-hot encoding for Embarked
df_num_test = df_test[['Pclass', 'SibSp', 'Fare']]  # choose numerical features
X_test = pd.concat([df_num_test, gender_dummies_test, embarked_dummies_test], axis=1)  # concatenate all dataframes into one
X_test.head()

# %%
y_hat = tree_clf.predict(X_test)
y_hat  # a numpy array with prediction results

# %%
# combine the final dataframe from two arrays
df_submit_dt = pd.DataFrame({
                              'PassengerId': df_test.PassengerId, 
                              'Survived': y_hat,
                            })
df_submit_dt.head()

# %%
df_submit_dt.to_csv('submit_simple_dt.csv', index=False)  

# %%
from sklearn import tree
fig, ax = plt.subplots(figsize=(25, 20))
tree.plot_tree(tree_clf)

# %%
tree_clf_simple = DecisionTreeClassifier(criterion='entropy', max_depth=3)  
tree_clf_simple.fit(X, y)

# %%
fig, ax = plt.subplots(figsize=(25, 20))
tree.plot_tree(tree_clf_simple)

# %%
# get feature and class names for visualization
cls_names = ['died', 'survived']  # died 0, survived 1
cls_names

# %%
feature_list = X.columns.values
feature_list

# %% [markdown]
# You can see a huge tree, which shows the overfitting problem we will discuss in the future. Let's set the `criterion='entropy'` and `max_depth=3` to generate a simpler tree.

# %%
fig, ax = plt.subplots(figsize=(25, 20))
tree.plot_tree(
    tree_clf_simple, 
    feature_names=feature_list, 
    class_names=cls_names, 
    filled=True, 
    proportion=True, 
    node_ids=True
    )

# %%
df_p

# %%
# use the new tree to make predictions

p_pred_simple = tree_clf_simple.predict(df_p)
p_pred_simple