【python】学習データと推論データの分布を確認する方法

#Calfornia hosing dataの読み込み
# サンプルデータ作成
data = sklearn.datasets.fetch_california_housing()
df_1=pd.DataFrame(data.data, columns=data.feature_names)

#データ分割
X_train_1, X_test_1, y_train_1, y_test_1 = train_test_split(df_1.iloc[:,:-1], df_1.Price, test_size=0.3, random_state=42)

#Taitanicデータ読み込み
#サンプルデータ作成
titanic = fetch_openml(data_id=40945, as_frame=True)
df_2=titanic.data

#データ分割
train_2,test_2=train_test_split(df_2, test_size=0.5)

pip install matplotlib-venn

def get_venn_plot(train: pd.DataFrame, test: pd.DataFrame):
    """show venn plot from train/test_dataset

    Args:
        train (pd.DataFrame): target_train_df
        test (pd.DataFrame): target_test_df
    """

    columns = test.columns
    columns_num = len(columns)
    n_cols = 4
    n_rows = columns_num // n_cols + 1

    fig, axes = plt.subplots(figsize=(n_cols*3, n_rows*3),
                                ncols=n_cols, nrows=n_rows)

    for col, ax in zip(columns, axes.ravel()):
        venn2(
            subsets=(set(train[col].unique()), set(test[col].unique())),
            set_labels=('Train', 'Test'),
            ax=ax
        )
        ax.set_title(col)

    fig.tight_layout()

get_venn_plot(train_2,test_2)

def get_numeric_features_plot( train: pd.DataFrame, test: pd.DataFrame, cont_features: list, height, figsize,hspace=.3):
    """Show Numeric Features Distribution

    Args:
        train (pd.DataFrame): train_df
        test (pd.DataFrame): test_df
        cont_features (list): target_features
        height ([float]): plot_height
        figsize ([float]): plot_size
        hspace (float, optional): space of figs. Defaults to .3.
    """

    ncols = 2
    nrows = int(math.ceil(len(cont_features)/2))

    fig, axs = plt.subplots(
        ncols=ncols, nrows=nrows, figsize=(height*2, height*nrows))
    plt.subplots_adjust(right=1.5, hspace=hspace)

    for i, feature in enumerate(cont_features):
        plt.subplot(nrows, ncols, i+1)

        # Distribution of target features
        sns.distplot(train[feature], label='Train',
                        hist=True, color='#e74c3c')
        sns.distplot(test[feature], label='Test',
                        hist=True, color='#2ecc71')
        plt.xlabel('{}'.format(feature), size=figsize, labelpad=15)
        plt.ylabel('Density', size=figsize, labelpad=15)
        plt.tick_params(axis='x', labelsize=figsize)
        plt.tick_params(axis='y', labelsize=figsize)
        plt.legend(loc='upper right', prop={'size': figsize})
        plt.legend(loc='upper right', prop={'size': figsize})
        plt.title('Distribution of {} Feature'.format(
            feature), size=figsize, y=1.05)

    plt.show()


get_numeric_features_plot(train=X_train_1,test=X_test_1,
                        cont_features=X_train_1.columns,
                        height=5,figsize=12,hspace=.4)

def categorical_count_plot( train: pd.DataFrame, test: pd.DataFrame, cat_features: list, height, figsize, hspace=.3):
    """Show Numeric Features Distribution

    Args:
        train (pd.DataFrame): train_df
        test (pd.DataFrame): test_df
        cat_features (list): target_features
        height ([float]): plot_height
        figsize ([float]): plot_size
        hspace (float, optional): space of figs. Defaults to .3.
    """

    ncols = 2
    nrows = int(math.ceil(len(cat_features)/2))
    train["type"] = "train"
    test["type"] = "test"
    whole_df = pd.concat([train, test], axis=0).reset_index(drop=True)

    fig, axs = plt.subplots(
        ncols=ncols, nrows=nrows, figsize=(height*2, height*nrows))
    plt.subplots_adjust(right=1.5, hspace=hspace)

    for i, feature in enumerate(cat_features):
        plt.subplot(nrows, ncols, i+1)

        # Distribution of target features
        ax=sns.countplot(data=whole_df, x=feature, hue="type")
        ax.set_xticklabels(ax.get_xticklabels(),rotation = 90)
        plt.xlabel('{}'.format(feature), size=figsize, labelpad=15)
        plt.ylabel('Density', size=figsize, labelpad=15)
        plt.tick_params(axis='x', labelsize=figsize)
        plt.tick_params(axis='y', labelsize=figsize)
        plt.legend(loc='upper right', prop={'size': figsize})
        plt.legend(loc='upper right', prop={'size': figsize})
        plt.title('Count of {} Feature'.format(feature), size=figsize, y=1.05)

    plt.show()

#category itemの指定
cat_items=['sex','boat']


categorical_count_plot(train=train_2,test=test_2,
                        cat_features=cat_items,
                        height=5,figsize=12,hspace=.4)