我正在使用一些python代码来预测银行发放的贷款的违约率。

我已经计算出训练集上的WOE和信息值（IV） （使用以下代码：https://github.com/Sundar0989/WOE-and-IV/blob/master/WOE_IV.ipynb?fbclid=IwAR1MvEfyGsdyTre0uPJC5WRl91dfue_t0vH5qJezwm2mAg6sjHZJg9MyDYo）。

我们还得出了2个高基数变量。但是，我们不知道如何将这些WOE分数添加到整个集合中。我们如何解决这个问题？我们如何进一步使用WOE预测目标变量？

代码：

import os 
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import scipy,pylab

读取从库，特征选择部分1接收的数据，将整个（训练）分为训练集：indexs_traintrain，验证集：indexs_val和测试集：indexs_test（70/30分为训练集和验证集-测试集和70/30分组训练-验证）

Training =
pd.read_excel('/Users/enjo/Documents/Master/DM/Data_DSC2019_STUDENTS/DSC2019_Training.xlsx',na_values=np.nan)  

Status = Training.iloc[:,-1]
Data = Training.iloc[:,0:45]

Data_missing = Data.isna()
Data_missing = Data_missing.sum()
print(Data_missing/len(Data))

"""
drop variables with more than 80% missing
"""

Drop = ['FREE_CASH_FLOW_AMT','A2_MTHS_FIRST_PCX_COREPROF_CNT','A2_MONTHS_IN_BELGIUM_CNT','A2_MTHS_snC_FIRST_COREPROF_CNT','MONTHS_SINCE_LAST_REFUSAL_CNT']  

DroppedTraining = Training.copy()
for element in Drop:
    DroppedTraining.drop(element,axis=1,inplace=True)

import numpy as np
from sklearn import datasets
from sklearn import svm
from sklearn import preprocessing
Data_preprocessed=[] #contains preprocessed data
from Preprocessing_continuous import Preprocessing_continuous #import function for preprocessing
from Preprocessing_discrete import Preprocessing_discrete #import function for preprocessing
from sklearn.model_selection import train_test_split

indices=np.arange(26962)
indices_train,indices_test  = train_test_split(indices,test_size=0.3,random_state=0)
indices_traintrain,indices_val  = train_test_split(indices_train,random_state=0)

Training['target']= Training['Label_Default'].apply(lambda x:1 if x=='Y' else 0)

Highcardinalityset=[]
Highcardinalityset = Training[['Type','INDUSTRY_CD_3','INDUSTRY_CD_4','Managing_Sales_Office_Nbr','Postal_Code_L','Product_Desc','CREDIT_TYPE_CD','accOUNT_PURPOSE_CD','A2_MARITAL_STATUS_CD','FINANCIAL_PRODUCT_TYPE_CD','A2_EMPLOYMENT_STATUS_CD','A2_RESIDENT_STATUS_CD','target']]
Highcardinalityset = Highcardinalityset.iloc[indices_traintrain]

在github上找到的功能

import pandas as pd
import numpy as np
import pandas.core.algorithms as algos
from pandas import Series
import scipy.stats.stats as stats
import re
import traceback
import string

max_bin = 20
force_bin = 3

# define a binning function
def mono_bin(Y,X,n = max_bin):

    df1 = pd.DataFrame({"X": X,"Y": Y})
    justmiss = df1[['X','Y']][df1.X.isnull()]
    notmiss = df1[['X','Y']][df1.X.notnull()]
    r = 0
    while np.abs(r) < 1:
        try:
            d1 = pd.DataFrame({"X": notmiss.X,"Y": notmiss.Y,"Bucket": pd.qcut(notmiss.X,n)})
            d2 = d1.groupby('Bucket',as_index=True)
            r,p = stats.spearmanr(d2.mean().X,d2.mean().Y)
            n = n - 1 
        except Exception as e:
            n = n - 1

    if len(d2) == 1:
        n = force_bin         
        bins = algos.quantile(notmiss.X,np.linspace(0,1,n))
        if len(np.unique(bins)) == 2:
            bins = np.insert(bins,1)
            bins[1] = bins[1]-(bins[1]/2)
        d1 = pd.DataFrame({"X": notmiss.X,"Bucket": pd.cut(notmiss.X,np.unique(bins),include_lowest=True)}) 
        d2 = d1.groupby('Bucket',as_index=True)

    d3 = pd.DataFrame({},index=[])
    d3["MIN_VALUE"] = d2.min().X
    d3["MAX_VALUE"] = d2.max().X
    d3["COUNT"] = d2.count().Y
    d3["EVENT"] = d2.sum().Y
    d3["NONEVENT"] = d2.count().Y - d2.sum().Y
    d3=d3.reset_index(drop=True)

    if len(justmiss.index) > 0:
        d4 = pd.DataFrame({'MIN_VALUE':np.nan},index=[0])
        d4["MAX_VALUE"] = np.nan
        d4["COUNT"] = justmiss.count().Y
        d4["EVENT"] = justmiss.sum().Y
        d4["NONEVENT"] = justmiss.count().Y - justmiss.sum().Y
        d3 = d3.append(d4,ignore_index=True)

    d3["EVENT_RATE"] = d3.EVENT/d3.COUNT
    d3["NON_EVENT_RATE"] = d3.NONEVENT/d3.COUNT
    d3["DIST_EVENT"] = d3.EVENT/d3.sum().EVENT
    d3["DIST_NON_EVENT"] = d3.NONEVENT/d3.sum().NONEVENT
    d3["WOE"] = np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)
    d3["IV"] = (d3.DIST_EVENT-d3.DIST_NON_EVENT)*np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)
    d3["VAR_NAME"] = "VAR"
    d3 = d3[['VAR_NAME','MIN_VALUE','MAX_VALUE','COUNT','EVENT','EVENT_RATE','NONEVENT','NON_EVENT_RATE','DIST_EVENT','DIST_NON_EVENT','WOE','IV']]       
    d3 = d3.replace([np.inf,-np.inf],0)
    d3.IV = d3.IV.sum()

    return(d3)

def char_bin(Y,X):

    df1 = pd.DataFrame({"X": X,'Y']][df1.X.notnull()]    
    df2 = notmiss.groupby('X',index=[])
    d3["COUNT"] = df2.count().Y
    d3["MIN_VALUE"] = df2.sum().Y.index
    d3["MAX_VALUE"] = d3["MIN_VALUE"]
    d3["EVENT"] = df2.sum().Y
    d3["NONEVENT"] = df2.count().Y - df2.sum().Y

    if len(justmiss.index) > 0:
        d4 = pd.DataFrame({'MIN_VALUE':np.nan},'IV']]      
    d3 = d3.replace([np.inf,0)
    d3.IV = d3.IV.sum()
    d3 = d3.reset_index(drop=True)

    return(d3)

def data_vars(df1,target):

    stack = traceback.extract_stack()
    filename,lineno,function_name,code = stack[-2]
    vars_name = re.compile(r'\((.*?)\).*$').search(code).groups()[0]
    final = (re.findall(r"[\w']+",vars_name))[-1]

    x = df1.dtypes.index
    count = -1

    for i in x:
        if i.upper() not in (final.upper()):
            if np.issubdtype(df1[i],np.number) and len(Series.unique(df1[i])) > 2:
                conv = mono_bin(target,df1[i])
                conv["VAR_NAME"] = i
                count = count + 1
            else:
                conv = char_bin(target,df1[i])
                conv["VAR_NAME"] = i            
                count = count + 1

            if count == 0:
                iv_df = conv
            else:
                iv_df = iv_df.append(conv,ignore_index=True)

    iv = pd.DataFrame({'IV':iv_df.groupby('VAR_NAME').IV.max()})
    iv = iv.reset_index()
    return(iv_df,iv)

final_iv,IV = data_vars(Highcardinalityset,Highcardinalityset.target)

final_iv

IV.sort_values('IV')

IV.to_csv('test.csv')

transform_vars_list = Highcardinalityset.columns.difference(['target'])
transform_prefix = 'new_' # leave this value blank if you need replace the original column values

transform_vars_list

for var in transform_vars_list:
    small_df = final_iv[final_iv['VAR_NAME'] == var]
    transform_dict = dict(zip(small_df.MAX_VALUE.astype(str),small_df.WOE.astype(str)))    
    replace_cmd = ''
    replace_cmd1 = ''
    for i in sorted(transform_dict.items()):        
        replace_cmd = replace_cmd + str(i[1]) + str(' if x <= ') + str(i[0]) + ' else '
        replace_cmd1 = replace_cmd1 + str(i[1]) + str(' if x == "') + str(i[0]) + '" else '
    replace_cmd = replace_cmd + '0'
    replace_cmd1 = replace_cmd1 + '0'
    if replace_cmd != '0':
        try:
            Highcardinalityset[transform_prefix + var] = Highcardinalityset[var].apply(lambda x: eval(replace_cmd))
        except:
            Highcardinalityset[transform_prefix + var] = Highcardinalityset[var].apply(lambda x: eval(replace_cmd1))

Highcardinalityset['Postal_Code_L'].value_counts()
Highcardinalityset['new_Postal_Code_L'].value_counts()

Highcardinalityset['Managing_Sales_Office_Nbr'].value_counts()
Highcardinalityset['new_Managing_Sales_Office_Nbr'].value_counts()

很高兴看到高WOE：该邮政编码很有趣：违约风险很高！

Highcardinalityset.to_excel("Highcardinalitysettraintrain.xlsx")

TrainingWOE = DroppedTraining[['Managing_Sales_Office_Nbr',"Postal_Code_L"]]
TrainingWOE["Postal_Code_L_WOE"]=Highcardinalityset[["new_Postal_Code_L"]]
TrainingWOE["Managing_Sales_Office_Nbr_WOE"]=Highcardinalityset[["new_Managing_Sales_Office_Nbr"]]

删除因IV值低而不相关的变量

Drop = ["accOUNT_PURPOSE_CD","A2_MARITAL_STATUS_CD","A2_EMPLOYMENT_STATUS_CD","A2_RESIDENT_STATUS_CD","INDUSTRY_CD_3","INDUSTRY_CD_4","Type"]
DroppedTrainingAfterIVcalc = DroppedTraining.copy()
for element in Drop:
    DroppedTrainingAfterIVcalc.drop(element,inplace=True)

剩余的预处理（44-5（由于缺少太多）-7（由于低iv）+1（已添加目标变量））