使用this scikit-learn教程中有关理解决策树结构的一些指导,我有一个想法,也许是看看在两个连接的节点之间发生的特征的组合,可能会对潜在的“交互作用”术语有所了解。也就是说,通过查看给定特征y
跟随给定特征x
的频率,我们可能能够确定x
和y
之间是否存在更高阶的交互作用,以及模型中的其他变量。
这是我的设置。基本上,此对象只是解析树的结构,使我们可以轻松遍历节点并确定每个节点发生了什么。
import numpy as np
class TreeInteractionFinder(object):
def __init__(
self,model,feature_names = None):
self.model = model
self.feature_names = feature_names
self._parse_tree_structure()
self._node_and_leaf_compute()
def _parse_tree_structure(self):
self.n_nodes = self.model.tree_.node_count
self.children_left = self.model.tree_.children_left
self.children_right = self.model.tree_.children_right
self.feature = self.model.tree_.feature
self.threshold = self.model.tree_.threshold
self.n_node_samples = self.model.tree_.n_node_samples
self.predicted_values = self.model.tree_.value
def _node_and_leaf_compute(self):
''' Compute node depth and whether each node is a leaf '''
node_depth = np.zeros(shape=self.n_nodes,dtype=np.int64)
is_leaves = np.zeros(shape=self.n_nodes,dtype=bool)
# Seed is the root node id and its parent depth
stack = [(0,-1)]
while stack:
node_idx,parent_depth = stack.pop()
node_depth[node_idx] = parent_depth + 1
# If we have a test (where "test" means decision-test) node
if self.children_left[node_idx] != self.children_right[node_idx]:
stack.append((self.children_left[node_idx],parent_depth + 1))
stack.append((self.children_right[node_idx],parent_depth + 1))
else:
is_leaves[node_idx] = True
self.is_leaves = is_leaves
self.node_depth = node_depth
接下来,我将在某些数据集上训练一个较深的树。波士顿住房数据集给了我一些有趣的结果,因此我在示例中使用了它:
from sklearn.datasets import load_boston as load_dataset
from sklearn.tree import DecisionTreeRegressor as model
bunch = load_dataset()
X,y = bunch.data,bunch.target
feature_names = bunch.feature_names
model = model(
max_depth=20,min_samples_leaf=2
)
model.fit(X,y)
finder = TreeInteractionFinder(model,feature_names)
from collections import defaultdict
feature_combos = defaultdict(int)
# Traverse the tree fully,counting the occurrences of features at the current and next indices
for idx in range(finder.n_nodes):
curr_node_is_leaf = finder.is_leaves[idx]
curr_feature = finder.feature_names[finder.feature[idx]]
if not curr_node_is_leaf:
# Test to see if we're at the end of the tree
try:
next_idx = finder.feature[idx + 1]
except IndexError:
break
else:
next_node_is_leaf = finder.is_leaves[next_idx]
if not next_node_is_leaf:
next_feature = finder.feature_names[next_idx]
feature_combos[frozenset({curr_feature,next_feature})] += 1
from pprint import pprint
pprint(sorted(feature_combos.items(),key=lambda x: -x[1]))
pprint(sorted(zip(feature_names,model.feature_importances_),key=lambda x: -x[1]))
哪种产量:
$ python3 *py
[(frozenset({'AGE','LSTAT'}),4),(frozenset({'RM',3),(frozenset({'AGE','NOX'}),(frozenset({'NOX','CRIM'}),'DIS'}),(frozenset({'LSTAT',2),'RM'}),(frozenset({'TAX',1),'INDUS'}),(frozenset({'PTRATIO'}),'PTRATIO'}),(frozenset({'RM'}),(frozenset({'NOX'}),(frozenset({'DIS',(frozenset({'ZN',(frozenset({'CRIM',1)]
[('RM',0.60067090411997),('LSTAT',0.22148824141475706),('DIS',0.068263421165279),('CRIM',0.03893906506019243),('NOX',0.028695328014265362),('PTRATIO',0.014211478583574726),('AGE',0.012467751974477529),('TAX',0.011821058983765207),('B',0.002420619208623876),('INDUS',0.0008323703650693053),('ZN',0.00018976111002551332),('CHAS',0.0),('RAD',0.0)]
添加标准以排除作为叶子的“下一个”节点后,结果似乎有所改善。
现在,frozenset({'AGE','LSTAT'})
是一种很常见的特征组合-即建筑物的年龄以及“人口的较低地位百分比”的组合(无论这是一种衡量标准,低收入率)。从model.feature_importances_
来看,LSTAT
和AGE
都是相对重要的销售价格预测指标,这使我相信AGE * LSTAT
的这些功能组合可能会有用。
这甚至是吠叫正确的树吗(也许是双关语)?计算给定树中的顺序特征组合是否可以说明模型中的潜在相互作用?