以下是一种相当简单的方法来处理我的问题,仅旨在处理事情的手段。更强大的实现可能涉及使用scikit learn中的Imputer类,这意味着它也可以执行mode、median等操作,并且更擅长处理稀疏/密集矩阵。
这是基于Vivek Kumar对原始问题的评论,建议将数据分成堆栈并以这种方式进行插补,然后重新组合它们。
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
class WithinClassMeanImputer(BaseEstimator, TransformerMixin):
def __init__(self, replace_col_index, class_col_index = None, missing_values=np.nan):
self.missing_values = missing_values
self.replace_col_index = replace_col_index
self.y = None
self.class_col_index = class_col_index
def fit(self, X, y = None):
self.y = y
return self
def transform(self, X):
y = self.y
classes = np.unique(y)
stacks = []
if len(X) > 1 and len(self.y) = len(X):
if( self.class_col_index == None ):
for aclass in classes:
with_missing = X[(y == aclass) &
(X[:, self.replace_col_index] == self.missing_values)]
without_missing = X[(y == aclass) &
(X[:, self.replace_col_index] != self.missing_values)]
column = without_missing[:, self.replace_col_index]
mean = np.mean(column[without_missing[:, self.replace_col_index] != self.missing_values])
with_missing[:, self.replace_col_index] = mean
stacks.append(np.concatenate((with_missing, without_missing)))
else:
for aclass in classes:
with_missing = X[(X[:, self.class_col_index] == aclass) &
(X[:, self.replace_col_index] == self.missing_values)]
without_missing = X[(X[:, self.class_col_index] == aclass) &
(X[:, self.replace_col_index] != self.missing_values)]
column = without_missing[:, self.replace_col_index]
mean = np.mean(column[without_missing[:, self.replace_col_index] != self.missing_values])
with_missing[:, self.replace_col_index] = mean
stacks.append(np.concatenate((with_missing, without_missing)))
if len(stacks) > 1 :
X = np.concatenate(stacks)
return X