如何使用pandas计算所有列之间的相关性并删除高度相关的列？

我相信这必须以迭代的方式完成：

uncorrelated_features = features.copy()

# Loop until there's nothing to drop
while True:
    # Calculating the correlation matrix for the remaining list of features
    cor = uncorrelated_features.corr().abs()

    # Generating a square matrix with all 1s except for the main axis
    zero_main = np.triu(np.ones(cor.shape), k=1) +
        np.tril(np.ones(cor.shape), k=-1)

    # Using the zero_main matrix to filter out the main axis of the correlation matrix
    except_main = cor.where(zero_main.astype(bool))

    # Calculating some metrics for each column, including the max correlation,
    # mean correlation and the name of the column
    mertics = [(except_main[column].max(), except_main[column].mean(), column) for column in except_main.columns]

    # Sort the list to find the most suitable candidate to drop at index 0
    mertics.sort(key=lambda x: (x[0], x[1]), reverse=True)

    # Check and see if there's anything to drop from the list of features
    if mertics[0][0] > 0.5:
        uncorrelated_features.drop(mertics[0][2], axis=1, inplace=True)
    else:
        break

to_drop = pd.DataFrame(to_drop).fillna(True)
to_drop = list(to_drop[to_drop['SalePrice'] <.4 ].index)
df_h1.drop(to_drop,axis=1)

df_h1是我的数据框，SalePrice是因变量...我认为改变这个值可能适用于所有其他问题

from statsmodels.stats.outliers_influence import variance_inflation_factor

def vif(X):
    vif = pd.DataFrame()
    vif['Variables'] = X.columns
    vif["VIF"] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]
    return vif

其中 X 是 DataFrame。涉及到多重共线性的列的 VIF 值将大于 10。对于可以通过其他可用列的线性组合完美再现的列，它的 VIF 值将是无穷大。因此，现在逐一删除列，直到所有无穷大值和更高的 VIF 值被删除。

我用自己的方式编写代码，而不使用任何for循环来从pandas数据框中删除高协方差数据。

#get co variance of data
coVar = df.corr() # or df.corr().abs()
threshold = 0.5 # 
"""
1. .where(coVar != 1.0) set NaN where col and index is 1
2. .where(coVar >= threshold) if not greater than threshold set Nan
3. .fillna(0) Fill NaN with 0
4. .sum() convert data frame to serise with sum() and just where is co var greater than threshold sum it
5. > 0 convert all Series to Boolean
"""

coVarCols = coVar.where(coVar != 1.0).where(coVar >=threshold).fillna(0).sum() > 0

# Not Boolean Becuase we need to delete where is co var greater than threshold 
coVarCols = ~coVarCols

# get where you want
df[coVarCols[coVarCols].index]

我希望这可以帮助您使用自己的pandas函数来处理任何for循环，这可以帮助提高您在大型数据集上的速度。

correlatedColumns = []
corr = df.corr()
indices = corr.index
columns = corr.columns
posthreshold = 0.7
negthreshold = -0.7

for c in columns:
    for r in indices:
        if c != r and (corr[c][r] > posthreshold or corr[c][r] < negthreshold):
            correlatedColumns.append({"column" : c , "row" : r , "val" :corr[c][r] })
            

print(correlatedColumns)

这是我上个月在工作中使用的方法。也许不是最好或最快的方式，但它可以很好地工作。在这里，df是我的原始Pandas数据框：

dropvars = []
threshold = 0.95
df_corr = df.corr().stack().reset_index().rename(columns={'level_0': 'Var 1', 'level_1': 'Var 2', 0: 'Corr'})
df_corr = df_corr[(df_corr['Corr'].abs() >= threshold) & (df_corr['Var 1'] != df_corr['Var 2'])]
while len(df_corr) > 0:
    var = df_corr['Var 1'].iloc[0]
    df_corr = df_corr[((df_corr['Var 1'] != var) & (df_corr['Var 2'] != var))]
    dropvars.append(var)
df.drop(columns=dropvars, inplace=True)

def uncorrelated_features(df, threshold=0.7):
    """
    Returns a subset of df columns with Pearson correlations
    below threshold.
    """

    corr = df.corr().abs()
    keep = []
    for i in range(len(corr.iloc[:,0])):
        above = corr.iloc[:i,i]
        if len(keep) > 0: above = above[keep]
        if len(above[above < threshold]) == len(above):
            keep.append(corr.columns.values[i])

    return df[keep]

l=[]
corr_matrix = df.corr().abs()

for ci in corr_matrix.columns: 
    for cj in corr_matrix.columns: 
        if (corr_matrix[ci][cj]>0.8 and ci!=cj):
            l.append(ci)
            
l = np.array(l)
to_drop = np.unique(l)
df.drop(to_drop, axis=1, inplace=True)