Question

我正在努力删除异常值，而我正在使用Klodian Dhana的函数来研究异常值（https://datascienceplus.com/identify-describe-plot-and-removing-the-outliers-from-the-dataset/#comment-3592066903）。

我的数据集由95000个观察组成，分为1050组，我想知道是否有办法检查组中的异常值，而不是1050次公式。

Data(DF)

Group   Height 
 Gr1    2
 Gr1    5
 Gr1    5
 Gr2    75
 Gr2    72
 Gr2    44
 Gr3    4
 Gr3    25
 Gr3    42
 …      …
 Gr1050 43

所以我想检查一下该组的异常公式，但是将它放在一个DF中。

我不是很专家所以我做了我的研究，发现在这种情况下可以使用by()，ddply()和tapply()函数。我认为循环也很有用。

Answer 1

看看以下示例。我通过从Cauchy分布中采样生成一些样本数据，以获得足够宽的尾部，根据Tukey的IQR异常值标准给出异常值。

# Sample data
set.seed(2017);
df <- cbind.data.frame(
    Group = rep(c("Gr1", "Gr2", "Gr3"), each = 20),
    Height = unlist(lapply(c(10, 20, 30), function(x) rcauchy(20, x))));
head(df);
#  Group    Height
#1   Gr1  9.757403
#2   Gr1  1.476820
#3   Gr1 20.300998
#4   Gr1 11.277766
#5   Gr1  9.118874
#6   Gr1  9.133723

# Split based on Group
ll <- split(df, df$Group);

# Flag entries based on 1.5 IQR
ll <- lapply(ll, function(x) {
    x$outlier <- ifelse(
        x$Height < quantile(x$Height, 0.25) - 1.5 * IQR(x$Height) |
        x$Height > quantile(x$Height, 0.75) + 1.5 * IQR(x$Height),
        TRUE,
        FALSE);
    return(x);
})

# Optionally replace outiers with NA
ll <- lapply(ll, function(x) {
    x$Height[x$outlier] <- NA;
    return(x);
});

# Optionally combine into single dataframe
df.filtered <- do.call(rbind.data.frame, ll);
head(df.filtered);
#      Group    Height outlier
#Gr1.1   Gr1  9.757403   FALSE
#Gr1.2   Gr1        NA    TRUE
#Gr1.3   Gr1        NA    TRUE
#Gr1.4   Gr1 11.277766   FALSE
#Gr1.5   Gr1  9.118874   FALSE
#Gr1.6   Gr1  9.133723   FALSE

在离群值检测分析之前和之后可视化分布。

# Show a comparative plot
library(ggplot2);
df.all <- rbind.data.frame(
    cbind.data.frame(df, src = "pre-outlier analysis"),
    cbind.data.frame(df.filtered[, -3], src = "post-outlier analysis"));
gg <- ggplot(df.all, aes(x = Group, y = Height));
gg <- gg + geom_boxplot() + facet_wrap(~ src, scale = "free_y");

Answer 2

这是我的方法，它利用了tidyverse包dplyr：

        #--create dataframe

    Group = c("Gr1", "Gr1", "Gr1","Gr1",
              "Gr1", "Gr1", "Gr1","Gr1", 
              "Gr2", "Gr2", "Gr2","Gr2",
              "Gr2", "Gr2", "Gr2","Gr2",
              "Gr3", "Gr3", "Gr3","Gr3",
              "Gr3", "Gr3", "Gr3","Gr3")

    Height = c(1,21,22,23,
              241,24,29,30,
               2,50,49,50,
               51,50,4900,50,
               10,10,3000,10,
               10,10,2,10) 

    grp_df = data.frame(Group, Height)

    library(dplyr) #--for group_by and summarise functions

    library(outliers) #--for outlier function

    new_df <- grp_df %>%
      group_by(Group) %>%
      summarise(lower_outlier = outlier(Height, opposite=TRUE),
                higher_outlier = outlier(Height, opposite=FALSE))

按组运行功能

2 个答案: