Question

我有一个包含100多列和100万行的数据框。一列是文本数据。文本数据列包含大量句子。我已经编写了一个清理数据的代码，但它没有清理。我想删除所有停用词，“the”，“you”，“like”“for”等等。

scorel= function(sentences, pos.words, .progress='none')
{
  require(plyr)
 require(stringr)


scores = laply(sentences, function(sentence, pos.words)
{

# clean up sentences with R's regex-driven global substitute, gsub():
sentence = gsub('[[:punct:]]', '', sentence)
sentence = gsub('[[:cntrl:]]', '', sentence)
sentence = gsub('\\d+', '', sentence)
sentence = gsub("@\\w+ *", "", sentence)
# and convert to lower case:
sentence = tolower(sentence)

# split into words. str_split is in the stringr package
word.list = str_split(sentence, '\\s+')
words = unlist(word.list)
# compare our words to the dictionaries of positive & negative terms
pos.matches = match(words, pos.words)
# match() returns the position of the matched term or NA
# we just want a TRUE/FALSE:
#     pos.matches = !is.na(pos.matches)

pos.matches=!is.na(pos.matches)

# and conveniently enough, TRUE/FALSE will be treated as 1/0 by sum():
#score = sum(pos.matches)
score = sum(pos.matches)
return(score)
 }, #pos.words, neg.words, .progress=.progress )
  pos.words, .progress=.progress )

 scores.df = data.frame(score=scores, text=sentences)
 return(scores.df)
}
 Data <- read.csv("location", stringsAsFactors=FALSE)
 Data<-Data[!duplicated(Data), ]
 Text <- data.frame(as.factor(Data$speech))
 names(Text)<-"Conversation"
 textf<-Text$Conversation
 textf<- unique(textf)
Text <- as.factor(textf)

score<- scorel(Text, disgust, .progress='text')

Answer 1

使用tm包，如下所示：

corpus <- Corpus(VectorSource(sentence)) # Convert input data to corpus
corpus <- tm_map(corpus, removeWords, stopwords('english')) # Remove stop word using tm package
dataframe<-data.frame(text=unlist(sapply(corpus, `[`, "content")), 
                  stringsAsFactors=F) # Convert data back to data frame from corpus
sentence<-as.character(dataframe)

R控制台输出如下：

> sentence=c('this is an best example','A person is nice')
> sentence
[1] "this is an best example" "A person is nice"       
> corpus <- Corpus(VectorSource(sentence))
> corpus <- tm_map(corpus, removeWords, stopwords('english'))
> dataframe<-data.frame(text=unlist(sapply(corpus, `[`, "content")), 
+                       stringsAsFactors=F)
> sentence<-as.character(dataframe)
> sentence
[1] "c(\"   best example\", \"A person  nice\")"

使用R清理文本数据

1 个答案: