用于过滤掉字符串中常见单词的JavaScript代码

Question

我正在尝试构建读取一个字符串的JavaScript代码（比如说英文文本的句子），然后输出另一串（逗号分隔的）“不常见”的字符串。类似的东西：

    var sentence="The dog ran to the other side of the field."; 

    var common_words="the, it is, we all, a, an, by, to, you, me, he, she, they, we, how, it, i, are, to, for, of"; 

- 一些JavaScript代码 -

    var uncommon_words="dog, ran, other, side, field"; 

我该怎么做？

Answer 1

你走了：

function getUncommon(sentence, common) {
    var wordArr = sentence.match(/\w+/g),
        commonObj = {},
        uncommonArr = [],
        word, i;

    common = common.split(',');
    for ( i = 0; i < common.length; i++ ) {
        commonObj[ common[i].trim() ] = true;
    }

    for ( i = 0; i < wordArr.length; i++ ) {
        word = wordArr[i].trim().toLowerCase();
        if ( !commonObj[word] ) {
            uncommonArr.push(word);
        }
    }

    return uncommonArr;
}

现场演示： http://jsfiddle.net/simevidas/knXkS/

Answer 2

你要删除的单词叫做停止词，女巫是：

["a", "able", "about", "across", "after", "all", "almost", "also", "am", "among", "an", "and", "any", "are", "as", "at", "be", "because", "been", "but", "by", "can", "cannot", "could", "dear", "did", "do", "does", "either", "else", "ever", "every", "for", "from", "get", "got", "had", "has", "have", "he", "her", "hers", "him", "his", "how", "however", "i", "if", "in", "into", "is", "it", "its", "just", "least", "let", "like", "likely", "may", "me", "might", "most", "must", "my", "neither", "no", "nor", "not", "of", "off", "often", "on", "only", "or", "other", "our", "own", "rather", "said", "say", "says", "she", "should", "since", "so", "some", "than", "that", "the", "their", "them", "then", "there", "these", "they", "this", "tis", "to", "too", "twas", "us", "wants", "was", "we", "were", "what", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "would", "yet", "you", "your", "ain't", "aren't", "can't", "could've", "couldn't", "didn't", "doesn't", "don't", "hasn't", "he'd", "he'll", "he's", "how'd", "how'll", "how's", "i'd", "i'll", "i'm", "i've", "isn't", "it's", "might've", "mightn't", "must've", "mustn't", "shan't", "she'd", "she'll", "she's", "should've", "shouldn't", "that'll", "that's", "there's", "they'd", "they'll", "they're", "they've", "wasn't", "we'd", "we'll", "we're", "weren't", "what'd", "what's", "when'd", "when'll", "when's", "where'd", "where'll", "where's", "who'd", "who'll", "who's", "why'd", "why'll", "why's", "won't", "would've", "wouldn't", "you'd", "you'll", "you're", "you've"]

这里是来源： http://99webtools.com/list-english-stop-words.php

所以你的代码应该是

function getNoneStopWords(sentence) {
        var common = getStopWords();
        var wordArr = sentence.match(/\w+/g),
            commonObj = {},
            uncommonArr = [],
            word, i;

        for (i = 0; i < common.length; i++) {
            commonObj[ common[i].trim() ] = true;
        }

        for (i = 0; i < wordArr.length; i++) {
            word = wordArr[i].trim().toLowerCase();
            if (!commonObj[word]) {
                uncommonArr.push(word);
            }
        }
        return uncommonArr;
    }

    function getStopWords() {
        return ["a", "able", "about", "across", "after", "all", "almost", "also", "am", "among", "an", "and", "any", "are", "as", "at", "be", "because", "been", "but", "by", "can", "cannot", "could", "dear", "did", "do", "does", "either", "else", "ever", "every", "for", "from", "get", "got", "had", "has", "have", "he", "her", "hers", "him", "his", "how", "however", "i", "if", "in", "into", "is", "it", "its", "just", "least", "let", "like", "likely", "may", "me", "might", "most", "must", "my", "neither", "no", "nor", "not", "of", "off", "often", "on", "only", "or", "other", "our", "own", "rather", "said", "say", "says", "she", "should", "since", "so", "some", "than", "that", "the", "their", "them", "then", "there", "these", "they", "this", "tis", "to", "too", "twas", "us", "wants", "was", "we", "were", "what", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "would", "yet", "you", "your", "ain't", "aren't", "can't", "could've", "couldn't", "didn't", "doesn't", "don't", "hasn't", "he'd", "he'll", "he's", "how'd", "how'll", "how's", "i'd", "i'll", "i'm", "i've", "isn't", "it's", "might've", "mightn't", "must've", "mustn't", "shan't", "she'd", "she'll", "she's", "should've", "shouldn't", "that'll", "that's", "there's", "they'd", "they'll", "they're", "they've", "wasn't", "we'd", "we'll", "we're", "weren't", "what'd", "what's", "when'd", "when'll", "when's", "where'd", "where'll", "where's", "who'd", "who'll", "who's", "why'd", "why'll", "why's", "won't", "would've", "wouldn't", "you'd", "you'll", "you're", "you've"];
    }

Answer 3

这个怎么样？

sentence.replace(/\b(?:the|it is|we all|an?|by|to|you|[mh]e|she|they|we...)\b/ig, '');

这应该删除你句子中的所有常用词。只需按照您想要的方式拆分剩余的字符串。

Answer 4

首先构建一个常用词的关联数组，然后对序列进行标记以输出其中未包含的任何单词。 E.g。

var excluded = new Object();
common_words = common_words.split(",");
for (var i in common_words) {
    excluded[common_words[i].trim().toLowerCase()] = true;
}
var result = new Array();
var match = sentence.match(/\w+/g);
for (var i in match) {
    if (!excluded[match[i].toLowerCase()]) {
        result.push(match[i]);
    }
}
var uncommon_words = result.join(", ");

Answer 5

这是一个开始，我想：

var sentence_arr = sentence.split(/(?=\w)\b|\W/);
var common_arr = common_words.split(', ');

var uncommon_arr = array();
for(var i = 0; i < sentence_arr.length; i++) {
    for ( var j = 0; j < common_arr.length; j++ ) {
        if ( sentence_arr[i].toLowerCase() != common_arr[j].toLowerCase() ) {
            uncommon_arr.push(sentence_arr[i].toLowerCase());
    }
}

var uncommon_words = uncommon_arr.join(', ');

完全未经测试，但重点是你将两个句子分开，并针对该列表的每个成员单独检查每个单词。有点天真，并且完全没有规模，但对于这样的小例子会很好。

Answer 6

String#diff函数返回差异列表（不常见的术语）。这些术语可以作为数组或字符串提供。

你称之为：sentence.diff(terms)。以下是单元测试：

var sentence = 'The dog ran to the other side of the field.';
var terms    = 'the, it is, we all, a, an, by, to, you, me, he, she, they, we, how, it, i, are, to, for, of';
// NOTE: The "terms" variable could also be an array.

(sentence.diff(terms).toString() === 'dog,ran,other,side,field')
  ? console.log('pass')
  : console.log('fail');

以下是' String.diff '函数定义：

String.prototype.diff = function(terms){
  if (!terms) {
    return [];
  }

  if (typeof terms === 'string') {
    terms = terms.split(/,[\s]*/);
  }

  if (typeof terms !== 'object' || !Array.isArray(terms)) {
    return [];
  }

  terms = terms.map(function(term){
    return term.toLowerCase();
  });

  var words = this.split(/[\W]/).filter(function(word){
    return word.length;
  });

  return words.filter(function(word){
    return terms.indexOf(word.toLowerCase()) < 0;
  });
};