Question

我正在使用来自Kaggle的电影评论数据集进行分类任务。我苦苦挣扎的部分是一系列功能，其中一个的输出成为下一个的输入。

具体来说，在提供的代码中，函数“ word_token”采用输入“ phraselist”，对其进行标记化，然后返回标记为“ phrasedocs”的标记化文档。唯一的问题是它似乎不起作用，因为当我将理论文件“ phrasedocs”输入到下一个函数“ process_token”中时，会得到：

NameError：名称'phrasedocs'未定义

我完全愿意接受我忽略了一些简单的事情，但是我已经研究了几个小时，但我无法弄清楚。我将不胜感激。

我曾尝试对代码进行校对和调试，但是我的Python专业知识不是很好。

# This function obtains data from train.tsv

def processkaggle(dirPath, limitStr):
    # Convert the limit argument from a string to an int
    limit = int(limitStr)
    os.chdir(dirPath)
    f = open('./train.tsv', 'r')
    # Loop over lines in the file and use their first limit
    phrasedata = []
    for line in f:
        # Ignore the first line starting with Phrase, then read all lines
        if (not line.startswith('Phrase')):
            # Remove final end of line character
            line = line.strip()
            # Each line has four items, separated by tabs
            # Ignore the phrase and sentence IDs, keep the phrase and sentiment
            phrasedata.append(line.split('\t')[2:4])
    return phrasedata


# Randomize and subset data

def random_phrase(phrasedata):
    random.shuffle(phrasedata) # phrasedata initiated in function processkaggle
    phraselist = phrasedata[:limit]
    for phrase in phraselist[:10]:
        print(phrase)
    return phraselist


# Tokenization

def word_token(phraselist):
    phrasedocs=[]
    for phrase in phraselist:
        tokens=nltk.word_tokenize(phrase[0])
        phrasedocs.append((tokens, int(phrase[1])))
    return phrasedocs


# Pre-processing

# Convert all tokens to lower case
def lower_case(doc):
    return [w.lower() for w in doc]

# Clean text, fixing confusion over apostrophes
def clean_text(doc):
    cleantext=[]
    for review_text in doc:
        review_text = re.sub(r"it 's", "it is", review_text)
        review_text = re.sub(r"that 's", "that is", review_text)
        review_text = re.sub(r"\'s", "\'s", review_text)
        review_text = re.sub(r"\'ve", "have", review_text)
        review_text = re.sub(r"wo n't", "will not", review_text)
        review_text = re.sub(r"do n't", "do not", review_text)
        review_text = re.sub(r"ca n't", "can not", review_text)
        review_text = re.sub(r"sha n't", "shall not", review_text)
        review_text = re.sub(r"n\'t", "not", review_text)
        review_text = re.sub(r"\'re", "are", review_text)
        review_text = re.sub(r"\'d", "would", review_text)
        review_text = re.sub(r"\'ll", "will", review_text)
        cleantext.append(review_text)
    return cleantext

# Remove punctuation and numbers
def rem_no_punct(doc):
    remtext = []
    for text in doc:
        punctuation = re.compile(r'[-_.?!/\%@,":;\'{}<>~`()|0-9]')
        word = punctuation.sub("", text)
        remtext.append(word)
    return remtext

# Remove stopwords
def rem_stopword(doc):
    stopwords = nltk.corpus.stopwords.words('english')
    updatestopwords = [word for word in stopwords if word not in ['not','no','can','has','have','had','must','shan','do','should','was','were','won','are','cannot','does','ain','could','did','is','might','need','would']]
    return [w for w in doc if not w in updatestopwords]

# Lemmatization
def lemmatizer(doc):
    wnl = nltk.WordNetLemmatizer()
    lemma = [wnl.lemmatize(t) for t in doc]
    return lemma

# Stemming
def stemmer(doc):
    porter = nltk.PorterStemmer()
    stem = [porter.stem(t) for t in doc]
    return stem

# This function combines all the previous pre-processing functions into one, which is helpful
#   if I want to alter these settings for experimentation later

def process_token(phrasedocs):
    phrasedocs2 = []
    for phrase in phrasedocs:
        tokens = nltk.word_tokenize(phrase[0])
        tokens = lower_case(tokens)
        tokens = clean_text(tokens)
        tokens = rem_no_punct(tokens)
        tokens = rem_stopword(tokens)
        tokens = lemmatizer(tokens)
        tokens = stemmer(tokens)
        phrasedocs2.append((tokens, int(phrase[1]))) # Any words that pass through the processing
                                                        # steps above are added to phrasedocs2
    return phrasedocs2


dirPath = 'C:/Users/J/kagglemoviereviews/corpus'
processkaggle(dirPath, 5000) # returns 'phrasedata'
random_phrase(phrasedata) # returns 'phraselist'
word_token(phraselist) # returns 'phrasedocs'
process_token(phrasedocs) # returns phrasedocs2


NameError                                 Traceback (most recent call last)
<ipython-input-120-595bc4dcf121> in <module>()
      5 random_phrase(phrasedata) # returns 'phraselist'
      6 word_token(phraselist) # returns 'phrasedocs'
----> 7 process_token(phrasedocs) # returns phrasedocs2
      8 
      9 

NameError: name 'phrasedocs' is not defined

Answer 1

只需在函数内部定义“ phrasedocs”（从外部看不到），函数返回应捕获在变量中，编辑您的代码：

dirPath = 'C:/Users/J/kagglemoviereviews/corpus'
phrasedata = processkaggle(dirPath, 5000) # returns 'phrasedata'
phraselist = random_phrase(phrasedata) # returns 'phraselist'
phrasedocs = word_token(phraselist) # returns 'phrasedocs'
phrasedocs2 = process_token(phrasedocs) # returns phrasedocs2

Answer 2

您仅在函数中创建了变量pussydocs。因此，并未为该函数以外的所有其他代码定义该变量。当您将变量调用为函数的输入时，python找不到任何这样的变量。您必须在您的主代码中创建一个名为「短语文档」的变量。

如何修复NameError：未定义名称'phrasedocs'

2 个答案: