Question

我有一个训练有素的分类器一直正常工作。

我尝试修改它以使用循环处理多个.csv文件，但是这已经打破了它，原来的代码（工作正常）现在返回与之前的.csv文件相同的错误处理没有任何问题。

我很困惑，无法看到什么会突然导致这个错误出现，因为之前一切正常。原始（工作）代码是;

    # -*- coding: utf-8 -*-

    import csv
    import pandas
    import numpy as np
    import sklearn.ensemble as ske
    import re
    import os
    import collections
    import pickle
    from sklearn.externals import joblib
    from sklearn import model_selection, tree, linear_model, svm


    # Load dataset
    url = 'test_6_During_100.csv'
    dataset = pandas.read_csv(url)
    dataset.set_index('Name', inplace = True)
    ##dataset = dataset[['ProcessorAffinity','ProductVersion','Handle','Company',
    ##            'UserProcessorTime','Path','Product','Description',]]

    # Open file to output everything to
    new_url = re.sub('\.csv$', '', url)
    f = open(new_url + " output report", 'w')
    f.write(new_url + " output report\n")
    f.write("\n")


    # shape
    print(dataset.shape)
    print("\n")
    f.write("Dataset shape " + str(dataset.shape) + "\n")
    f.write("\n")

    clf = joblib.load(os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            'classifier/classifier.pkl'))


    Class_0 = []
    Class_1 = []
    prob = []

    for index, row in dataset.iterrows():
        res = clf.predict([row])
        if res == 0:
            if index in malware:
                Class_0.append(index)
            elif index in Class_1:
                Class_1.append(index)           
            else:
                print "Is ", index, " recognised?"
                designation = raw_input()

                if designation == "No":
                    Class_0.append(index)
                else:
                    Class_1.append(index)

    dataset['Type']  = 1                    
    dataset.loc[dataset.index.str.contains('|'.join(Class_0)), 'Type'] = 0

    print "\n"

    results = []

    results.append(collections.OrderedDict.fromkeys(dataset.index[dataset['Type'] == 0]))
    print (results)

    X = dataset.drop(['Type'], axis=1).values
    Y = dataset['Type'].values


    clf.set_params(n_estimators = len(clf.estimators_) + 40, warm_start = True)
    clf.fit(X, Y)
    joblib.dump(clf, 'classifier/classifier.pkl')

    output = collections.Counter(Class_0)

    print "Class_0; \n"
    f.write ("Class_0; \n")

    for key, value in output.items():    
        f.write(str(key) + " ; " + str(value) + "\n")
        print(str(key) + " ; " + str(value))

    print "\n"
    f.write ("\n") 

    output_1 = collections.Counter(Class_1)

    print "Class_1; \n"
    f.write ("Class_1; \n")

    for key, value in output_1.items():    
        f.write(str(key) + " ; " + str(value) + "\n")
        print(str(key) + " ; " + str(value))

    print "\n" 

    f.close()

我的新代码是相同的，但包含在几个嵌套循环中，以保持脚本运行，同时有文件要在文件夹内处理，新代码（导致错误的代码）在下面;

# -*- coding: utf-8 -*-

import csv
import pandas
import numpy as np
import sklearn.ensemble as ske
import re
import os
import time
import collections
import pickle
from sklearn.externals import joblib
from sklearn import model_selection, tree, linear_model, svm

# Our arrays which we'll store our process details in and then later print out data for
Class_0 = []
Class_1 = []
prob = []
results = []

# Open file to output our report too
timestr = time.strftime("%Y%m%d%H%M%S")

f = open(timestr + " output report.txt", 'w')
f.write(timestr + " output report\n")
f.write("\n")

count = len(os.listdir('.'))

while (count > 0):
    # Load dataset
    for filename in os.listdir('.'):
            if filename.endswith('.csv') and filename.startswith("processes_"):

                url = filename

                dataset = pandas.read_csv(url)
                dataset.set_index('Name', inplace = True)

                clf = joblib.load(os.path.join(
                        os.path.dirname(os.path.realpath(__file__)),
                        'classifier/classifier.pkl'))               

                for index, row in dataset.iterrows():
                    res = clf.predict([row])
                    if res == 0:
                        if index in Class_0:
                            Class_0.append(index)
                        elif index in Class_1:
                            Class_1.append(index)           
                        else:
                            print "Is ", index, " recognised?"
                            designation = raw_input()

                            if designation == "No":
                                Class_0.append(index)
                            else:
                                Class_1.append(index)

                dataset['Type']  = 1                    
                dataset.loc[dataset.index.str.contains('|'.join(Class_0)), 'Type'] = 0

                print "\n"

                results.append(collections.OrderedDict.fromkeys(dataset.index[dataset['Type'] == 0]))
                print (results)

                X = dataset.drop(['Type'], axis=1).values
                Y = dataset['Type'].values


                clf.set_params(n_estimators = len(clf.estimators_) + 40, warm_start = True)
                clf.fit(X, Y)
                joblib.dump(clf, 'classifier/classifier.pkl')

                os.remove(filename) 


output = collections.Counter(Class_0)

print "Class_0; \n"
f.write ("Class_0; \n")

for key, value in output.items():    
    f.write(str(key) + " ; " + str(value) + "\n")
    print(str(key) + " ; " + str(value))

print "\n"
f.write ("\n") 

output_1 = collections.Counter(Class_1)

print "Class_1; \n"
f.write ("Class_1; \n")

for key, value in output_1.items():    
    f.write(str(key) + " ; " + str(value) + "\n")
    print(str(key) + " ; " + str(value))

print "\n" 

f.close()

错误（IndexError: index 1 is out of bounds for size 1）引用了预测行res = clf.predict([row])。据我所知，问题在于没有足够的＆＃34;类＆＃34;或数据的标签类型（我是二进制分类器）？但我一直在使用这个确切的方法（在嵌套循环之外）没有任何问题。

https://codeshare.io/Gkpb44 - 包含上述.csv文件的.csv数据的代码共享链接。

Answer 1

问题是[row]是一个长度为1的数组。您的程序尝试访问索引1，该索引不存在（索引以0开头）。看起来您可能想要res = clf.predict(row)或者再看一下行变量。希望这会有所帮助。

Answer 2

所以我意识到问题是什么。

我已经创建了一个加载分类器的格式，然后使用warm_start我重新调整数据以更新分类器以尝试模拟增量/在线学习。当我处理其中包含两种类型的数据时，这很有效。但是，如果数据只是正数，那么当我重新拟合分类器时，它就会破坏它。

目前我已注释掉以下内容;

clf.set_params(n_estimators = len(clf.estimators_) + 40, warm_start = True)
clf.fit(X, Y)
joblib.dump(clf, 'classifier/classifier.pkl')

解决了这个问题。继续前进我可能会添加（还有另一个！）条件语句，看看我是否应该重新拟合数据。

我很想删除这个问题，但是由于我在搜索过程中没有发现任何涉及这个事实的内容，我认为如果有人发现他们有同样的问题我会留下答案。

Python机器学习训练有素的分类器错误索引超出范围

2 个答案: