在火车数据上运行模型后，如何将逻辑回归应用于测试数据

Question

我一直在尝试学习机器学习，但无法弄清楚如何将算法应用于测试数据。在此示例中，我一直在努力尝试将用于训练数据的逻辑回归模型应用于一组新的测试数据。这两个数据集位于两个不同的csv文件中：titanic_train.csv和titanic_test.csv。我可以将模型应用于火车数据，但不能将其应用于测试数据。

我正在使用Anaconda的Jupiter笔记本和python 3运行模型。

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

#this where i load the data

train = pd.read_csv('titanic_train.csv')
test = pd.read_csv('titanic_test.csv')

#impute age


def impute_age(cols):
    Age = cols[0]
    Pclass = cols[1]

if pd.isnull(Age):
    if Pclass == 1:
        return 37
    elif Pclass == 2:
        return 29
    else:
        return 24
else:
    return Age


def convert_data(dataset):
    temp_data = dataset.copy()
    temp_data['Age'] = temp_data[['Age','Pclass']].apply(impute_age,axis=1)
    sex = pd.get_dummies(temp_data['Sex'],drop_first=True)
    embark = pd.get_dummies(temp_data['Embarked'],drop_first=True)
    temp_data.drop(['Sex','Embarked','Name','Ticket'],axis=1,inplace=True)
    temp_data = pd.concat([temp_data,sex,embark],axis=1)
    temp_data.drop('Cabin',axis=1,inplace=True)
    temp_data.dropna(inplace=True)
    return temp_data

train_dataset = convert_data(train) # titanic_train.csv
test_dataset = convert_data(test) # titanic_test.csv


from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = 
train_test_split(train.drop('Survived',axis=1), 
                                            train['Survived'], test_size=0.30, 
                                            random_state=101)

#next is the logistic regression

from sklearn.linear_model import LogisticRegression
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train)


# i then get an error 

#ValueError: could not convert string to float: 'S'
#i changed train.drop to train_dataset.drop and then passed it into the model ie this

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = 
train_test_split(train_dataset.drop('Survived',axis=1), 
                                            train['Survived'], 
test_size=0.30, 
                                            random_state=101)


from sklearn.linear_model import LogisticRegression
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train)

#output was:
#LogisticRegression(C=1.0, class_weight=None, dual=False, 
#fit_intercept=True,
#      intercept_scaling=1, max_iter=100, multi_class='warn',
 #     n_jobs=None, penalty='l2', random_state=None, solver='warn',
  #    tol=0.0001, verbose=0, warm_start=False)


#then i tried your code

predictions = logmodel.predict(test.drop('Survived', axis = 1))

#this then gives me the report

from sklearn.metrics import classification_report
print(classification_report(test['Survived'],predictions))

#and get the error   KeyError: "['Survived'] not found in axis"



#so i tried changing it

predictions = logmodel.predict(test_dataset)
from sklearn.metrics import classification_report
print(classification_report(test['Survived'],predictions))

#and got a new error:         KeyError: 'Survived'

只是为了清除列印在火车上的列和测试数据中的任何混乱

print (train.columns)
print (test.columns)

#Index(['PassengerId', 'Survived', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp',
   'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked'],
  dtype='object')
#Index(['PassengerId', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp', 'Parch',
   'Ticket', 'Fare', 'Cabin', 'Embarked'],
  dtype='object')

我希望能够将模型应用于其他文件titanic_test.csv中找到的新数据

Answer 1

首先，创建您正在训练数据集上执行的操作的功能。因为同样的操作也必须应用于测试数据集。所以你应该有这样的东西：

train = pd.read_csv('titanic_train.csv')
test = pd.read_csv('titanic_test.csv')

#impute age


def impute_age(cols):
    Age = cols[0]
    Pclass = cols[1]

    if pd.isnull(Age):
        if Pclass == 1:
            return 37
        elif Pclass == 2:
            return 29
        else:
            return 24
    else:
        return Age


def convert_data(dataset):
    temp_data = dataset.copy()
    temp_data['Age'] = temp_data[['Age','Pclass']].apply(impute_age,axis=1)
    sex = pd.get_dummies(temp_data['Sex'],drop_first=True)
    embark = pd.get_dummies(temp_data['Embarked'],drop_first=True)
    temp_data.drop(['Sex','Embarked','Name','Ticket'],axis=1,inplace=True)
    temp_data = pd.concat([temp_data,sex,embark],axis=1)
    temp_data.drop('Cabin',axis=1,inplace=True)
    temp_data.dropna(inplace=True)
    return temp_data

train_dataset = convert_data(train) # titanic_train.csv
test_dataset = convert_data(test) # titanic_test.csv


from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(train.drop('Survived',axis=1), 
                                                train['Survived'], 
test_size=0.30, 
                                                random_state=101)

#next is the logistic regression

from sklearn.linear_model import LogisticRegression
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train)

predictions = logmodel.predict(test.drop('Survived', axis = 1))

#this then gives me the report

from sklearn.metrics import classification_report
print(classification_report(test['Survived'],predictions))