Question

我有一个pandas数据框，其中包含有关用户发送的消息的信息。对于我的模型，我有兴趣预测邮件的丢失收件人i，e给定邮件的收件人A，B，C我想预测谁应该成为收件人的一部分。

我正在使用OneVsRestClassifier和LinearSVC进行多标签分类。对于功能，我想使用邮件的收件人。主体和身体。

由于收件人是用户列表，我想使用MultiLabelBinarizer转换该列。对于Subject和Body，我想使用TFIDF

我的输入pickle文件包含如下数据：所有值都是除收件人之外的字符串，这是一个set（）

[[message_id,sent_time,subject,body,set(recipients),message_type, is_sender]]

我在管道中使用功能联合与自定义变换器实现此目的如下。

from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.svm import SVC, LinearSVC
import pickle
import pandas as pd
import numpy as np

if __name__ == "__main__":
class ColumnSelector(BaseEstimator, TransformerMixin):
    def __init__(self, column):
        self.column = column

    def fit(self, X, y=None, **fit_params):
        return self

    def transform(self, X, y=None, **fit_params):
        return X[self.column]

class MultiLabelTransformer(BaseEstimator, TransformerMixin):

    def __init__(self, column):
        self.column = column

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        mlb = MultiLabelBinarizer()
        return mlb.fit_transform(X[self.column])

pipeline = Pipeline([
    ('features', FeatureUnion([
        ('subject_tfidf', Pipeline([
                    ('selector', ColumnSelector(column='Subject')),
                    ('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
                    ])),
        ('body_tfidf', Pipeline([
            ('selector', ColumnSelector(column='Body')),
            ('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
        ])),
        ('recipients_binarizer', Pipeline([
            ('multi_label', MultiLabelTransformer(column='CoRecipients'))
        ])),
    ])),
    ('classifier', OneVsRestClassifier(LinearSVC(), n_jobs=-1))
])

top_recips = ['A', 'B', 'C, 'D]
corpus_data = pickle.load(
    open("E:\\Data\\messages_items.pkl", "rb"))
df = pd.DataFrame(corpus_data, columns=[
    'MessageId', 'SentTime', 'Subject', 'Body', 'Recipients', 'MessageType', 'IsSender'])

df = df.dropna()

# add co recipients and top recipients columns
df['CoRecipients'] = df['Recipients'].apply(
    lambda r: [x for x in r if x not in top_recips])
df['TopRecipients'] = df['Recipients'].apply(
    lambda r: [x for x in top_recips if x in r])

# drop rows where top recipients = 0
df = df.loc[df['TopRecipients'].str.len() > 0]
df_train = df.loc[df['SentTime'] <= '2017-10-15']
df_test = df.loc[(df['SentTime'] > '2017-10-15') & (df['MessageType'] == 'Meeting')]

mlb = MultiLabelBinarizer(classes=top_recips)

train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])

test_x = df_train[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.fit_transform(df_train['TopRecipients'])

print "train"
pipeline.fit(train_x, train_y)

print "predict"
predictions = pipeline.predict(test_x)

print "done"

我不确定我是否正确地对CoRecipients列进行了特征化。结果看起来不对。任何线索？

更新1

更改MLB变压器的代码如下：

class MultiLabelTransformer(BaseEstimator, TransformerMixin):
        def __init__(self, column):
            self.column = column

        def fit(self, X, y=None):
            self.mlb = MultiLabelBinarizer()
            self.mlb.fit(X[self.column])
            return self

        def transform(self, X):
            return self.mlb.transform(X[self.column])

并修复了测试集以使用df_test

mlb = MultiLabelBinarizer(classes=top_recips)

train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])

test_x = df_test[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.transform(df_test['TopRecipients'])

见下面的KeyError

Traceback (most recent call last):
  File "E:\Projects\NLP\FeatureUnion.py", line 99, in <module>
    predictions = pipeline.predict(test_x)
  File "C:\Python27\lib\site-packages\sklearn\utils\metaestimators.py", line 115, in <lambda>
    out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
  File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 306, in predict
    Xt = transform.transform(Xt)
  File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 768, in transform
    for name, trans, weight in self._iter())
  File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 779, in __call__
    while self.dispatch_one_batch(iterator):
  File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 625, in dispatch_one_batch
    self._dispatch(tasks)
  File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 588, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "C:\Python27\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py", line 111, in apply_async
    result = ImmediateResult(func)
  File "C:\Python27\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py", line 332, in __init__
    self.results = batch()
  File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 131, in __call__
    return [func(*args, **kwargs) for func, args, kwargs in self.items]
  File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 571, in _transform_one
    res = transformer.transform(X)
  File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 426, in _transform
    Xt = transform.transform(Xt)
  File "E:\Projects\NLP\FeatureUnion.py", line 37, in transform
    return self.mlb.transform(X[self.column])
  File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 765, in transform
    yt = self._transform(y, class_to_index)
  File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 789, in _transform
    indices.extend(set(class_mapping[label] for label in labels))
  File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 789, in <genexpr>
    indices.extend(set(class_mapping[label] for label in labels))
KeyError: u'cf3024@gmail.com'

＆GT;更新2

工作代码

 class MultiLabelTransformer(BaseEstimator, TransformerMixin):
        def __init__(self, column, classes):
            self.column = column
            self.classes = classes
    def fit(self, X, y=None):
        self.mlb = MultiLabelBinarizer(classes=self.classes)
        self.mlb.fit(X[self.column])
        return self

    def transform(self, X):
        return self.mlb.transform(X[self.column])

# drop rows where top recipients = 0
df = df.loc[df['TopRecipients'].str.len() > 0]
df_train = df.loc[df['SentTime'] <= '2017-10-15']
df_test = df.loc[(df['SentTime'] > '2017-10-15') &
                 (df['MessageType'] == 'Meeting')]

mlb = MultiLabelBinarizer(classes=top_recips)

train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])

test_x = df_test[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.transform(df_test['TopRecipients'])

# get all unique co-recipients
co_recips = list(set([a for b in df.CoRecipients.tolist() for a in b]))

# create pipeline
pipeline = Pipeline([
    ('features', FeatureUnion(
        # list of features
        transformer_list=[
            ('subject_tfidf', Pipeline([
                    ('selector', ColumnSelector(column='Subject')),
                    ('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
                    ])),
            ('body_tfidf', Pipeline([
                ('selector', ColumnSelector(column='Body')),
                ('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
            ])),
            ('recipients_binarizer', Pipeline([
                ('multi_label', MultiLabelTransformer(column='CoRecipients', classes=co_recips))
            ]))
        ],
        # weight components in FeatureUnion
        transformer_weights={
            'subject_tfidf': 3.0,
            'body_tfidf': 1.0,
            'recipients_binarizer': 1.0,
        }
    )),
    ('classifier', OneVsRestClassifier(LinearSVC(), n_jobs=-1))
])

print "train"
pipeline.fit(train_x, train_y)

print "predict"
predictions = pipeline.predict(test_x)

Answer 1

您正在为MultiLabelBinarizer进行转换错误。您适合训练和测试数据。那不正确。

您应该始终适合训练数据并使用转换测试数据。

你两次犯了这个错误：

在MultiLabelTransformer中，您可以转换“共同收件人”
在转换test_y期间，转换'TopRecipients'

问题是当测试数据在“共同收件人”或“TopRecipients”中具有不同（或新）值时，返回的阵列将具有与训练时间不同的形状。这将导致错误的结果。

更改您的代码：

class MultiLabelTransformer(BaseEstimator, TransformerMixin):

    #Updated
    def __init__(self, column, classes):
        self.column = column
        self.classes = classes

    def fit(self, X, y=None):
        # Updated
        self.mlb = MultiLabelBinarizer(classes = self.classes)
        self.mlb.fit(X[self.column])
        return self

    def transform(self, X):
        return self.mlb.transform(X[self.column])

并且

test_y = mlb.transform(df_train['TopRecipients'])

在管道内：

....
....
('multi_label', MultiLabelTransformer(column='CoRecipients',
                                      classes=set([a for b in df.CoRecipients.tolist() for a in b]))
....
....

虽然test_y中的最后一次更改不会影响返回的数组，因为您在top_recips期间使用mlb = MultiLabelBinarizer(classes=top_recips)指定了类，但它仍然更好地仅进行转换（并且从不适合或fit_transform）测试数据。

如何使用FeatureUnion转换PipeLine中的多个功能？

1 个答案: