Question

我来自R，所以scikit API对我来说仍然很困惑。我正在学习本教程http://michelleful.github.io/code-blog/2015/06/20/pipelines/以了解管道。因此，让我们创建一个假数据集仅供参考：

x1,x2,y
foo,zoo,1
bar,moo,2
goo,too,3
roo,zoo,4
too,moo,5

我的目标很简单：在y上训练线性回归，使用x1和x2的单独tfidf矩阵，加上x1和x2的一些自定义特征（即字长等）。

让我们从仅使用x1中的tfidf的简单任务开始。这是完整的代码：

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_log_error
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.metrics import fbeta_score, make_scorer
from sklearn.base import BaseEstimator, TransformerMixin

import pandas as pd
import numpy as np
import time
import re
import math

def clip_RMSLE(y, y_pred, **kwargs):
    y_pred[y_pred < 0] = 0.0
    to_sum = [(math.log(y_pred[i] + 1) - math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
    return (sum(to_sum) * (1.0/len(y))) ** 0.5

class ColumnNgram(BaseEstimator, TransformerMixin):
    def __init__(self, colname, tokenizer, ngram_rg):
        self.colname = colname
        self.tokenizer = tokenizer
        self.ngram_rg = ngram_rg
        self.tfidf = None

    def transform(self, df, y=None):
         tfidf = TfidfVectorizer(tokenizer=self.tokenizer, ngram_range=self.ngram_rg)
         return tfidf.fit_transform(df[self.colname].values)

    def fit(self, df, y=None):
        return self


start = time.time()
seed = 1991
ngram_rg = (1,2)
RMSLE = make_scorer(clip_RMSLE, greater_is_better=False)

def tokenizer(text):
    if text:
        result = re.findall('[a-z]{2,}', text.lower())
    else:
        result = []
    return result

df = pd.read_csv('fake.csv', sep=',')
y = df['y'].values

pipeline = Pipeline([('tfidf', ColumnNgram('x1', tokenizer, ngram_rg)),
('linear_reg', LinearRegression(n_jobs=1))
])

kfold = KFold(n_splits=2, random_state=seed)
results = cross_val_score(pipeline, df, y, cv=kfold, scoring=RMSLE)
print(results)
print(results.mean())

end = time.time()
print('Timeto finish this thing: %0.2fs' % (end - start))

我收到错误ValueError: dimension mismatch，可能是因为某些条款不会出现在列车/验证折叠中。这样做的正确方法是什么？谢谢！

Scikit学习Custom Transformer尺寸不匹配

0 个答案: