Sklearn 中常用方法

分割训练集与测试集

from sklearn.model_selection import train_test_split
X = a.iloc[:,0:-1]
Y = a[\"label\"]
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.5,random_state=0)
Y_train

统计Series值出现次数

a[\"feature_1\"].value_counts()

异常数据处理

删除
a.replace(\'?\', np.nan).dropna(how = \'any\')

独热编码

import pandas as pd
a = pd.DataFrame([[1,2,3],
                  [4,5,6],
                  [1,8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])
from sklearn.preprocessing import OneHotEncoder
hotCoder=OneHotEncoder(sparse = False, handle_unknown = \"ignore\")
hot = hotCoder.fit_transform(a)
pd.DataFrame(hot)

b = pd.DataFrame([[1,2,3],
                  [4,5,6],
                  [10,8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])
hotCoder.transform(b)

多项式扩展

import pandas as pd
a = pd.DataFrame([[1,2,3],
                  [4,5,6],
                  [1,8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])
from sklearn.preprocessing import PolynomialFeatures
polyCoder = PolynomialFeatures(degree=2, include_bias=True, interaction_only=False)
df = polyCoder.fit_transform(a)
pd.DataFrame(df, columns=polyCoder.get_feature_names())

标准化

import pandas as pd
a = pd.DataFrame([[1,2,3],
                  [4,5,6],
                  [7,8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])
from sklearn.preprocessing import StandardScaler
ssCoder = StandardScaler()
df = ssCoder.fit_transform(a)
pd.DataFrame(df)

规范化，归一化

import pandas as pd
a = pd.DataFrame([[1,2,3],
                  [4,5,6],
                  [7,8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])
from sklearn.preprocessing import MinMaxScaler
ssCoder = MinMaxScaler(feature_range=[-1,2])
df = ssCoder.fit_transform(a)
pd.DataFrame(df)

LabelEncoder

from sklearn.preprocessing import LabelEncoder
import pandas as pd
a = pd.DataFrame([[\"b\",2,3],
                  [\"a\",5,6],
                  [\"a\",8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])
laCoder = LabelEncoder()
b = pd.DataFrame(laCoder.fit_transform(a[\"feature_1\"]))
pd.concat([a,b],axis=1)

dataframe样本采样

df = a.sample(frac=0.66)
df = a.sample(n=3)
pd.concat([a,df])

LinearRegression

import numpy as np
X = np.mat([[1,1],[2,1],[3,1],[4,1]])
Y = np.mat([[3.2],[4.7],[7.3],[8.5]])
from sklearn.linear_model import LinearRegression
model = LinearRegression(fit_intercept=False)
model.fit(X,Y)
model.coef_

model.score(X,Y)

Ridge

from sklearn.linear_model import Ridge
for alpha in [0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 2, 3, 5, 10]:
    clf = Ridge(alpha=alpha, max_iter=2000, solver=\"auto\",fit_intercept=True)
    clf.fit(X_train, Y_train)
    print(\"Ridge:\",mse(Y_test.values, clf.predict(X_test)))
    print(clf.n_iter_)

Lasso

from sklearn.linear_model import Lasso
for alpha in [0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 2, 3]:
    clf = Lasso(alpha=alpha, max_iter=100, fit_intercept=True)
    clf.fit(X_train, Y_train)
    print(\"Lasso:\",mse(Y_test.values, clf.predict(X_test)))
    print(clf.n_iter_)

模型评估

from sklearn.metrics import mean_squared_error
print(\"LinearRegression:\",mean_squared_error(Y_test.values, clf.predict(X_test)))

混淆矩阵

pd.crosstab(Y_test,knn.predict(X_test),rownames=[\"label\"],colnames=[\"predict\"])

保存模型

from sklearn.externals import joblib
joblib.dump(enc,\'rf.model\')
enc2 = joblib.load(\'rf.model\')

b = enc2.transform(a).toarray()
pd.DataFrame(b)

绘制函数图像

import numpy as np
import matplotlib.pyplot as plt
x=np.linspace(-5,5,1000)  #这个表示在-5到5之间生成1000个x值
y=[1/(1+np.exp(-i)) for i in x]  #对上述生成的1000个数循环用sigmoid公式求对应的y
plt.plot(x,y)  #用上述生成的1000个xy值对生成1000个点
plt.show()  #绘制图像

Df拷贝

import pandas as pd
a = pd.DataFrame([[1,2,3],
                  [4,5,6],
                  [7,8,9]],columns = [\"feature_1\", \"feature_2\", \"label\"])

df = a.copy()
df.drop(columns=[\"feature_1\"],inplace=True)
print(id(a))
print(id(df))
a

Python拷贝

import copy
a = [1,2,[1,2]]
b = copy.deepcopy(a)
a[2][0] = -1

b

CountVectorizer

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer
corpus = [
    \'我 爱 你\',
    \'我 恨 你\'
]
y = [0,1]
vectorizer = CountVectorizer(token_pattern=\"[a-zA-Z|\\u4e00-\\u9fa5]+\")
count = vectorizer.fit_transform(corpus)
print(vectorizer.get_feature_names())  
print(count.toarray())

transformer = TfidfTransformer()
tfidf_matrix = transformer.fit_transform(count)
print(tfidf_matrix.toarray())

tfidf_vec = TfidfVectorizer(token_pattern=\"[a-zA-Z|\\u4e00-\\u9fa5]+\") 
tfidf_matrix = tfidf_vec.fit_transform(corpus)
print(tfidf_matrix.toarray())

from sklearn.naive_bayes import GaussianNB
model = GaussianNB()
model.fit(tfidf_matrix.toarray(),y)
print(model.predict(tfidf_matrix.toarray()))
corpus = [
    \'仇 恨\',
    \'爱 你\'
]
tfidf_matrix = tfidf_vec.transform(corpus)
model.predict(tfidf_matrix.toarray())

TfidfVectorizer

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer
from sklearn.model_selection import train_test_split
df = pd.read_csv(\"datas/bayes.txt\",header=None)
X = df[1]
Y = df[0]
tfCoder = TfidfVectorizer(token_pattern=\"[a-zA-Z|\\u4e00-\\u9fa5]+\") 
X = tfCoder.fit_transform(X).toarray()
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0, random_state=42)
from sklearn.naive_bayes import GaussianNB
model = GaussianNB()
model.fit(X_train,y_train)
print(model.predict(X_train))
print(y_train.values)

apply

from sklearn import preprocessing
import pandas as pd
enc = preprocessing.OneHotEncoder(categorical_features=[0,1])
a = pd.DataFrame([[1,\"A\",\"a\"],
                  [0,\"B\",\"b\"],
                  [2,\"C\",\"c\"]],columns = [\"ebayno\", \"p_sku\", \"sale\"])
def f(x):
    i = x.index 
    v = x.values*2
    print(v)
    
    return pd.Series(v,i)
a.apply(f)

tfidf

corpus=[\"hi peter\", 
    \"hi tom\"] 

from sklearn.feature_extraction.text import TfidfVectorizer
tfidf2 = TfidfVectorizer(norm=None)
re = tfidf2.fit_transform(corpus)
print(tfidf2.vocabulary_)
print(tfidf2.get_feature_names())
print(re.todense())