プログラミング

複数のデータの訓練、テストスコアを比較

2018年1月23日 by 河副太智 Leave a Comment

過学習でないかどうかを調べる

訓練セットスコアとテストセットの値が非常に近い場合は適合不足
0.9や1の場合は過学習を疑う

#決定木
from sklearn.tree import DecisionTreeClassifier
ki = DecisionTreeClassifier(random_state=0).fit(train_X, train_y)
print("ketteiki training score{:.2f}".format(ki.score(train_X,train_y)))
print("ketteiki test score{:.2f}".format(ki.score(test_X,test_y)))



#ランダムフォレスト
from sklearn.ensemble import RandomForestClassifier
mori = RandomForestClassifier(random_state=0).fit(train_X,train_y)
print("mori training score{:.2f}".format(mori.score(train_X,train_y)))
print("mori test score{:.2f}".format(mori.score(test_X,test_y)))


#ロジスティック回帰
from sklearn.linear_model import LogisticRegression
logi = LogisticRegression(C=100).fit(train_X,train_y)
print("logi training score{:.2f}".format(logi.score(train_X,train_y)))
print("logi test score{:.2f}".format(logi.score(test_X,test_y)))


# #KNN
from sklearn.neighbors import KNeighborsClassifier
KNN = KNeighborsClassifier(4).fit(train_X,train_y)
print("KNN training score{:.2f}".format(KNN.score(train_X,train_y)))
print("KNN test score{:.2f}".format(KNN.score(test_X,test_y)))

# #SVC
from sklearn.svm import SVC
svc = SVC(probability=True).fit(train_X,train_y)
print("svc training score{:.2f}".format(svc.score(train_X,train_y)))
print("svc test score{:.2f}".format(svc.score(test_X,test_y)))

# #AdaBoostClassifier
from sklearn.ensemble import AdaBoostClassifier
ada = AdaBoostClassifier().fit(train_X,train_y)
print("ada training score{:.2f}".format(ada.score(train_X,train_y)))
print("ada test score{:.2f}".format(ada.score(test_X,test_y)))

# #GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingClassifier
gra = GradientBoostingClassifier().fit(train_X,train_y)
print("gra training score{:.2f}".format(gra.score(train_X,train_y)))
print("gra test score{:.2f}".format(gra.score(test_X,test_y)))

# #GaussianNB
from sklearn.naive_bayes import GaussianNB
gaus = GaussianNB().fit(train_X,train_y)
print("gaus training score{:.2f}".format(gaus.score(train_X,train_y)))
print("gaus test score{:.2f}".format(gaus.score(test_X,test_y)))

# #LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
lda = LinearDiscriminantAnalysis().fit(train_X,train_y)
print("lda training score{:.2f}".format(lda.score(train_X,train_y)))
print("lda test score{:.2f}".format(lda.score(test_X,test_y)))

# #QuadraticDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
qua = QuadraticDiscriminantAnalysis().fit(train_X,train_y)
print("qua training score{:.2f}".format(qua.score(train_X,train_y)))
print("qua test score{:.2f}".format(qua.score(test_X,test_y)))

#決定木

from sklearn.tree import DecisionTreeClassifier

ki = DecisionTreeClassifier(random_state=0).fit(train_X, train_y)

print("ketteiki training score{:.2f}".format(ki.score(train_X,train_y)))

print("ketteiki test score{:.2f}".format(ki.score(test_X,test_y)))

#ランダムフォレスト

from sklearn.ensemble import RandomForestClassifier

mori = RandomForestClassifier(random_state=0).fit(train_X,train_y)

print("mori training score{:.2f}".format(mori.score(train_X,train_y)))

print("mori test score{:.2f}".format(mori.score(test_X,test_y)))

#ロジスティック回帰

from sklearn.linear_model import LogisticRegression

logi = LogisticRegression(C=100).fit(train_X,train_y)

print("logi training score{:.2f}".format(logi.score(train_X,train_y)))

print("logi test score{:.2f}".format(logi.score(test_X,test_y)))

# #KNN

from sklearn.neighbors import KNeighborsClassifier

KNN = KNeighborsClassifier(4).fit(train_X,train_y)

print("KNN training score{:.2f}".format(KNN.score(train_X,train_y)))

print("KNN test score{:.2f}".format(KNN.score(test_X,test_y)))

# #SVC

from sklearn.svm import SVC

svc = SVC(probability=True).fit(train_X,train_y)

print("svc training score{:.2f}".format(svc.score(train_X,train_y)))

print("svc test score{:.2f}".format(svc.score(test_X,test_y)))

# #AdaBoostClassifier

from sklearn.ensemble import AdaBoostClassifier

ada = AdaBoostClassifier().fit(train_X,train_y)

print("ada training score{:.2f}".format(ada.score(train_X,train_y)))

print("ada test score{:.2f}".format(ada.score(test_X,test_y)))

# #GradientBoostingClassifier

from sklearn.ensemble import GradientBoostingClassifier

gra = GradientBoostingClassifier().fit(train_X,train_y)

print("gra training score{:.2f}".format(gra.score(train_X,train_y)))

print("gra test score{:.2f}".format(gra.score(test_X,test_y)))

# #GaussianNB

from sklearn.naive_bayes import GaussianNB

gaus = GaussianNB().fit(train_X,train_y)

print("gaus training score{:.2f}".format(gaus.score(train_X,train_y)))

print("gaus test score{:.2f}".format(gaus.score(test_X,test_y)))

# #LinearDiscriminantAnalysis

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

lda = LinearDiscriminantAnalysis().fit(train_X,train_y)

print("lda training score{:.2f}".format(lda.score(train_X,train_y)))

print("lda test score{:.2f}".format(lda.score(test_X,test_y)))

# #QuadraticDiscriminantAnalysis

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

qua = QuadraticDiscriminantAnalysis().fit(train_X,train_y)

print("qua training score{:.2f}".format(qua.score(train_X,train_y)))

print("qua test score{:.2f}".format(qua.score(test_X,test_y)))

pipコマンド

2018年1月19日 by 河副太智 Leave a Comment

パッケージインストール

python -m pip install <PackageName>

or

python -m pip install <PackageName>==<VersionNumber>

or

python -m pip install numpy==1.11.0

python -m pip install <PackageName>

python -m pip install <PackageName>==<VersionNumber>

python -m pip install numpy==1.11.0

一括インストール

python -m pip install -r requirements.txt


テキストを作成

requirements.txt

numpy==1.11.0
six==1.10.0

python -m pip install -r requirements.txt

テキストを作成

requirements.txt

numpy==1.11.0

six==1.10.0

パッケージのアンインストール

python -m pip uninstall <PackageName>

1	python -m pip uninstall <PackageName>

インストール済みパッケージの確認

python -m pip freeze

1	python -m pip freeze

pipの有無

python -m pip -V

pip 8.1.2 from C:¥python27¥lib¥site-packages (python 2.7)と出れば
インストール済

python -m pip -V

pip 8.1.2 from C:¥python27¥lib¥site-packages (python 2.7)と出れば

インストール済

pipのインストール
https://bootstrap.pypa.io/get-pip.py
からダウンロードしてから

python get-pip.py

1	python get-pip.py

pipのアップグレード

python -m pip install --upgrade pip

1	python -m pip install --upgrade pip

複数の分類器で一気に比較

2018年1月19日 by 河副太智 Leave a Comment

複数の分類器

import pandas as pd
from sklearn.model_selection import train_test_split

df = pd.read_csv('train.csv')

df = df.drop(['Cabin','Name','PassengerId','Ticket'],axis=1)
train_X = df.drop('Survived', axis=1)
train_y = df.Survived
(train_X, test_X ,train_y, test_y) = train_test_split(train_X, train_y, test_size = 0.3, random_state = 666)


#決定木
from sklearn.tree import DecisionTreeClassifier
ki = DecisionTreeClassifier(random_state=0).fit(train_X, train_y)
print(ki.score(train_X,train_y))



#ランダムフォレスト
from sklearn.ensemble import RandomForestClassifier
mori = RandomForestClassifier(random_state=0).fit(train_X,train_y)
print(mori.score(train_X,train_y))


#ロジスティック回帰
from sklearn.linear_model import LogisticRegression
logi = LogisticRegression(C=100).fit(train_X,train_y)
print(logi.score(train_X,train_y))


#KNN
from sklearn.neighbors import KNeighborsClassifier
KNN = KNeighborsClassifier(4).fit(train_X,train_y)
print(KNN.score(train_X,train_y))

#SVC
from sklearn.svm import SVC
svc = SVC(probability=True).fit(train_X,train_y)
print(svc.score(train_X,train_y))

#AdaBoostClassifier
from sklearn.ensemble import AdaBoostClassifier
ada = AdaBoostClassifier().fit(train_X,train_y)
print(ada.score(train_X,train_y))

#GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingClassifier
gra = GradientBoostingClassifier().fit(train_X,train_y)
print(gra.score(train_X,train_y))

#GaussianNB
from sklearn.naive_bayes import GaussianNB
gaus = GaussianNB().fit(train_X,train_y)
print(gaus.score(train_X,train_y))

#LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
lda = LinearDiscriminantAnalysis().fit(train_X,train_y)
print(lda.score(train_X,train_y))

#QuadraticDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
qua = QuadraticDiscriminantAnalysis().fit(train_X,train_y)
print(qua.score(train_X,train_y))

import pandas as pd

from sklearn.model_selection import train_test_split

df = pd.read_csv('train.csv')

df = df.drop(['Cabin','Name','PassengerId','Ticket'],axis=1)

train_X = df.drop('Survived', axis=1)

train_y = df.Survived

(train_X, test_X ,train_y, test_y) = train_test_split(train_X, train_y, test_size = 0.3, random_state = 666)

#決定木

from sklearn.tree import DecisionTreeClassifier

ki = DecisionTreeClassifier(random_state=0).fit(train_X, train_y)

print(ki.score(train_X,train_y))

#ランダムフォレスト

from sklearn.ensemble import RandomForestClassifier

mori = RandomForestClassifier(random_state=0).fit(train_X,train_y)

print(mori.score(train_X,train_y))

#ロジスティック回帰

from sklearn.linear_model import LogisticRegression

logi = LogisticRegression(C=100).fit(train_X,train_y)

print(logi.score(train_X,train_y))

#KNN

from sklearn.neighbors import KNeighborsClassifier

KNN = KNeighborsClassifier(4).fit(train_X,train_y)

print(KNN.score(train_X,train_y))

#SVC

from sklearn.svm import SVC

svc = SVC(probability=True).fit(train_X,train_y)

print(svc.score(train_X,train_y))

#AdaBoostClassifier

from sklearn.ensemble import AdaBoostClassifier

ada = AdaBoostClassifier().fit(train_X,train_y)

print(ada.score(train_X,train_y))

#GradientBoostingClassifier

from sklearn.ensemble import GradientBoostingClassifier

gra = GradientBoostingClassifier().fit(train_X,train_y)

print(gra.score(train_X,train_y))

#GaussianNB

from sklearn.naive_bayes import GaussianNB

gaus = GaussianNB().fit(train_X,train_y)

print(gaus.score(train_X,train_y))

#LinearDiscriminantAnalysis

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

lda = LinearDiscriminantAnalysis().fit(train_X,train_y)

print(lda.score(train_X,train_y))

#QuadraticDiscriminantAnalysis

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

qua = QuadraticDiscriminantAnalysis().fit(train_X,train_y)

print(qua.score(train_X,train_y))

Out[]:
0.982343499197
0.967897271268
0.807383627608
0.796147672552
0.886035313002
0.837881219904
0.898876404494
0.796147672552
0.799357945425
0.813804173355

データフレームでスライスが使えない

2018年1月16日 by 河副太智 Leave a Comment

TypeError: unhashable type: ‘slice’

とエラーが出てデータフレームでスライスが使えない場合は以下のように
ilocを使う

train = pd.read_csv('train.csv', header = 0, dtype={'Age': np.float64})
test  = pd.read_csv('test.csv' , header = 0, dtype={'Age': np.float64})
full_data = [train, test]


dataset = pd.DataFrame(np.random.rand(10, 10))#random無くてもいける
y=train.iloc[0::, 1::]
X=train.iloc[0::, 0]

train = pd.read_csv('train.csv', header = 0, dtype={'Age': np.float64})

test = pd.read_csv('test.csv' , header = 0, dtype={'Age': np.float64})

full_data = [train, test]

dataset = pd.DataFrame(np.random.rand(10, 10))#random無くてもいける

y=train.iloc[0::, 1::]

X=train.iloc[0::, 0]

指定の文字を別の文字に置き換える

2018年1月16日 by 河副太智 Leave a Comment

for dataset in full_data:
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
 	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')

    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')

print (train[['Title', 'Survived']].groupby(['Title'], as_index=False).mean())

for dataset in full_data:

dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\

'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')

dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')

dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')

dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')

print (train[['Title', 'Survived']].groupby(['Title'], as_index=False).mean())

意味の同じ要素を一つに統合

2018年1月16日 by 河副太智 Leave a Comment

‘Ms’, ‘Miss’の２つは同じ意味なのでこういったものを一つに統合

O’Driscoll, Miss. Bridget

Samaan, Mr. Youssef

Arnold-Franchi, Mrs. Josef (Josefine Franchi)

Panula, Master. Juha Niilo

Nosworthy, Mr. Richard Cater

Harper, Mrs. Henry Sleeper (Myna Haxtun)

Faunthorpe, Mrs. Lizzie (Elizabeth Anne Wilkinson)

Ostby, Mr. Engelhart Cornelius

Woolner, Mr. Hugh

def get_title(name):
	title_search = re.search(' ([A-Za-z]+)\.', name)
	# If the title exists, extract and return it.
	if title_search:
		return title_search.group(1)
	return ""

for dataset in full_data:
    dataset['Title'] = dataset['Name'].apply(get_title)

print(pd.crosstab(train['Title'], train['Sex']))

def get_title(name):

title_search = re.search(' ([A-Za-z]+)\.', name)

# If the title exists, extract and return it.

if title_search:

return title_search.group(1)

return ""

for dataset in full_data:

dataset['Title'] = dataset['Name'].apply(get_title)

print(pd.crosstab(train['Title'], train['Sex']))

上記の結果以下のように名前のタイトルの一覧が出る

Sex       female  male
Title                 
Capt           0     1
Col            0     2
Countess       1     0
Don            0     1
Dr             1     6
Jonkheer       0     1
Lady           1     0
Major          0     2
Master         0    40
Miss         182     0
Mlle           2     0
Mme            1     0
Mr             0   517
Mrs          125     0
Ms             1     0
Rev            0     6
Sir            0     1

Sex female male

Title

Capt 0 1

Col 0 2

Countess 1 0

Don 0 1

Dr 1 6

Jonkheer 0 1

Lady 1 0

Major 0 2

Master 0 40

Miss 182 0

Mlle 2 0

Mme 1 0

Mr 0 517

Mrs 125 0

Ms 1 0

Rev 0 6

Sir 0 1

同じ意味の物を統合する

for dataset in full_data:
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
 	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')

    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')

print (train[['Title', 'Survived']].groupby(['Title'], as_index=False).mean())

for dataset in full_data:

dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\

'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')

dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')

dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')

dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')

print (train[['Title', 'Survived']].groupby(['Title'], as_index=False).mean())

結果

   Title  Survived
0  Master  0.575000
1    Miss  0.702703
2      Mr  0.156673
3     Mrs  0.793651
4    Rare  0.347826

Title Survived

0 Master 0.575000

1 Miss 0.702703

2 Mr 0.156673

3 Mrs 0.793651

4 Rare 0.347826