最初にインストール
|
1 2 |
pip install pydotplus brew install graphviz |
|
1 2 3 4 5 6 7 8 9 10 11 |
#可視化 import pydotplus from IPython.display import Image from graphviz import Digraph from sklearn.externals.six import StringIO dot_data = StringIO() tree.export_graphviz(clf, out_file=dot_data,feature_names=train_X.columns, max_depth=3) graph = pydotplus.graph_from_dot_data(dot_data.getvalue()) graph.write_pdf("graph.pdf") Image(graph.create_png()) |
モデルスコア1に近いほど精度が高い
|
1 2 3 4 5 6 |
from sklearn.metrics import (roc_curve, auc, accuracy_score) pred = clf.predict(test_X) fpr, tpr, thresholds = roc_curve(test_y, pred, pos_label=1) auc(fpr, tpr) accuracy_score(pred, test_y) |
インポート用コード
|
1 2 3 4 5 6 7 8 |
pd.read_csv(filename) # From a CSV file pd.read_table(filename) # From a delimited text file (like TSV) pd.read_excel(filename) # From an Excel file pd.read_sql(query, connection_object) # Reads from a SQL table/database pd.read_json(json_string) # Reads from a JSON formatted string, URL or file. pd.read_html(url) # Parses an html URL, string or file and extracts tables to a list of dataframes pd.read_clipboard() # Takes the contents of your clipboard and passes it to read_table() pd.DataFrame(dict) # From a dict, keys for columns names, values for data as lists |
選択、特定
|
1 2 3 4 5 6 |
df[col] # Returns column with label col as Series df[[col1, col2]] # Returns Columns as a new DataFrame s.iloc[0] # Selection by position (selects first element) s.loc[0] # Selection by index (selects element at index 0) df.iloc[0,:] # First row df.iloc[0,0] # First element of first column |
データクリーニング、データクレンジング
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 |
Data CleaningPython df.columns = ['a','b','c'] # Renames columns pd.isnull() # Checks for null Values, Returns Boolean Array pd.notnull() # Opposite of s.isnull() df.dropna() # Drops all rows that contain null values df.dropna(axis=1) # Drops all columns that contain null values df.dropna(axis=1,thresh=n) # Drops all rows have have less than n non null values df.fillna(x) # Replaces all null values with x s.fillna(s.mean()) # Replaces all null values with the mean (mean can be replaced with almost any function from the statistics section) s.astype(float) # Converts the datatype of the series to float s.replace(1,'one') # Replaces all values equal to 1 with 'one' s.replace([1,3],['one','three']) # Replaces all 1 with 'one' and 3 with 'three' df.rename(columns=lambda x: x + 1) # Mass renaming of columns df.rename(columns={'old_name': 'new_ name'}) # Selective renaming df.set_index('column_one') # Changes the index df.rename(index=lambda x: x + 1) # Mass renaming of index |
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
Machine LearningPython # Import libraries and modules import numpy as np import pandas as pd from sklearn.model_selection import train_test_split from sklearn import preprocessing from sklearn.ensemble import RandomForestRegressor from sklearn.pipeline import make_pipeline from sklearn.model_selection import GridSearchCV from sklearn.metrics import mean_squared_error, r2_score from sklearn.externals import joblib # Load red wine data. dataset_url = 'http://mlr.cs.umass.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv' data = pd.read_csv(dataset_url, sep=';') # Split data into training and test sets y = data.quality X = data.drop('quality', axis=1) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123, stratify=y) # Declare data preprocessing steps pipeline = make_pipeline(preprocessing.StandardScaler(), RandomForestRegressor(n_estimators=100)) # Declare hyperparameters to tune hyperparameters = { 'randomforestregressor__max_features' : ['auto', 'sqrt', 'log2'], 'randomforestregressor__max_depth': [None, 5, 3, 1]} # Tune model using cross-validation pipeline clf = GridSearchCV(pipeline, hyperparameters, cv=10) clf.fit(X_train, y_train) # Refit on the entire training set # No additional code needed if clf.refit == True (default is True) # Evaluate model pipeline on test data pred = clf.predict(X_test) print r2_score(y_test, pred) print mean_squared_error(y_test, pred) # Save model for future use joblib.dump(clf, 'rf_regressor.pkl') # To load: clf2 = joblib.load('rf_regressor.pkl') |