Opening notes¶
slides are on https://bujniewicz.github.io/mdm-ml-for-developers/¶
code is on https://github.com/bujniewicz/mdm-ml-for-developers/¶
I'm a developer, not a data scientist¶
What is "Machine Learning"?¶
Magic?¶
Maths?¶
What's in it for developers¶
Solving different kinds of problems¶
Those described by data and with fault tolerance¶
Common myths and mistakes¶
ML code needs to be in notebooks¶
ML code can't be clean¶
ML code can't be tested¶
In [2]:
dataframe = pd.read_csv('bayes/adult.data', header=None)
dataframe.head(10)
Out[2]:
In [3]:
class SalaryModel():
# Code omitted for clarity.
def train(self):
if self.classifier is None:
self.classifier = GaussianNB()
self.classifier.fit(self.features, self.values)
def predict(self, features):
if self.classifier is None:
self.train()
return self.classifier.predict(features)
class SalaryModelValidator():
# Code omitted for clarity.
def validate(self):
self.model.set_training_dataframe(self.training_dataframe)
self.model.train()
features = self.test_dataframe[SalaryModel.FEATURE_NAMES]
predicted = self.model.predict(features)
actual = self.test_dataframe[[SalaryModel.Columns.SALARY]]
return accuracy_score(actual, predicted)
In [4]:
class SalaryModelDatasetProcessor():
# Code omitted for clarity.
SALARY_MAP = {' <=50K': 0, ' >50K': 1, ' <=50K.': 0, ' >50K.': 1}
WORKCLASS_MAP = {' State-gov': 0, ' Self-emp-not-inc': 1, ' Private': 2,
' Federal-gov': 3, ' Local-gov': 4, ' Self-emp-inc': 5,
' Without-pay': 6}
@classmethod
def prep_dataframe(cls, dataset_file):
dataframe = pd.read_csv(dataset_file, header=None, names=cls.COLUMNS)
for column in cls.COLUMNS:
dataframe[column] = dataframe[column].replace(' ?', np.nan)
dataframe = dataframe.dropna()
for dimension, feature in SalaryModel.Columns.__dict__.items():
value_map = getattr(SalaryModelMapping,'{}_MAP'.format(dimension),
None)
if value_map:
dataframe[feature] = dataframe[feature].map(value_map).astype(int)
return dataframe
In [5]:
from bayes.naive_model import SalaryModelValidator
dataframe = SalaryModelValidator.prep_dataframe('bayes/adult.data')
dataframe.head(10)
Out[5]:
In [6]:
SalaryModelValidator.run('bayes/adult.data', 'bayes/adult.test')
In [7]:
from bayes.model import SalaryModelValidator, SalaryModel
dataframe = SalaryModelValidator.prep_dataframe('bayes/adult.data')
dataframe.head(10)
Out[7]:
In [8]:
SalaryModelValidator.run('bayes/adult.data', 'bayes/adult.test')
Historical use case: preemptively spinning up new servers¶
In [9]:
dataframe = pd.read_csv('knn/data.csv', header=None)
dataframe.head(10)
Out[9]:
In [10]:
class GeoModel():
# Code omitted for clarity.
def train(self):
if self.regressor is None:
self.regressor = KNeighborsRegressor(self.k_neighbors)
self.regressor.fit(self.features, self.values)
def predict(self, features):
if self.regressor is None:
self.train()
return self.regressor.predict(features)
In [11]:
class GeoModelDatasetProcessor():
# Code omitted for clarity.
@classmethod
def prep_dataframe(cls, dataset_file):
dataframe = pd.read_csv(dataset_file, header=None, names=cls.COLUMNS)
scaler = MinMaxScaler()
features = [GeoModel.Columns.LATITUDE, GeoModel.Columns.LONGTITUDE]
value = GeoModel.Columns.ALTITUDE
to_normalize = dataframe[features]
normalized = scaler.fit_transform(to_normalize)
normalized_dataframe = pd.DataFrame(normalized, columns=features)
normalized_dataframe[value] = dataframe[value]
return normalized_dataframe
In [12]:
class GeoModelValidator():
# Code omitted for clarity.
def validate_fold(self):
test_rows = random.sample(self.dataframe.index.tolist(),
int(len(self.dataframe) / self.k_fold))
test_dataframe = self.dataframe.iloc[test_rows]
training_dataframe = self.dataframe.drop(test_rows)
self.model.set_training_dataframe(training_dataframe)
self.model.train()
columns = [GeoModel.Columns.LATITUDE, GeoModel.Columns.LONGTITUDE]
predicted = self.model.predict(test_dataframe[columns])
actual = test_dataframe[[GeoModel.Columns.ALTITUDE]]
predicted_df = test_dataframe.copy()
predicted_df[GeoModel.Columns.ALTITUDE] = predicted
error_df = test_dataframe.copy()
error = [predicted[index][0] -
actual.iloc[index][GeoModel.Columns.ALTITUDE]
for index in range(len(actual))]
error_df[GeoModel.Columns.ALTITUDE] = error
return predicted, actual, Heatmap(predicted_df, test_dataframe, error_df)
In [13]:
from knn.model import GeoModel, GeoModelValidator, Heatmap, MidpointNormalize
dataframe = GeoModelValidator.prep_dataframe('knn/data.csv')
dataframe.head(10)
Out[13]:
In [14]:
validation = None
for iteration in GeoModelValidator.run('knn/data.csv', 20, 2, 1):
validation = iteration
print('Mean absolute error: {:0.3f}m'.format(validation.error))
def plot(dataframe, kwargs):
dataframe.plot(kind='scatter',
x=GeoModel.Columns.LATITUDE,
y=GeoModel.Columns.LONGTITUDE,
c=GeoModel.Columns.ALTITUDE,
figsize=(12, 10),
**kwargs)
def plot_altitude(title):
dataframe = getattr(validation.heatmap, title)
plot(dataframe, {'title': title, 'colormap': 'plasma'})
def plot_error():
error_min = validation.heatmap.error[GeoModel.Columns.ALTITUDE].min()
error_max = validation.heatmap.error[GeoModel.Columns.ALTITUDE].max()
plot(validation.heatmap.error, {'title': 'error', 'colormap': 'seismic',
'norm': MidpointNormalize(error_min,
error_max, 0.)})
In [16]:
plot_altitude('predicted')
In [17]:
plot_altitude('actual')
In [18]:
plot_error()
Closing notes¶