Getting Started
Installation
You need to install Mlfoundry library in order to get started with monitoring. Click here to view the steps to setup the library.
Quick Start
Lets us consider a simple sklearn model trained on iris dataset to get started with monitoring.
Creating a simple sklearn model
import pandas as pd
import numpy as np
import mlfoundry as mlf
from sklearn import datasets
from sklearn import svm
from sklearn.model_selection import train_test_split
iris = datasets.load_iris()
iris_frame = pd.DataFrame(iris.data, columns = iris.feature_names)
X = iris.data
y = np.array([iris.target_names[i] for i in iris.target])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# Model Training
clf = svm.SVC(gamma='scale', kernel='rbf', probability=True)
clf.fit(X, y)
prediction_probabilities = list(clf.predict_proba(X))
prediction = list(clf.predict(X))
Logging the Model and the schema
Let us log the model, along with model schema and define the desired custom metrics.
client = mlf.get_client()
run = client.create_run(project_name="iris-demo")
model_version = run.log_model(
name="iris-sklearn",
model=clf,
framework="sklearn",
description="sklearn model, rbf kernel",
model_schema={
"features": [
{"name": "sepal length (cm)", "type": "float"},
{"name": "sepal width (cm)", "type": "float"},
{"name": "petal length (cm)", "type": "float"},
{"name": "petal width (cm)", "type": "float"},
],
"prediction": "categorical",
},
custom_metrics=[{"name": "log_loss", "type": "metric", "value_type": "float"}],
)
Logging the predictions and actuals
from datetime import datetime
for i in range(150):
features = {iris.feature_names[j]: float(X[i][j]) for j in range(4)}
prediction_data = {
"value": prediction[i],
"probabilities": {
iris.target_names[j]: float(prediction_probabilities[i][j])
for j in range(3)
},
}
id = client.generate_hash_from_data(features=features, timestamp=datetime.utcnow())
client.log_predictions(
model_version_fqn=model_version.fqn,
predictions=[
mlf.Prediction(
data_id=id,
features=features,
prediction_data=prediction_data,
raw_data={},
)
],
)
client.log_actuals(
model_version_fqn=model_version.fqn,
actuals=[mlf.Actual(data_id=id, value=y[i])],
)
And with these log-lines, truefoundry generates monitoring dashboards for your models. You can open the monitoring dashboard with by clicking on this link.

Monitoring Dashboard
Updated 4 months ago