Below is a workflow for how to deploy a model with Tensorflow.js using a client-side script. I used the apples-or-tomatoes-image-classification image dataset on Kaggle.
import tensorflow as tf
import tensorflowjs as tfjsimport os
import matplotlib.pyplot as plt
import numpy as np
Subfunctions
def load_image_tf(image_path, img_w, img_h):
img = tf.io.read_file(image_path)
img = tf.image.decode_jpeg(img, channels=3)
img = tf.image.resize(img, (img_w, img_h))# Scale the image value from -1 to 1
img = tf.keras.applications.inception_v3.preprocess_input(img)
return img
def load_different_images(X, Y, label, PATH, img_w, img_h):
path_list = [os.path.join(PATH, i) for i in os.listdir(PATH)]for i in path_list:
X.append(load_image_tf(i, img_w, img_h))
Y.append(label)
return X, Y
def filter_an_image(rgb_image, x_dist=20):y_dist = x_dist
w, h, rbg = rgb_image.shape
LT = np.mean(rgb_image[0:x_dist, 0:y_dist,:])
RT = np.mean(rgb_image[w-x_dist:w, 0:y_dist,:])
LB = np.mean(rgb_image[0:x_dist, h-y_dist:h,:])
RB = np.mean(rgb_image[w-x_dist:w, h-y_dist:h,:])
threshhold = 0.95
if (LT > threshhold) and (RT > threshhold) and (LB > threshhold) and (RB > threshhold):
# _light_background_onecolor
out = 'object_only'
else:
out = 'object_w_background_mixedcolors'
return out
def view_images(rgb_image_list, rgb_image_list_label, x_dist):
# View a sample image
i = np.random.permutation(np.arange(len(rgb_image_list)))[0]fig, ax0 = plt.subplots(nrows=1, ncols=1, figsize=(4,4))
ax0.imshow(rgb_image_list[i])
ax0.axis('off')
ax0.set_title(f"i={i}, {rgb_image_list_label[i]}", size=20)
plt.show()
# Image values are from -1 to 1
print('max value: ', np.max(rgb_image_list[i]))
print('min value: ', np.min(rgb_image_list[i]))
y_dist = x_dist
w, h, rbg = rgb_image_list[i].shape
print('LT:', np.mean(rgb_image_list[i][0:x_dist, 0:y_dist,:]))
print('RT:', np.mean(rgb_image_list[i][w-x_dist:w, 0:y_dist,:]))
print('LB:', np.mean(rgb_image_list[i][0:x_dist, h-y_dist:h,:]))
print('RB:', np.mean(rgb_image_list[i][w-x_dist:w, h-y_dist:h,:]))
def plotting_training_results(history):acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
plt.figure(figsize=(8, 8))
plt.subplot(2, 1, 1)
plt.plot(acc, label='Training Accuracy')
plt.plot(val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.ylabel('Accuracy')
plt.ylim([min(plt.ylim()),1])
plt.title(f'Training and Validation Accuracy: train={np.max(acc)}, val={np.max(val_acc)}')
plt.subplot(2, 1, 2)
plt.plot(loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.ylabel('Loss/Cross Entropy')
plt.ylim([0,1.0])
plt.title(f'Training and Validation Loss: train={np.min(loss)}, val={np.min(val_loss)}')
plt.xlabel('epoch')
plt.show()
def sequential_padder(X_train_filtered, Y_train_filtered):# Train class count
from collections import Counter
c = Counter(Y_train_filtered)
# 1 = apple, 0 = tomatoe
class_key = list(c.keys())
print('class_key: ', class_key)
class_value = list(c.values())
print('class_value: ', class_value)
max_class = np.argmax(c)
print('max_class: ', max_class)
images_to_add_per_class = [class_value[max_class] - i for i in class_value]
print('images_to_add_per_class: ', images_to_add_per_class)
# --------------------------------------------
# Need to account for when a class does not need padding: assign directly
X_train_filtered_pad = X_train_filtered
Y_train_filtered_pad = Y_train_filtered
for ind, samples2pad in enumerate(images_to_add_per_class):
print('Number of values to pad:', samples2pad)
# Identify class number
class_num = class_key[ind] # class_key: [1, 0]
print('Class number that needs padding:', class_num)
# Need to find the index of Y_train_filtered/X_train_filtered that belong to class_num
class_num_index = [index for index, val in enumerate(Y_train_filtered) if val == class_num]
print('class_num_index:', class_num_index)
# Select X and Y for the specified class number index
X_class_num_select = [X_train_filtered[i] for i in class_num_index]
Y_class_num_select = [Y_train_filtered[i] for i in class_num_index]
# Find the number of samples for the specified class number
curSamples = len(X_class_num_select)
print('Number of samples to repeat for this specific class:', curSamples)
# If the number of class samples are greater than the amount to pad, use class samples in sequential order
if curSamples > samples2pad:
# Loop over every element in a because it is 3d matrix, and add each 3d piece of information one loop at a time
for i in range(samples2pad):
X_train_filtered_pad.append(X_class_num_select[i])
Y_train_filtered_pad.append(Y_class_num_select[i])
else:
# If the number of class samples are less than the amount to pad, repeat the full class sample an even amount of times
num_of_full_loops = int(samples2pad/curSamples)
print('num_of_full_loops:', num_of_full_loops)
for i in range(num_of_full_loops):
for j in range(curSamples):
X_train_filtered_pad.append(X_class_num_select[j])
Y_train_filtered_pad.append(Y_class_num_select[j])
remaining_vals = samples2pad - num_of_full_loops*curSamples
print('remaining_vals:', remaining_vals)
for i in range(remaining_vals):
X_train_filtered_pad.append(X_class_num_select[i])
Y_train_filtered_pad.append(Y_class_num_select[i])
# --------------------------------------------
print('Length of Y matrix after padding:', len(Y_train_filtered_pad))
# --------------------------------------------
# Confirm that classes are even after padding
c = Counter(Y_train_filtered_pad)
# 1 = apple, 0 = tomatoe
class_key = list(c.keys())
print('class_key: ', class_key)
class_value = list(c.values())
print('class_value: ', class_value)
max_class = np.argmax(c)
print('max_class: ', max_class)
images_to_add_per_class = [class_value[max_class] - i for i in class_value]
print('images_to_add_per_class: ', images_to_add_per_class)
return X_train_filtered_pad, Y_train_filtered_pad
Load data
# Set desired dimensions of images
w, h, rgb = 512, 512, 3# Using a dataset from Kaggle
img_w = w
img_h = h
X_train = []
Y_train = []
label = 1 #'apple'
PATH_apples = "/kaggle/input/apples-or-tomatoes-image-classification/train/apples"
X_train, Y_train = load_different_images(X_train, Y_train, label, PATH_apples, img_w, img_h)
label = 0 # 'tomatoe'
PATH_tomatoes = "/kaggle/input/apples-or-tomatoes-image-classification/train/tomatoes"
X_train, Y_train = load_different_images(X_train, Y_train, label, PATH_tomatoes, img_w, img_h)
X_test = []
Y_test = []
label = 1 #'apple'
PATH_apples = "/kaggle/input/apples-or-tomatoes-image-classification/test/apples"
X_test, Y_test = load_different_images(X_test, Y_test, label, PATH_apples, img_w, img_h)
label = 0 # 'tomatoe'
PATH_tomatoes = "/kaggle/input/apples-or-tomatoes-image-classification/test/tomatoes"
X_test, Y_test = load_different_images(X_test, Y_test, label, PATH_tomatoes, img_w, img_h)
# Define the corner bounding box length to decide if an object is only displayed
x_dist = 20
Filter data: only keep images with same colored background in all four corners
X_train_filtered = []
Y_train_filtered = []# Remove images that have mixed color edge backgrounds
for ind, img in enumerate(X_train):
if filter_an_image(img, x_dist=x_dist) == 'object_only':
X_train_filtered.append(X_train[ind])
Y_train_filtered.append(Y_train[ind])
X_test_filtered = []
Y_test_filtered = []
for ind, img in enumerate(X_test):
if filter_an_image(img, x_dist=x_dist) == 'object_only':
X_test_filtered.append(X_test[ind])
Y_test_filtered.append(Y_test[ind])
print('Train: Original image count: ', len(X_train))
print('Train: Filtered image count: ', len(X_train_filtered))
print('Test: Original image count: ', len(X_test))
print('Test: Filtered image count: ', len(X_test_filtered))
# View a sample image
view_images(X_train_filtered, Y_train_filtered, x_dist)
Pad images
print('length of X_train_filtered BEFORE padding: ', len(X_train_filtered))
print('length of Y_train_filtered BEFORE padding: ', len(Y_train_filtered))X_train_filtered_pad, Y_train_filtered_pad = sequential_padder(X_train_filtered, Y_train_filtered)
print('length of X_train_filtered AFTER padding: ', len(X_train_filtered))
print('length of Y_train_filtered AFTER padding: ', len(Y_train_filtered))
print('length of X_test_filtered BEFORE padding: ', len(X_test_filtered))
print('length of Y_test_filtered BEFORE padding: ', len(Y_test_filtered))X_test_filtered_pad, Y_test_filtered_pad = sequential_padder(X_test_filtered, Y_test_filtered)
print('length of X_test_filtered AFTER padding: ', len(X_test_filtered))
print('length of Y_test_filtered AFTER padding: ', len(Y_test_filtered))
Create a Dataset
BATCH_SIZE = 1train_ds = tf.data.Dataset.from_tensor_slices((X_train_filtered_pad, Y_train_filtered_pad))
train_ds = train_ds.shuffle(len(X_train_filtered_pad))
train_ds = train_ds.batch(BATCH_SIZE, drop_remainder=True)
test_ds = tf.data.Dataset.from_tensor_slices((X_test_filtered, Y_test_filtered))
test_ds = test_ds.shuffle(len(X_test_filtered))
test_ds = test_ds.batch(BATCH_SIZE, drop_remainder=True)
Add an edge detection pre-processing transformation before Model training
An edge detection pre-processing transformation was used to make the image capture the outline of the objects in the image, it uses a vertical edge detection kernel. Initial transfer learning and fine-tuning with the Mobilenet (also known as Imagenet) base model did not give accurate results greater than chance performance (accuracy=0.5) for classifying an apple from a tomatoe. Therefore, edge detection was key to bring out key features in the images, like the stem and seeds.
Notice that the function is applied to the dataset because when deploying the model, the edge detection process has to be performed outside of the model because the functions tf.expand_dims, tf.nn.conv2d, and tf.reshape are part of the TFOpLambda class when the model.json file is created. TFOpLambda class items are not recognizable in Tensorflow.js, thus all non-keras.layer functions need to be performed in javascript directly using the library in reference [1].
# Treat the data outside the model with this recreation of edge_detection for Both Python and JavaScript/Tensorflow.js
def edge_detection2(image_3D):# Vertical edge detection filter
eg = 1000
k_rgb_values = tf.constant([[[[ 1000.], [ 1000.], [ 1000.]],
[[ 0.], [ 0.], [ 0.]],
[[-1000.], [-1000.], [-1000.]]],
[[[ 1000.], [ 1000.], [ 1000.]],
[[ 0.], [ 0.], [ 0.]],
[[-1000.], [-1000.], [-1000.]]],
[[[ 1000.], [ 1000.], [ 1000.]],
[[ 0.], [ 0.], [ 0.]],
[[-1000.], [-1000.], [-1000.]]]], dtype=tf.float32)
padding='SAME'
strides = 1
image_4D = tf.expand_dims(image_3D, axis=0)
image_filter = tf.nn.conv2d(image_4D, k_rgb_values, strides=strides, padding=padding)
# print('image_filter.shape: ', image_filter.shape)
return image_filter
# Check that function is correct
i = np.random.permutation(np.arange(len(X_train_filtered_pad)))[0]
image_3D = X_train_filtered_pad[i]
image_filter = edge_detection2(image_3D)image_filter = tf.reshape(image_filter, [w, h])
print('image_filter.shape: ', image_filter.shape)
# View a sample image
fig, ax0 = plt.subplots(nrows=1, ncols=1, figsize=(4,4))
ax0.imshow(image_filter)
ax0.axis('off')
ax0.set_title(f"i={i}, {Y_train_filtered_pad[i]}", size=20)
plt.show()
# Apply tranformation to dataset
train_ds_edfilt = train_ds.map(lambda x, y: (edge_detection2(x), y))
test_ds_edfilt = test_ds.map(lambda x, y: (edge_detection2(x), y))print('Train shape after edge detection transformation: ', list(train_ds_edfilt.take(1).as_numpy_iterator())[0][0].shape)
print('Test shape after edge detection transformation: ', list(test_ds_edfilt.take(1).as_numpy_iterator())[0][0].shape)
# Also, outside of the model, reshape the image back to a 4D tensor
train_ds_edfilt = train_ds_edfilt.map(lambda x, y: (tf.reshape(x, [1, w, h, 1]), y))
test_ds_edfilt = test_ds_edfilt.map(lambda x, y: (tf.reshape(x, [1, w, h, 1]), y))
print('Train shape needed to insert into model (4D tensor): ', list(train_ds_edfilt.take(1).as_numpy_iterator())[0][0].shape)
print('Test shape needed to insert into model (4D tensor):', list(test_ds_edfilt.take(1).as_numpy_iterator())[0][0].shape)
Define callbacks
patience = 60
early_stopping = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=patience, mode='min')
Way 0: Use mobilenet model directly
# Resize the orginally loaded 512x512x3 images to 224x224x3 because mobilenet requires this size image
train_ds_224by224by3 = train_ds.map(lambda x, y: (tf.image.resize(x, [224, 224]), y))
test_ds_224by224by3 = test_ds.map(lambda x, y: (tf.image.resize(x, [224, 224]), y))
pretrained_model = tf.keras.applications.MobileNetV2()
result = pretrained_model.predict(train_ds_224by224by3) # Probabilities of each class
predictions = tf.keras.applications.mobilenet_v2.decode_predictions(result)
predictions
As one can see, there is no “tomatoe” class and only a “Granny_Smith” class for apple.
d = {}
c = 0
for i in range(len(predictions)):class_label = Y_train_filtered_pad[i]
d[f'{i}_{class_label}'] = predictions[i][0][1]
# 1 = apple, 0 = tomatoe
if class_label == 1 and predictions[i][0][1] == 'Granny_Smith':
c = c + 1
elif class_label == 0 and predictions[i][0][1] != 'Granny_Smith':
c = c + 1
print('accuracy: ', c/len(predictions))
d
After calculating the accuracy for the dataset, we can see that Mobilenet performs at chance level accuracy at classifying an apple and “not an apple”. A chance level predictive model is not satisfactory for deployment, so let’s try to fine-tune the Mobilenet model with our specific image dataset to see if there are improvements.
Way 1: Transfer Learning — Do not change the weights of the pre-trained model, add new class output, train the model with new data
# Step 0: load the base model
input_layer = tf.keras.layers.Input(shape=(224, 224, 3))base_model = tf.keras.applications.mobilenet_v2.MobileNetV2(input_shape=(224, 224, 3),
alpha=1.0,
weights='imagenet',
input_tensor=input_layer,
include_top=False,
pooling=None,
classes=1000)
# Step 1: Decide if one wants to FINE TUNE (change the weights of the pre-trained model) OR
# perform TRANSFER LEARNING (do not change the weights)
# base_model.trainable = False, means to NOT change the weights of the base_model
# Not changing the weights of the base_model means that one is performing transfer learning,
# This implies that the data used to train the model is similar to the new data that one wants
# to predict, thus the weights of the old data can be used.
base_model.trainable = False
# Step 2: Make a model using the base_model# Functional API format
# ----------------------------
# Initialize the model
x = base_model(input_layer)
# This outputs (None, 7, 7, 1280)
# ----------------------------
x = tf.keras.layers.Conv2D(3, (3,3), strides=(1, 1), padding='same')(x)
x = edge_detection2(x)
# This outputs (1, None, 7, 7, 1)
x = tf.reshape(x, [-1, 7, 7, 1])
# This outputs (None, 7, 7, 1)
# ----------------------------
# Way 4: train=0.97,val=0.78, train_loss=14, val_loss=82
x = tf.keras.layers.Conv2D(8, (3,3), activation='relu', input_shape=(7, 7, 1))(x)
x = tf.keras.layers.MaxPooling2D(2, 2)(x)
x = tf.keras.layers.GlobalAveragePooling2D()(x)
x = tf.keras.layers.Dropout(0.2)(x)
# ----------------------------
# Add a layer to output the number of new classes
outputs = tf.keras.layers.Dense(1, activation='sigmoid', name='outputs')(x)
model_tl = tf.keras.Model(input_layer, outputs)
model_tl.summary()
I found that using only a GlobalAveragePooling2D and Dropout layer was not sufficient to train the plain images of the original dataset, because the overall shape of an apple is similar to a tomatoe. Convolutional layers, like the edge detection function was key to identify differences in the stem and form (ie: seeds) of apples and tomatoes.
# Step 3: Compile the model
base_learning_rate = 0.0001model_tl.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),
metrics=[tf.keras.metrics.BinaryAccuracy(threshold=0, name='accuracy')])
EPOCHS = 20
# Step 4: Train the model
history = model_tl.fit(train_ds_224by224by3,
validation_data=test_ds_224by224by3,
epochs=EPOCHS)
plotting_training_results(history)
Transfer learning shows overfitting, so next I remove some of the layers to see if prediction results are better using only initial model layers from the base model.
Way 2: Fine-tuning — Change the weights of the pre-trained model, add new class output, train the model with new data
# Step 0: load the base model
input_layer = tf.keras.layers.Input(shape=(224, 224, 3))base_model = tf.keras.applications.mobilenet_v2.MobileNetV2(input_shape=(224, 224, 3),
alpha=1.0,
weights='imagenet',
input_tensor=input_layer,
include_top=False,
pooling=None,
classes=1000)
# Step 1: Decide if one wants to FINE TUNE (change the weights of the pre-trained model) OR
# perform TRANSFER LEARNING (do not change the weights)
base_model.trainable = True
# First determine how many layers are in the pre-trained model
print('Total number of base_model layers:', len(base_model.layers))# Decide from which layer to change the weights
fine_tune_at = int( len(base_model.layers) - len(base_model.layers)/4 )
print('fine_tune_at 3/4ths of the layers: ', fine_tune_at)
# Logic is that the last layers have case specific information about the data.
# So just take the first layers that have to do with general image classification.
# Freeze all the layers before the `fine_tune_at` layer
for layer in base_model.layers[:fine_tune_at]:
layer.trainable = False
Total number of base_model layers: 154
fine_tune_at 3/4ths of the layers: 115
Repeat Step 2 above, under Transfer Learning, to declare the model variables. Next, declare the model and train the model again only using the first 3/4ths of the model layers.
model_ft = tf.keras.Model(input_layer, outputs)
model_ft.summary()
# Step 3: Compile the model
base_learning_rate = 0.0001model_ft.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),
metrics=[tf.keras.metrics.BinaryAccuracy(threshold=0, name='accuracy')])
EPOCHS = 20
# Step 4: Train the model
history = model_ft.fit(train_ds_224by224by3,
validation_data=test_ds_224by224by3,
epochs=EPOCHS)
plotting_training_results(history)
The results are a lot better after removing the last 39 layers, meaning that the loss descends over time as the accuracy increases similarly over time. As mentioned before, it makes sense that the results may be better if the last layers are removed because the last layers may have case specific information (detailed features) about the old data that could prevent training with new data.
Way 3: MaxPooling Convolutional model
Despite that fact that the fine-tuning gave better results, it was of interest to use a model that could be deployed easily using Tensorflow.js. As mentioned before, non tf.keras.layers components cannot be serialized correctly to create a deployable model. The model architecture below can be serialized using Tensorflow.js without any errors, however the transfer learning and fine-tuning models in Way 1 and 2 would cause serialization errors due to the edge_detection2 functions belonging to the TFOpLambda class.
def MPCNN_arch_edge_detection(w, h):# Functional API format
input_layer = tf.keras.layers.Input(shape=(w, h, 1), dtype='float32')
initializer = tf.keras.initializers.HeUniform()
x = tf.keras.layers.Conv2D(8, (3,3), activation='relu', kernel_initializer=initializer, input_shape=(w, h, 1))(input_layer)
x = tf.keras.layers.MaxPooling2D(2, 2)(x)
x = tf.keras.layers.Conv2D(16, (3,3), activation='relu')(x)
x = tf.keras.layers.MaxPooling2D(2, 2)(x)
x = tf.keras.layers.Conv2D(32, (3,3), activation='relu')(x)
x = tf.keras.layers.MaxPooling2D(2,2)(x)
x = tf.keras.layers.Conv2D(64, (3,3), activation='relu')(x)
x = tf.keras.layers.MaxPooling2D(2,2)(x)
x = tf.keras.layers.Dropout(0.2)(x)
x = tf.keras.layers.Conv2D(128, (3,3), activation='relu')(x)
x = tf.keras.layers.MaxPooling2D(2,2)(x)
x = tf.keras.layers.Dropout(0.2)(x)
x = tf.keras.layers.Conv2D(256, (3,3), activation='relu')(x)
x = tf.keras.layers.MaxPooling2D(2,2)(x)
x = tf.keras.layers.Dropout(0.2)(x) # Add a Dropout layer to prevent overfitting
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.Dense(512, activation='relu')(x)
outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)
model = tf.keras.Model(inputs=input_layer, outputs=outputs)
return model
model = MPCNN_arch_edge_detection(w, h)# Step 3: Compile the model
base_learning_rate = 0.0001
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),
metrics=[tf.keras.metrics.BinaryAccuracy(threshold=0, name='accuracy')])
EPOCHS = 50
# Step 4: Train the model
# Notice that dataset pre-processing was performed
# outside of the model to mimick how images should be treated in the deployed
# webapp.
history = model.fit(train_ds_edfilt,
validation_data=test_ds_edfilt,
epochs=EPOCHS)
plotting_training_results(history)
The results show that there is some over-fitting of the training data, so this model could be improved.
# Check that function is correct
i = np.random.permutation(np.arange(len(X_test_filtered)))[0]
image_3D = X_test_filtered[i]
image_filter = edge_detection2(image_3D)
image_4D = tf.reshape(image_filter, [1, w, h, 1])# ----------------------------------------------
# Plot the image
plot_image = tf.reshape(image_filter, [w, h])
# View a sample image
fig, (ax0, ax1) = plt.subplots(nrows=1, ncols=2, figsize=(4,4))
ax0.imshow(image_3D)
ax0.axis('off')
ax1.imshow(plot_image)
ax1.axis('off')
true_out = "tomatoe" if Y_test_filtered[i] == 0 else "apple"
ax1.set_title(f"i={i}, true={true_out}", size=12)
plt.show()
# ----------------------------------------------
edge_detection_model_predictions = model.predict(image_4D)
print('edge_detection_model_predictions: ', edge_detection_model_predictions)
index = np.floor(edge_detection_model_predictions)
if index == 0:
pred_mpcnn = 'tomatoe'
else:
pred_mpcnn = 'apple'
# ----------------------------------------------
# Non-trained mobilenet
pretrained_model = tf.keras.applications.MobileNetV2()
image_4D = tf.expand_dims(image_3D, axis=0)
test_224by224by3 = tf.image.resize(image_4D, [224, 224])
result = pretrained_model.predict(test_224by224by3) # Probabilities of each class
predictions = tf.keras.applications.mobilenet_v2.decode_predictions(result)
pred_mobilenet = predictions[0][0]
# ----------------------------------------------
# Transfer learning mobilenet
edge_detection_model_tl = model_tl.predict(test_224by224by3)
print('edge_detection_model_tl: ', edge_detection_model_tl)
index = np.floor(edge_detection_model_tl)
if index == 0:
pred_tl = 'tomatoe'
else:
pred_tl = 'apple'
# ----------------------------------------------
# Fine-tuning mobilenet
edge_detection_model_ft = model_ft.predict(test_224by224by3)
print('edge_detection_model_ft: ', edge_detection_model_ft)
index = np.floor(edge_detection_model_ft)
if index == 0:
pred_ft = 'tomatoe'
else:
pred_ft = 'apple'
# ----------------------------------------------
print('pred_mpcnn:', pred_mpcnn)
print('pred_mobilenet:', pred_mobilenet)
print('pred_tl:', pred_tl)
print('pred_ft:', pred_ft)
The web service that I tested with was GitHub Pages. It is a free web service that creates webpages from repository files. Simply open a GitHub account, and create a repository. Next activate the repository to be a GitHub Page by clicking on the “Settings” tab, scroll down to the “GitHub Pages” section. Under the “Source” dropdown, select “master branch” or the branch you want to use for deployment. Click on the “Save” button. Start making your HTML and javascript webapps. The url follows the pattern https://<your-github-username>.github.io/<repository-name>. It may take a few minutes for your changes to be reflected on the deployed website.
To deploy a simple client-side image classification model, I made an index.html page and called the Way 3: MaxPooling Convolutional model that was created in Python, using the model.json and .bin files.
Next, I serialized the Way 3: MaxPooling Convolutional model such that it can be deployed in the browser. By running the command below, a folder called tfjs_model is created that contains the serialized model containing several files (model.json and 5 .bin files). Move the model.json and .bin files in the repository, as shown above.
# Convert the saved model to TensorFlow.js format
tfjs.converters.save_keras_model(model, 'tfjs_model')
Below is a client-side HTML script that loads an image from the GitHub repo and gives it to two Tensorflow models.
<!DOCTYPE html>
<html>
<head></head>
<body><h1 style='text-align: center; margin-bottom: -35px;'>Image labeling webapp</h1>
<br><br>
<!-- Drop down menu: Select image to classify -->
<label for="select_image">Select an image to classsify/label:</label>
<select id="select_this_image">
<option value="tomatoe">tomatoe</option>
<option value="apple">apple</option>
</select>
<!-- Select model to classify -->
<h2 id="select_model" style='text-align: left; display:none'>Select different models to classify the image</h2>
<div style="width:100%;height:100%;position:absolute;vertical-align:middle;text-align:left;">
<button id="run_mobilenet_button" onclick="run_mobilenet()" style="display:none">mobilenet</button> <button id="run_edgedetection_classifer_button" onclick="run_edgedetection_classifer()" style="display:none">edgedetection_classifer</button>
<div>
<!-- Once image and model are selected, make image appear in canvas and run model -->
<div style="width:100%;height:100%;position:absolute;vertical-align:middle;text-align:left;">
<canvas id="canvasId" width="224" height="224" style="display:none"></canvas>
<canvas id="canvasId2" width="224" height="224" style="display:none"></canvas>
<div>
<div id="output" style="font-family:courier;font-size:24px;height:300px"></div>
<style>
canvas {border: 1px solid black;}
</style>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@latest"> </script>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/mobilenet@1.0.0"> </script>
<script>
var canvasElement = document.getElementById("canvasId");
const ctx = canvasElement.getContext("2d");
var canvasElement2 = document.getElementById("canvasId2");
const ctx2 = canvasElement2.getContext("2d");
const outp = document.getElementById('output');
// -------------------------------------------------
// Way 0: eventlistener
const selectElement = document.getElementById('select_this_image');
selectElement.addEventListener("click", () => {
document.getElementById("select_model").style.display = "block";
document.getElementById("run_mobilenet_button").style.display = "block";
document.getElementById("run_edgedetection_classifer_button").style.display = "block";
var selected_image = document.getElementById("select_this_image").value;
});
// -------------------------------------------------
async function load_image_basedOn_selection() {
var selected_image = document.getElementById("select_this_image").value;
if (selected_image == 'apple') {
load_image = "apple3D.png";
} else if (selected_image == 'tomatoe') {
load_image = "tomatoe3D.png";
} else {
load_image = "tomatoe3D.png";
}
return load_image
}
// -------------------------------------------------
async function run_mobilenet() {
try {
const image = new Image();
image.src = await load_image_basedOn_selection();
image.onload = function (){
// Draw image on canvas
document.getElementById("canvasId").style.display = "block";
ctx.drawImage(image, 0,0);
// Convert the image element to a tensor using fromPixels
var tensor_image = tf.browser.fromPixels(image); // This is size 224,224,3
const eTensor = tensor_image.expandDims(0); // This is size 1,224,224,3
// Give image to model
mobilenet.load().then(model => {model.classify(eTensor).then(predictions => {for(var i = 0; i<predictions.length; i++){outp.innerHTML += "<br/>" + predictions[i].className + " : " + predictions[i].probability;} }); });
};
} catch (error) {
outp.innerHTML = error;
}
} // end of run_mobilenet
// -------------------------------------------------
async function run_edgedetection_classifer(){
const MODEL_URL = 'model.json';
const model = await tf.loadLayersModel(MODEL_URL);
const image = new Image();
image.src = await load_image_basedOn_selection();
image.onload = async function (){
// Draw image on canvas
document.getElementById("canvasId").style.display = "block";
ctx.drawImage(image, 0,0);
// Convert the image element to a tensor using fromPixels
var tensor_image = tf.browser.fromPixels(image); // This is size 224,224,3
const k_rgb_values = tf.tensor4d([[[[ 1000.], [ 1000.], [ 1000.]],[[ 0.], [ 0.], [ 0.]],[[-1000.], [-1000.], [-1000.]]], [[[ 1000.], [ 1000.], [ 1000.]], [[ 0.], [ 0.], [ 0.]], [[-1000.], [-1000.], [-1000.]]], [[[ 1000.], [ 1000.], [ 1000.]], [[ 0.], [ 0.], [ 0.]], [[-1000.], [-1000.], [-1000.]]]], shape=[3, 3, 3, 1], dtype='float32');
const image_4D = tensor_image.expandDims(0); // This is size 1,224,224,3
const image_4D_float = tf.cast(image_4D, 'float32')
const image_filter0 = tf.conv2d(image_4D_float, k_rgb_values, 1, 'same')
// Dispose of the intermediate tensors
tensor_image.dispose();
k_rgb_values.dispose();
image_4D.dispose();
image_4D_float.dispose();
// Make image values from -1 to 1
const b = tf.scalar(255);
const image_filter = image_filter0.div(b)
const shape_out = image_filter.shape;
// Dispose of the intermediate tensors
image_filter0.dispose();
b.dispose();
// -----------------------------------------------
// Ensure that tensor is 4d
const x = tf.reshape(image_filter, [1, shape_out[1], shape_out[2], shape_out[3]])
const boxes = tf.tensor2d([[0.25, 0.25, 0.75, 0.75]], [1, 4]);
const boxIndices = tf.tensor1d([0], 'int32');
const newSize = [512, 512];
const resizedTensor = tf.image.cropAndResize(x, boxes, boxIndices, newSize);
// Give image to model
const result = model.predict(resizedTensor);
// if (result == ) {
outp.innerHTML = result;
}; // end of image.onload
} // end of run_edgedetection_classifer
</script>
</body>
</html>
The index.html creates the webapp below such that one can select an image from a drop-down menu and then select one of two models: Way 0 — mobilenet model directly, Way 3 — MaxPooling Convolutional model.
Of course a more complex webapp can be made, however it was fun to start off making a simple and functional webapp. In the browser, the tomatoe image returns a value of 0 and apple images return a value greater than 0, about 0.1. Oddly, tomatoes and apples give values close to 0 and 1, respectively, in python, but in the browser the same images predict all close to zero. The next test is to try to train a model in the browser and then predict images using the trained model, without transferring model.json and .bin files from Python.
Happy practicing! 👋
🎁 Donate: support the blog! | 💻 GitHub | 🔔 Subscribe
- Tensorflow.js library: https://js.tensorflow.org/api/1.0.0/
Be the first to comment