Deep learning engineer

Sentiment triage of customer reviews

Train an in-memory naive Bayes classifier on six labelled reviews (Laplace smoothing k = 1) and classify four new reviews, with the probability of the best class, so that negative ones go to the support team.

#1. A naive Bayes text model with two classes;
if (== (dl bayesian exist "rx_reviews") 1) {dl bayesian delete "rx_reviews";};
dl bayesian create "rx_reviews" "[\"positive\", \"negative\"]";
#2. Labelled reviews of the last campaign;
dl bayesian add_sentence "rx_reviews" "positive" "great product, fast delivery";
dl bayesian add_sentence "rx_reviews" "positive" "excellent quality, I love it";
dl bayesian add_sentence "rx_reviews" "positive" "works perfectly, great value for money";
dl bayesian add_sentence "rx_reviews" "negative" "broken on arrival, very disappointed";
dl bayesian add_sentence "rx_reviews" "negative" "poor quality, it stopped working";
dl bayesian add_sentence "rx_reviews" "negative" "late delivery and bad support";
#3. Build the bags of words with Laplace smoothing k = 1;
dl bayesian init "rx_reviews" 1;
#4. Triage of new reviews: negatives go to the support team;
json load "rx_new" "[\"great quality, works well\", \"bad quality, stopped after a week\", \"fast delivery but poor support\", \"I love it, excellent value\"]";
-> "[out]" "";
for (-> "[i]" 0) (< [i] (json count "rx_new" /)) (++ "[i]") {
  json load "rx_p" (dl bayesian predict "rx_reviews" (json select "rx_new" (concat "/[" [i] "]")));
  -> "[out]" (concat [out] (json select "rx_p" "/input") " => " (json select "rx_p" "/prediction") " " (math round (* 100 (json select "rx_p" "/best_double")) 1) "% | ");
};
json load "rx_m" (dl bayesian show);
-> "[models]" (json count "rx_m" /);
dl bayesian delete "rx_reviews";
json unload "rx_new"; json unload "rx_p"; json unload "rx_m";
concat [out] "Bayesian models in memory during the run: " [models]

great quality, works well => positive 85.7% | bad quality, stopped after a week => negative 80% | fast delivery but poor support => negative 66.7% | I love it, excellent value => positive 94.1% | Bayesian models in memory during the run: 1

Categorise bank statement lines

Write a labelled training file of bank statement lines, train a document classifier (OpenNLP, 10 iterations) into a model file, load it and categorise new lines as salary, cash, grocery or fees; the files are deleted at the end.

#1. Training file: one labelled bank statement line per row;
file create "tmp/rx_bank_train.txt" "salary TRANSFER RECEIVED ACME CORP SALARY JUNE
salary TRANSFER RECEIVED ACME CORP SALARY JULY
salary TRANSFER RECEIVED ACME CORP SALARY AUGUST
cash CARD 2052 ATM WITHDRAWAL PARIS ROCH
cash CARD 2052 ATM WITHDRAWAL LYON GARE
cash CARD 2052 ATM WITHDRAWAL NANTES CENTRE
grocery CARD 2052 PAYMENT SUPERMARKET CARREFOUR
grocery CARD 2052 PAYMENT SUPERMARKET LIDL
grocery CARD 2052 PAYMENT SUPERMARKET MONOPRIX
fees MONTHLY ACCOUNT FEE PREMIUM OPTION
fees FOREIGN PAYMENT FEE OUTSIDE EURO ZONE
fees CARD ANNUAL FEE PREMIUM";
#2. Train the document classifier, 10 iterations, then load it;
dl n_bayesian create_model "en" "tmp/rx_bank_train.txt" 10 "tmp/rx_bank_model.bin";
if (== (dl n_bayesian exist "rx_bank") 0) {dl n_bayesian load "rx_bank" "tmp/rx_bank_model.bin";};
#3. Categorise the new lines of the statement;
json load "rx_lines" "[\"CARD 2052 PAYMENT SUPERMARKET AUCHAN\", \"ATM WITHDRAWAL BORDEAUX\", \"TRANSFER RECEIVED ACME CORP SALARY SEPTEMBER\", \"ACCOUNT FEE OCTOBER\"]";
-> "[out]" "";
for (-> "[i]" 0) (< [i] (json count "rx_lines" /)) (++ "[i]") {
  json load "rx_p" (dl n_bayesian predict "rx_bank" (json select "rx_lines" (concat "/[" [i] "]")));
  -> "[out]" (concat [out] (json select "rx_p" "/input") " => " (json select "rx_p" "/prediction") " " (math round (* 100 (json select "rx_p" "/best_double")) 1) "% | ");
};
json load "rx_m" (dl n_bayesian show);
-> "[n]" (json count "rx_m" /);
#4. Free the model and delete the files;
dl n_bayesian delete "rx_bank";
file delete "tmp/rx_bank_train.txt";
file delete "tmp/rx_bank_model.bin";
json unload "rx_lines"; json unload "rx_p"; json unload "rx_m";
concat [out] "classifiers loaded: " [n]

CARD 2052 PAYMENT SUPERMARKET AUCHAN => grocery 99.8% | ATM WITHDRAWAL BORDEAUX => cash 99.9% | TRANSFER RECEIVED ACME CORP SALARY SEPTEMBER => salary 100% | ACCOUNT FEE OCTOBER => fees 99.8% | classifiers loaded: 1

Predictive maintenance with a neural network

Train a feed-forward neural network on 24 sensor records (temperature, vibration, motor current) labelled normal, bearing wear or overheating, with 5-fold cross-validation and 25 % held back, then diagnose three motors.

#1. Sensor history: temperature in C, vibration in mm/s, motor current in A, state;
file create "tmp/rx_motor.csv" "61.5,1.28,7.9,normal
65.4,4.13,8.5,bearing_wear
91.3,1.68,12.0,overheating
62.5,1.26,8.1,normal
62.7,4.84,8.8,bearing_wear
89.5,1.26,10.6,overheating
60.2,1.13,8.1,normal
65.9,4.71,8.3,bearing_wear
88.9,1.68,11.2,overheating
65.4,1.28,8.5,normal
64.8,4.2,8.5,bearing_wear
87.7,1.73,11.6,overheating
61.1,1.06,7.8,normal
68.4,4.18,8.7,bearing_wear
89.3,1.3,11.5,overheating
64.6,0.9,7.9,normal
65.8,4.17,8.8,bearing_wear
87.8,1.31,11.9,overheating
63.3,1.34,8.6,normal
66.7,4.55,8.1,bearing_wear
89.8,1.48,11.3,overheating
59.5,1.05,7.8,normal
68.6,3.69,8.0,bearing_wear
88.7,1.89,11.8,overheating";
#2. Training configuration: 3 continuous inputs, 1 nominal output;
json load "rx_cfg" "{\"filePath\": \"tmp/rx_motor.csv\", \"modelPath\": \"tmp/rx_motor.md\", \"helperPath\": \"tmp/rx_motor.hl\", \"nbLoop\": \"5\", \"validationPercent\": \"0.25\", \"shuffle\": \"true\", \"seed\": \"1001\", \"cols\": []}";
json load "rx_names" "[\"temperature\", \"vibration\", \"current\", \"state\"]";
for (-> "[i]" 0) (< [i] 4) (++ "[i]") {
  json load "rx_col" "{}";
  json iobject "rx_col" / "index" (concat "" [i]) STR;
  json iobject "rx_col" / "title" (json select "rx_names" (concat "/[" [i] "]")) STR;
  json iobject "rx_col" / "type" (if (== [i] 3) {"out"} {"in"}) STR;
  json iarray "rx_cfg" "/cols" (json doc "rx_col") OBJ;
};
#3. Train a feedforward network with 5-fold cross-validation, 25 pct held back;
dl csv execute_config (json doc "rx_cfg");
#4. Load the model and its normalisation helper, diagnose three motors;
dl csv load_network "tmp/rx_motor.md" "tmp/rx_motor.hl";
-> "[r]" (concat "M1: " (dl csv predict "[63.0, 1.1, 8.1]") ", M2: " (dl csv predict "[67.5, 4.8, 8.7]") ", M3: " (dl csv predict "[90.2, 1.5, 11.8]"));
file delete "tmp/rx_motor.csv"; file delete "tmp/rx_motor.md"; file delete "tmp/rx_motor.hl";
json unload "rx_cfg"; json unload "rx_names"; json unload "rx_col";
[r]

M1: normal, M2: bearing_wear, M3: overheating

Train an image classifier on photos

Build the training configuration of an image network (100 x 100 RGB photos of two dog breeds, one hidden layer of 100 tanh neurons), train it for one minute and classify a new photo. Needs a folder of labelled photos, so it is not executed here.

#1. Open the training configuration file, images resized to 100 x 100 RGB;
file writer_open "rx_w" "tmp/rx_pets/imgTrainConfig.txt" true TEXT "utf-8";
dl img step1 create_training "rx_w" 100 100 true;
#2. Add the labelled photos, one folder per breed;
json load "rx_breeds" "[\"beagle\", \"poodle\"]";
for (-> "[b]" 0) (< [b] (json count "rx_breeds" /)) (++ "[b]") {
	-> "[id]" (json select "rx_breeds" (concat "/[" [b] "]"));
	-> "[dir]" (concat "tmp/rx_pets/" [id]);
	json load "rx_files" (file dir_list [dir]);
	for (-> "[i]" 0) (< [i] (json count "rx_files" /)) (++ "[i]") {
		-> "[cur_file]" (json select "rx_files" (concat "/[" [i] "]"));
		if (string ends_with [cur_file] ".jpg") {
			dl img step2 add_image "rx_w" (concat [dir] "/" [cur_file]) [id];
		};
	};
};
#3. One hidden layer of 100 neurons, tanh activation;
dl img step3 create_hidden_layer "rx_w" "100";
dl img step4 create_or_load_network "rx_w" "tanh" "tmp/rx_pets/network.eg";
#4. Train 1 minute, stop under 25 pct error or after 50 cycles, then test one photo;
dl img step5 train_network "rx_w" "console" 1 0.25 50 "tmp/rx_pets/network.eg";
dl img step6 predict "rx_w" "tmp/rx_pets/test/beagle_01.jpg" "beagle";
file writer_close "rx_w";
#5. Run the configuration, then classify a new photo with the saved network;
in editor {
	dl img execute_config "tmp/rx_pets/imgTrainConfig.txt"
};
dl img load_network "tmp/rx_pets/network.eg";
dl img predict "tmp/rx_pets/test/dog_42.jpg" true 100 100 "{\"0\": \"beagle\", \"1\": \"poodle\"}"

Confusion matrix of an intent classifier

A help desk classifier routes messages to billing, technical or shipping. Train it, run it on a held-out test set with the true classes, fill a 3 x 3 confusion matrix, and compute the accuracy (trace / total) and the recall of the billing class.

#1. Intent classifier of a help desk: three classes;
if (== (dl bayesian exist "rx_intent") 1) {dl bayesian delete "rx_intent";};
dl bayesian create "rx_intent" "[\"billing\", \"technical\", \"shipping\"]";
#2. Training set;
dl bayesian add_sentence "rx_intent" "billing" "I was charged twice on my invoice";
dl bayesian add_sentence "rx_intent" "billing" "refund the payment of my last invoice";
dl bayesian add_sentence "rx_intent" "billing" "wrong amount on my bill";
dl bayesian add_sentence "rx_intent" "technical" "the application crashes at login";
dl bayesian add_sentence "rx_intent" "technical" "error message when I open the app";
dl bayesian add_sentence "rx_intent" "technical" "the device does not start after the update";
dl bayesian add_sentence "rx_intent" "shipping" "my parcel has not arrived yet";
dl bayesian add_sentence "rx_intent" "shipping" "change the delivery address of my order";
dl bayesian add_sentence "rx_intent" "shipping" "the parcel was delivered to the wrong address";
dl bayesian init "rx_intent" 1;
#3. Held-out test set: sentence and true class index;
json load "rx_test" "[[\"double payment on the invoice\", 0], [\"the bill amount is wrong\", 0], [\"app error after the update\", 1], [\"login crashes every time\", 1], [\"where is my parcel\", 2], [\"wrong delivery address\", 2], [\"refund my order, the parcel never arrived\", 2], [\"I paid but the app shows an error\", 0], [\"the update changed my delivery address\", 2]]";
json load "rx_cls" "{\"billing\": 1, \"technical\": 2, \"shipping\": 3}";
#4. Confusion matrix: rows = true class, columns = predicted class;
-> "[cm]" (matrix zeros 3 3);
for (-> "[i]" 0) (< [i] (json count "rx_test" /)) (++ "[i]") {
  json load "rx_p" (dl bayesian predict "rx_intent" (json select "rx_test" (concat "/[" [i] "]/[0]")));
  -> "[t]" (+ 1 (json select "rx_test" (concat "/[" [i] "]/[1]")));
  -> "[p]" (json select "rx_cls" (concat "/" (json select "rx_p" "/prediction")));
  -> "[cm]" (matrix set [cm] [t] [p] (+ (matrix get [cm] [t] [p]) 1));
};
#5. Accuracy = trace / total, recall of the billing class;
-> "[acc]" (/ (matrix trace [cm]) (json count "rx_test" /));
-> "[rec]" (/ (matrix get [cm] 1 1) (stat sum (matrix row [cm] 1)));
dl bayesian delete "rx_intent";
json unload "rx_test"; json unload "rx_cls"; json unload "rx_p";
concat "confusion matrix " [cm] " | accuracy " (math round (* 100 [acc]) 1) " pct | billing recall " (math round (* 100 [rec]) 1) " pct"

confusion matrix [[2,1,0],[0,2,0],[0,0,4]] | accuracy 88.9 pct | billing recall 66.7 pct

Write the training plan of an image classifier

Before a long training, the image network is described step by step in a configuration file: input size, tagged images, hidden layer, output activation and training strategy. The script writes the file with the dl img steps 1 to 5, closes it (step 7) and reviews it.

#1. Open the training configuration file of an image classifier;
file writer_open "rx_p4h_w1" "tmp/tuto_p4h_img_config.txt" false TEXT "utf-8";
#2. 32x32 RGB inputs, two tagged images per class;
dl img step1 create_training "rx_p4h_w1" 32 32 true;
dl img step2 add_image "rx_p4h_w1" "images/brain.png" "brain";
dl img step2 add_image "rx_p4h_w1" "images/brain32x32.png" "brain";
dl img step2 add_image "rx_p4h_w1" "images/chat.png" "chat";
dl img step2 add_image "rx_p4h_w1" "images/copy.png" "chat";
#3. Hidden layer of 64 ReLU neurons with bias, softmax output;
dl img step3 create_hidden_layer "rx_p4h_w1" 64 "ReLU" true;
dl img step4 create_or_load_network "rx_p4h_w1" "SoftMax" "tmp/tuto_p4h_img.eg";
#4. Train 2 minutes or until the error is below 0.05;
dl img step5 train_network "rx_p4h_w1" "console" 2 0.05 50 "tmp/tuto_p4h_img.eg";
file writer_close "rx_p4h_w1";
#5. Review the generated file, then delete it;
-> "[config]" (file load "tmp/tuto_p4h_img_config.txt" "utf-8");
file delete "tmp/tuto_p4h_img_config.txt";
[config]

CreateTraining: width:32, height:32, rgb:1
Input: image:images/brain.png, identity:brain
Input: image:images/brain32x32.png, identity:brain
Input: image:images/chat.png, identity:chat
Input: image:images/copy.png, identity:chat
NetworkHidden: nbNeuron:64, activation:ReLU, hasBias:1
NetworkCreateOrLoad: outputActivation:SoftMax, saveNetworkPath:tmp/tuto_p4h_img.eg
Train: Mode:console, Minutes:2, StrategyError:0.05, StrategyCycles:50, saveNetworkPath:tmp/tuto_p4h_img.eg

Check a trained network on the XOR truth table

XOR is the classic problem a single neuron cannot solve. The script loads a trained multilayer network and its normalisation helper, predicts the 4 cases of the truth table and checks each output.

#1. Load the trained XOR network and its normalisation helper;
dl csv load_network "demo/xor.md" "demo/xor.hl";
#2. Truth table: the network must learn a non linear frontier;
-> "[table]" "";
for (-> "[a]" 0) (<= [a] 1) (++ "[a]") {
	for (-> "[b]" 0) (<= [b] 1) (++ "[b]") {
		-> "[out]" (dl csv predict (concat "[" [a] ".0, " [b] ".0]"));
		-> "[ok]" (if (== [out] (math abs (- [a] [b]))) {"ok"} {"WRONG"});
		-> "[table]" (concat [table] [a] " xor " [b] " = " [out] " " [ok] " | ");
	};
};
[table]

0 xor 0 = 0 ok | 0 xor 1 = 1 ok | 1 xor 0 = 1 ok | 1 xor 1 = 0 ok |

Scaled dot-product attention of a transformer

The core of a transformer: for 3 tokens, the script computes the scores Q.Kt / sqrt(d_k), a stable softmax on each row (max subtracted) and the output A.V.

#1. Queries, keys and values of 3 tokens, d_k = 4;
-> "[Q]" "[[1, 0, 1, 0], [0, 1, 0, 1], [1, 1, 0, 0]]";
-> "[K]" "[[1, 0, 1, 0], [0, 2, 0, 2], [1, 1, 1, 1]]";
-> "[V]" "[[1, 0], [0, 1], [0.5, 0.5]]";
#2. Scores = Q.Kt / sqrt d_k;
json load "rx_p4h_s" (matrix scalar_mul (matrix mul [Q] (matrix transpose [K])) (/ 1 (math sqrt 4)));
#3. Softmax on each row, max subtracted for stability;
json load "rx_p4h_a" "[]";
for (-> "[i]" 0) (< [i] 3) (++ "[i]") {
	-> "[row]" (json select "rx_p4h_s" (concat "/[" [i] "]"));
	-> "[m]" (stat max [row]);
	-> "[e0]" (math exp (- (json select "rx_p4h_s" (concat "/[" [i] "]/[0]")) [m]));
	-> "[e1]" (math exp (- (json select "rx_p4h_s" (concat "/[" [i] "]/[1]")) [m]));
	-> "[e2]" (math exp (- (json select "rx_p4h_s" (concat "/[" [i] "]/[2]")) [m]));
	-> "[z]" (+ [e0] [e1] [e2]);
	json iarray "rx_p4h_a" / (concat "[" (/ [e0] [z]) "," (/ [e1] [z]) "," (/ [e2] [z]) "]") ARRAY;
};
#4. Output = A.V;
concat "attention weights of token 1: " (json select "rx_p4h_a" "/[0]") " | output A.V: " (matrix mul (json doc "rx_p4h_a") [V])

attention weights of token 1: [
  0.4223187982515182,
  0.15536240349696362,
  0.4223187982515182
] | output A.V: [[0.6334781973772773,0.3665218026227227],[0.21239480869777927,0.7876051913022206],[0.424522403214174,0.5754775967858261]]

Xavier initialisation of a dense layer

A dense layer 8 to 4 is initialised with the Xavier (Glorot) uniform law, bound a = sqrt(6 / (fan_in + fan_out)). The script draws the weights, checks they are inside [-a, a] and runs one forward pass with ReLU. The weights change at each run.

#1. Layer 8 inputs to 4 neurons: Xavier uniform bound a = sqrt 6 / fan_in + fan_out;
-> "[a]" (math sqrt (/ 6 (+ 8 4)));
#2. W = a . 2R - 1 with R uniform in 0 1;
-> "[R]" (matrix random 4 8);
-> "[W]" (matrix scalar_mul (matrix sub (matrix scalar_mul [R] 2) (matrix ones 4 8)) [a]);
-> "[flat]" (string replace (string replace (string replace [W] "],[" ",") "[[" "[") "]]" "]");
#3. Forward pass of one input and ReLU;
json load "rx_p4h_h" (matrix mul [W] "[0.2, -0.1, 0.5, 0.3, -0.4, 0.1, 0.0, 0.6]");
-> "[active]" 0;
for (-> "[i]" 0) (< [i] 4) (++ "[i]") {
	if (> (json select "rx_p4h_h" (concat "/[" [i] "]")) 0) {++ "[active]";};
};
concat "shape " (matrix size [W]) " | bound a = " (math round [a] 4) " | all 32 weights in [-a, a]: " (<= (math max (math abs (stat min [flat])) (stat max [flat])) [a]) " | ReLU neurons active on the input: " [active] "/4"

shape [4,8] | bound a = 0.7071 | all 32 weights in [-a, a]: 1 | ReLU neurons active on the input: 2/4

Train a sigmoid neuron by gradient descent

A failure model p = sigmoid(w.x + b) is trained on 8 machines (hours of use, failure 0/1) by batch gradient descent on the cross entropy, 100 epochs, learning rate 2. The script plots the learning curve with the final weights in the title.

#1. Hours of use of a machine and failure 0/1;
json load "rx_p4h_x" "[1, 2, 3, 4, 5, 6, 7, 8]";
json load "rx_p4h_y" "[0, 0, 0, 1, 0, 1, 1, 1]";
-> "[w]" 0; -> "[b]" 0; -> "[lr]" 2;
json load "rx_p4h_loss" "[]";
#2. Batch gradient descent of a sigmoid neuron, cross entropy loss;
for (-> "[ep]" 0) (< [ep] 100) (++ "[ep]") {
	-> "[gw]" 0; -> "[gb]" 0; -> "[L]" 0;
	for (-> "[i]" 0) (< [i] 8) (++ "[i]") {
		-> "[xi]" (/ (json select "rx_p4h_x" (concat "/[" [i] "]")) 8);
		-> "[yi]" (json select "rx_p4h_y" (concat "/[" [i] "]"));
		-> "[p]" (math calc "1/(1+exp(-([w]*[xi]+[b])))");
		-> "[gw]" (math calc "[gw]+([p]-[yi])*[xi]/8");
		-> "[gb]" (math calc "[gb]+([p]-[yi])/8");
		-> "[L]" (math calc "[L]-([yi]*ln([p])+(1-[yi])*ln(1-[p]))/8");
	};
	-> "[w]" (math calc "[w]-[lr]*[gw]");
	-> "[b]" (math calc "[b]-[lr]*[gb]");
	json iarray "rx_p4h_loss" / [L] NUM;
};
#3. Learning curve;
plot chart (concat "{\"title\": \"Training loss, w=" (math round [w] 2) " b=" (math round [b] 2) "\", \"x_label\": \"epoch\", \"y_label\": \"cross entropy\", \"series\": [{\"name\": \"loss\", \"y\": " (json doc "rx_p4h_loss") "}]}")

The chart is shown in the Plot tab of the editor.