Add LSTM layer

NeuralNetwork/Examples/HelloLSTM.mo

@@ -0,0 +1,15 @@
+within NeuralNetwork.Examples;
+
+model HelloLSTM
+  extends Modelica.Icons.Example;
+  Modelica.Blocks.Sources.Sine sine(f = 1) annotation(
+    Placement(transformation(origin = {-60, 0}, extent = {{-20, -20}, {20, 20}})));
+  Utilities.SimpleLSTM lstm annotation(
+    Placement(transformation(origin = {40, -30}, extent = {{-20, -20}, {20, 20}})));
+equation
+  connect(sine.y, lstm.u) annotation(
+    Line(points = {{-38, 0}, {-20, 0}, {-20, -30}, {16, -30}}, color = {0, 0, 127}));
+
+  annotation(
+    experiment(StopTime = 1));
+end HelloLSTM;

NeuralNetwork/Examples/Utilities/SimpleLSTM.mo

@@ -0,0 +1,40 @@
+within NeuralNetwork.Examples.Utilities;
+
+block SimpleLSTM
+  extends NeuralNetwork.Networks.Interfaces.SISO(final u = layer.u[1], final y = layer.y[1]);
+  Layer.LSTM layer(
+    Wf = Wf,
+    Wi = Wi,
+    Wo = Wo,
+    Wc = Wc,
+    Uf = Uf,
+    Ui = Ui,
+    Uo = Uo,
+    Uc = Uc,
+    bf = bf,
+    bi = bi,
+    bo = bo,
+    bc = bc,
+    redeclare function sigma_g = NeuralNetwork.ActivationFunctions.Sigmoid,
+    redeclare function sigma_c = NeuralNetwork.ActivationFunctions.Tanh,
+    redeclare function sigma_h = NeuralNetwork.ActivationFunctions.Id,
+    clk = Clock(1,100)
+  ) annotation(
+    Placement(transformation(origin = {0, 0}, extent = {{-30, -30}, {30, 30}})));
+
+  parameter Real[2,1] Wf = {{0.9168842650273863}, {-0.41626278387433935}};
+  parameter Real[2,1] Wi = {{-0.3579903628947263}, {-0.7272669817156694}};
+  parameter Real[2,1] Wo = {{-0.3563311828380926}, {-0.23704936315985936}};
+  parameter Real[2,1] Wc = {{-0.24949461025816877}, {0.5663348709465208}};
+
+  parameter Real[2,2] Uf = {{0.9828238516747565, -0.4323916905269658}, {0.6597874743507453, -0.49797468723047333}};
+  parameter Real[2,2] Ui = {{0.20470995578249807, 0.31776625052682883}, {0.6305946426671325, 0.05439201192681886}};
+  parameter Real[2,2] Uo = {{0.13020845000176196, 0.13822326996697631}, {0.12134587258765883, -0.7152233831611636}};
+  parameter Real[2,2] Uc = {{0.37901032286011893, -0.4157978629827952}, {-0.8435991979907957, 0.19994829923186774}};
+
+  parameter Real[2] bf = {-0.5486002272036146, 0.09978158879021293};
+  parameter Real[2] bi = {0.5991231180820973, -0.9000209428865766};
+  parameter Real[2] bo = {0.2874148054581702, 0.07961885205233754};
+  parameter Real[2] bc = {-0.2478805511544202, 0.49672666080562666};
+
+end SimpleLSTM;

NeuralNetwork/Examples/Utilities/package.order

@@ -1,5 +1,6 @@
 NARX_Network
 SamplerMIMO
 TimeDelay
-SimpleNetwork
 SimpleEquation
+SimpleNetwork
+SimpleLSTM

NeuralNetwork/Examples/package.order

@@ -1,3 +1,4 @@
 HelloWorld
+HelloLSTM
 NARX
 Utilities

NeuralNetwork/Layer/Interfaces/Layer.mo

@@ -1,6 +1,6 @@
 within NeuralNetwork.Layer.Interfaces;
 
-partial model Layer
+partial block Layer
   // Topology of the neural network
   parameter Integer numInputs   "Specification of the inputs of the layer";
   parameter Integer numNeurons  "Number of neurons in the layer";

NeuralNetwork/Layer/LSTM.mo

@@ -0,0 +1,67 @@
+within NeuralNetwork.Layer;
+
+block LSTM "Block for a LSTM neural network layer"
+  extends Interfaces.Layer(final numInputs = size(Wf, 2), final numNeurons = size(bf, 1));
+
+  /* weights and biases */
+  parameter Real Wf[:, :]  "Weights of input and forget gate";
+  parameter Real Wi[:, :]  "Weights of input and input gate";
+  parameter Real Wo[:, :]  "Weights of input and output gate";
+  parameter Real Wc[:, :]  "Weights of input and memory cell gate";
+
+  parameter Real Uf[:, :]  "Weights of recurrent and forget gate";
+  parameter Real Ui[:, :]  "Weights of recurrent and input gate";
+  parameter Real Uo[:, :]  "Weights of recurrent and output gate";
+  parameter Real Uc[:, :]  "Weights of recurrent and memory cell gate";
+
+  parameter Real bf[:]  "Biases of forget gate";
+  parameter Real bi[:]  "Biases of input gate";
+  parameter Real bo[:]  "Biases of output gate";
+  parameter Real bc[:]  "Biases of memory cell gate";
+
+  replaceable function sigma_g = NeuralNetwork.ActivationFunctions.ActivationFunction "Activation function of the layer";
+  replaceable function sigma_c = NeuralNetwork.ActivationFunctions.ActivationFunction "Activation function of the layer";
+  replaceable function sigma_h = NeuralNetwork.ActivationFunctions.ActivationFunction "Activation function of the layer";
+
+  Clock clk "clock for sampling the input u";
+
+protected
+  Real x[numInputs]   "sampled input vector";
+  Real f[numNeurons]  "forget gate's activation vector";
+  Real i[numNeurons]  "input gate's activation vector";
+  Real o[numNeurons]  "output gate's activation vector";
+  Real ca[numNeurons] "cell input activation vector";
+  Real c[numNeurons] (each start = 0.0) "cell state vector";
+  Real h[numNeurons] (each start = 0.0) "hidden state vector";
+equation
+  x = sample(u, clk)  "input";
+  y = hold(h)         "output";
+
+  when clk then
+    f = sigma_g(Wf*x + Uf*previous(h) + bf)   "forget gate";
+    i = sigma_g(Wi*x + Ui*previous(h) + bi)   "input gate";
+    o = sigma_g(Wo*x + Uo*previous(h) + bo)   "output gate";
+    ca = sigma_c(Wc*x + Uc*previous(h) + bc)  "cell activation";
+    c = f .* previous(c) + i .* ca            "cell state";
+    h = o .* sigma_h(c)                       "hidden state";
+  end when;
+
+  annotation(
+    Icon(graphics = {
+      Ellipse(origin = {0, -60}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}),
+      Ellipse(origin = {0, -30}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}),
+      Ellipse(origin = {0, -10}, lineColor = {45, 45, 45}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{-2, 2}, {2, -2}}),
+      Ellipse(origin = {0, 0}, lineColor = {45, 45, 45}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{-2, 2}, {2, -2}}),
+      Ellipse(origin = {0, 10}, lineColor = {45, 45, 45}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{-2, 2}, {2, -2}}),
+      Ellipse(origin = {0, 30}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}),
+      Ellipse(origin = {0, 60}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}),
+      Line(points = {{-38, -8}, {-12, -56}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{-38, -4}, {-12, -26}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{-38, 4}, {-12, 26}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{-38, 8}, {-12, 56}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{12, -56}, {38, -8}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{12, -26}, {38, -4}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{12, 26}, {38, 4}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Line(points = {{12, 56}, {38, 8}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}),
+      Text(origin = {0, -80}, textColor = {0, 0, 255}, extent = {{60, 10}, {-60, -10}}, textString = "neurons = %numNeurons")}));
+end LSTM;

NeuralNetwork/Layer/package.order

@@ -1,3 +1,4 @@
 Interfaces
 Preprocessing
 Dense
+LSTM