diff --git a/NeuralNetwork/Examples/HelloLSTM.mo b/NeuralNetwork/Examples/HelloLSTM.mo new file mode 100644 index 0000000..6ac022d --- /dev/null +++ b/NeuralNetwork/Examples/HelloLSTM.mo @@ -0,0 +1,15 @@ +within NeuralNetwork.Examples; + +model HelloLSTM + extends Modelica.Icons.Example; + Modelica.Blocks.Sources.Sine sine(f = 1) annotation( + Placement(transformation(origin = {-60, 0}, extent = {{-20, -20}, {20, 20}}))); + Utilities.SimpleLSTM lstm annotation( + Placement(transformation(origin = {40, -30}, extent = {{-20, -20}, {20, 20}}))); +equation + connect(sine.y, lstm.u) annotation( + Line(points = {{-38, 0}, {-20, 0}, {-20, -30}, {16, -30}}, color = {0, 0, 127})); + + annotation( + experiment(StopTime = 1)); +end HelloLSTM; diff --git a/NeuralNetwork/Examples/Utilities/SimpleLSTM.mo b/NeuralNetwork/Examples/Utilities/SimpleLSTM.mo new file mode 100644 index 0000000..f212c9b --- /dev/null +++ b/NeuralNetwork/Examples/Utilities/SimpleLSTM.mo @@ -0,0 +1,40 @@ +within NeuralNetwork.Examples.Utilities; + +block SimpleLSTM + extends NeuralNetwork.Networks.Interfaces.SISO(final u = layer.u[1], final y = layer.y[1]); + Layer.LSTM layer( + Wf = Wf, + Wi = Wi, + Wo = Wo, + Wc = Wc, + Uf = Uf, + Ui = Ui, + Uo = Uo, + Uc = Uc, + bf = bf, + bi = bi, + bo = bo, + bc = bc, + redeclare function sigma_g = NeuralNetwork.ActivationFunctions.Sigmoid, + redeclare function sigma_c = NeuralNetwork.ActivationFunctions.Tanh, + redeclare function sigma_h = NeuralNetwork.ActivationFunctions.Id, + clk = Clock(1,100) + ) annotation( + Placement(transformation(origin = {0, 0}, extent = {{-30, -30}, {30, 30}}))); + + parameter Real[2,1] Wf = {{0.9168842650273863}, {-0.41626278387433935}}; + parameter Real[2,1] Wi = {{-0.3579903628947263}, {-0.7272669817156694}}; + parameter Real[2,1] Wo = {{-0.3563311828380926}, {-0.23704936315985936}}; + parameter Real[2,1] Wc = {{-0.24949461025816877}, {0.5663348709465208}}; + + parameter Real[2,2] Uf = {{0.9828238516747565, -0.4323916905269658}, {0.6597874743507453, -0.49797468723047333}}; + parameter Real[2,2] Ui = {{0.20470995578249807, 0.31776625052682883}, {0.6305946426671325, 0.05439201192681886}}; + parameter Real[2,2] Uo = {{0.13020845000176196, 0.13822326996697631}, {0.12134587258765883, -0.7152233831611636}}; + parameter Real[2,2] Uc = {{0.37901032286011893, -0.4157978629827952}, {-0.8435991979907957, 0.19994829923186774}}; + + parameter Real[2] bf = {-0.5486002272036146, 0.09978158879021293}; + parameter Real[2] bi = {0.5991231180820973, -0.9000209428865766}; + parameter Real[2] bo = {0.2874148054581702, 0.07961885205233754}; + parameter Real[2] bc = {-0.2478805511544202, 0.49672666080562666}; + +end SimpleLSTM; diff --git a/NeuralNetwork/Examples/Utilities/package.order b/NeuralNetwork/Examples/Utilities/package.order index b66e0c2..ab08499 100644 --- a/NeuralNetwork/Examples/Utilities/package.order +++ b/NeuralNetwork/Examples/Utilities/package.order @@ -1,5 +1,6 @@ NARX_Network SamplerMIMO TimeDelay -SimpleNetwork SimpleEquation +SimpleNetwork +SimpleLSTM diff --git a/NeuralNetwork/Examples/package.order b/NeuralNetwork/Examples/package.order index 954b851..5c140e1 100644 --- a/NeuralNetwork/Examples/package.order +++ b/NeuralNetwork/Examples/package.order @@ -1,3 +1,4 @@ HelloWorld +HelloLSTM NARX Utilities diff --git a/NeuralNetwork/Layer/Interfaces/Layer.mo b/NeuralNetwork/Layer/Interfaces/Layer.mo index 23b8060..e562111 100644 --- a/NeuralNetwork/Layer/Interfaces/Layer.mo +++ b/NeuralNetwork/Layer/Interfaces/Layer.mo @@ -1,6 +1,6 @@ within NeuralNetwork.Layer.Interfaces; -partial model Layer +partial block Layer // Topology of the neural network parameter Integer numInputs "Specification of the inputs of the layer"; parameter Integer numNeurons "Number of neurons in the layer"; diff --git a/NeuralNetwork/Layer/LSTM.mo b/NeuralNetwork/Layer/LSTM.mo new file mode 100644 index 0000000..f0e6fd0 --- /dev/null +++ b/NeuralNetwork/Layer/LSTM.mo @@ -0,0 +1,67 @@ +within NeuralNetwork.Layer; + +block LSTM "Block for a LSTM neural network layer" + extends Interfaces.Layer(final numInputs = size(Wf, 2), final numNeurons = size(bf, 1)); + + /* weights and biases */ + parameter Real Wf[:, :] "Weights of input and forget gate"; + parameter Real Wi[:, :] "Weights of input and input gate"; + parameter Real Wo[:, :] "Weights of input and output gate"; + parameter Real Wc[:, :] "Weights of input and memory cell gate"; + + parameter Real Uf[:, :] "Weights of recurrent and forget gate"; + parameter Real Ui[:, :] "Weights of recurrent and input gate"; + parameter Real Uo[:, :] "Weights of recurrent and output gate"; + parameter Real Uc[:, :] "Weights of recurrent and memory cell gate"; + + parameter Real bf[:] "Biases of forget gate"; + parameter Real bi[:] "Biases of input gate"; + parameter Real bo[:] "Biases of output gate"; + parameter Real bc[:] "Biases of memory cell gate"; + + replaceable function sigma_g = NeuralNetwork.ActivationFunctions.ActivationFunction "Activation function of the layer"; + replaceable function sigma_c = NeuralNetwork.ActivationFunctions.ActivationFunction "Activation function of the layer"; + replaceable function sigma_h = NeuralNetwork.ActivationFunctions.ActivationFunction "Activation function of the layer"; + + Clock clk "clock for sampling the input u"; + +protected + Real x[numInputs] "sampled input vector"; + Real f[numNeurons] "forget gate's activation vector"; + Real i[numNeurons] "input gate's activation vector"; + Real o[numNeurons] "output gate's activation vector"; + Real ca[numNeurons] "cell input activation vector"; + Real c[numNeurons] (each start = 0.0) "cell state vector"; + Real h[numNeurons] (each start = 0.0) "hidden state vector"; +equation + x = sample(u, clk) "input"; + y = hold(h) "output"; + + when clk then + f = sigma_g(Wf*x + Uf*previous(h) + bf) "forget gate"; + i = sigma_g(Wi*x + Ui*previous(h) + bi) "input gate"; + o = sigma_g(Wo*x + Uo*previous(h) + bo) "output gate"; + ca = sigma_c(Wc*x + Uc*previous(h) + bc) "cell activation"; + c = f .* previous(c) + i .* ca "cell state"; + h = o .* sigma_h(c) "hidden state"; + end when; + + annotation( + Icon(graphics = { + Ellipse(origin = {0, -60}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}), + Ellipse(origin = {0, -30}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}), + Ellipse(origin = {0, -10}, lineColor = {45, 45, 45}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{-2, 2}, {2, -2}}), + Ellipse(origin = {0, 0}, lineColor = {45, 45, 45}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{-2, 2}, {2, -2}}), + Ellipse(origin = {0, 10}, lineColor = {45, 45, 45}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{-2, 2}, {2, -2}}), + Ellipse(origin = {0, 30}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}), + Ellipse(origin = {0, 60}, lineColor = {0, 170, 255}, fillColor = {255, 255, 255}, fillPattern = FillPattern.Sphere, extent = {{10, 10}, {-10, -10}}), + Line(points = {{-38, -8}, {-12, -56}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{-38, -4}, {-12, -26}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{-38, 4}, {-12, 26}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{-38, 8}, {-12, 56}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{12, -56}, {38, -8}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{12, -26}, {38, -4}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{12, 26}, {38, 4}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Line(points = {{12, 56}, {38, 8}}, thickness = 0.5, arrow = {Arrow.None, Arrow.Filled}), + Text(origin = {0, -80}, textColor = {0, 0, 255}, extent = {{60, 10}, {-60, -10}}, textString = "neurons = %numNeurons")})); +end LSTM; diff --git a/NeuralNetwork/Layer/package.order b/NeuralNetwork/Layer/package.order index 5ed7343..65f5108 100644 --- a/NeuralNetwork/Layer/package.order +++ b/NeuralNetwork/Layer/package.order @@ -1,3 +1,4 @@ Interfaces Preprocessing Dense +LSTM