Merge pull request #385 from leondavi/loss_args_str

[NERLPLANNER] Add loss args

src_cpp/common/nerlWorker.cpp

@@ -4,14 +4,15 @@ using namespace nerlnet;
 
 NerlWorker::NerlWorker(int model_type, std::string &model_args_str , std::string &layer_sizes_str, std::string &layer_types_list, std::string &layers_functionality,
                     float learning_rate, int epochs, int optimizer_type, std::string &optimizer_args_str,
-                    int loss_method, int distributed_system_type, std::string &distributed_system_args_str)
+                    int loss_method, std::string &loss_args_str, int distributed_system_type, std::string &distributed_system_args_str)
 {
     _model_type = model_type;
     _model_args_str = model_args_str;
     _learning_rate = learning_rate;
     _epochs = epochs;
     _optimizer_type = optimizer_type;
     _loss_method = loss_method;
+    _loss_args_str = loss_args_str;
     _distributed_system_type = distributed_system_type;
     _distributed_system_args_str = distributed_system_args_str;
     _nerl_layers_linked_list = parse_layers_input(layer_sizes_str,layer_types_list,layers_functionality);

src_cpp/common/nerlWorker.h

@@ -12,7 +12,7 @@ namespace nerlnet
 
     NerlWorker(int model_type, std::string &model_args_str, std::string &layer_sizes_str, std::string &layer_types_list, std::string &layers_functionality,
                      float learning_rate, int epochs, int optimizer_type, std::string &optimizer_args_str,
-                     int loss_method, int distributed_system_type, std::string &distributed_system_args_str);
+                     int loss_method, std::string &loss_args_str, int distributed_system_type, std::string &distributed_system_args_str);
     ~NerlWorker();
 
     std::shared_ptr<NerlLayer> parse_layers_input(std::string &layer_sizes_str, std::string &layer_types_list, std::string &layers_functionality);
@@ -35,6 +35,7 @@ namespace nerlnet
     int _epochs;
     int _optimizer_type;
     int _loss_method;
+    std::string _loss_args_str;
     std::string _distributed_system_args_str;
     std::shared_ptr<std::vector<int>> _train_labels_count; // accumulates the number of each label in the training set
 

src_cpp/common/nerlWorkerFunc.h

@@ -58,7 +58,7 @@ enum {KERNEL_SIZE = -1, PADDING_SIZE = -2,STRIDE_SIZE = -3 ,POOLING_SIZE= -4 , I
 
 template <class NerlWorkerType>
 std::shared_ptr<NerlWorkerType> parse_model_params(std::string &model_type_str,std::string &model_args_str, std::string &learning_rate_str,std::string &epochs_str,
-                                    std::string &optimizer_type_str,std::string &loss_method_str,std::string &distributed_system_type_str,
+                                    std::string &optimizer_type_str,std::string &loss_method_str, std::string &loss_args_str, std::string &distributed_system_type_str,
                                     std::string &layer_sizes_str,std::string &layer_types_str,std::string &layers_functionality_str,
                                     std::string &optimizer_args_str,std::string &distributed_system_args_str)
 {
@@ -68,7 +68,7 @@ std::shared_ptr<NerlWorkerType> parse_model_params(std::string &model_type_str,s
     int optimizer_type = std::stoi(optimizer_type_str);
     int loss_method = std::stoi(loss_method_str);
     int distributed_system_type = std::stoi(distributed_system_type_str);
-    return std::make_shared<NerlWorkerType>(model_type, model_args_str, layer_sizes_str, layer_types_str, layers_functionality_str,learning_rate, epochs, optimizer_type, optimizer_args_str, loss_method, distributed_system_type, distributed_system_args_str);
+    return std::make_shared<NerlWorkerType>(model_type, model_args_str, layer_sizes_str, layer_types_str, layers_functionality_str,learning_rate, epochs, optimizer_type, optimizer_args_str, loss_method, loss_args_str, distributed_system_type, distributed_system_args_str);
 }
 
 static void parse_layer_sizes_str(std::string &layer_sizes_str, std::vector<int> &layers_types_vec, std::vector<LayerSizingParams_t> &out_layer_sizes_params)

src_cpp/common/utilities.h

@@ -18,6 +18,11 @@ std::vector<std::string> split_strings_by_comma(std::string &str);
 bool is_integer_number(const std::string &input_str);
 std::vector<std::string> matchRegex(std::string &input, std::regex re);
 
+inline bool substr_in_string(const std::string &input, const std::string &substr)
+{
+    return input.find(substr) != std::string::npos;
+}
+
 inline bool is_big_endian(void)
 {
     union {

src_cpp/common/worker_definitions_ag.h

@@ -1,7 +1,7 @@
 #pragma once
 
 // This file was auto generated
-// Generated by Nerlplanner version: 1.0.3
+// Generated by Nerlplanner version: 1.0.4
 
 namespace nerlnet {
 

src_cpp/opennnBridge/nerlWorkerNIF.h

@@ -10,18 +10,18 @@
 using namespace nerlnet;
 
 static std::shared_ptr<NerlWorkerOpenNN> create_nerlworker(std::string &model_type_str, std::string &model_args_str, std::string &learning_rate_str,
- std::string &epochs_str, std::string &optimizer_type_str, std::string &loss_method_str,
+ std::string &epochs_str, std::string &optimizer_type_str, std::string &loss_method_str, std::string &loss_args_str,
  std::string &distributed_system_type_str, std::string &layer_sizes_str, std:: string &layer_types_str,
  std::string &layers_functionality_str, std::string &optimizer_args_str, std::string &distributed_system_args_str) //all should be const reference
 {
-    std::shared_ptr<NerlWorkerOpenNN> new_worker = parse_model_params<NerlWorkerOpenNN>(model_type_str,model_args_str,learning_rate_str,epochs_str,optimizer_type_str,loss_method_str,distributed_system_type_str,layer_sizes_str,
+    std::shared_ptr<NerlWorkerOpenNN> new_worker = parse_model_params<NerlWorkerOpenNN>(model_type_str,model_args_str,learning_rate_str,epochs_str,optimizer_type_str,loss_method_str, loss_args_str, distributed_system_type_str,layer_sizes_str,
     layer_types_str,layers_functionality_str,optimizer_args_str,distributed_system_args_str);
     return new_worker;
 }
 static ERL_NIF_TERM new_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     enum{ARG_MODEL_ID,ARG_MODEL_TYPE, ARG_MODEL_ARGS, ARG_LAYERS_SIZES, ARG_LAYERS_TYPES, ARG_LAYERS_FUNCTIONALITY_CODES, ARG_LEARNING_RATE, ARG_EPOCHS, ARG_OPTIMIZER_TYPE,
-         ARG_OPTIMIZER_ARGS, ARG_LOSS_METHOD, ARG_DISTRIBUTED_SYSTEM_TYPE, ARG_DISTRIBUTED_SYSTEM_ARGS};
+         ARG_OPTIMIZER_ARGS, ARG_LOSS_METHOD, ARG_LOSS_ARGS, ARG_DISTRIBUTED_SYSTEM_TYPE, ARG_DISTRIBUTED_SYSTEM_ARGS};
 
     unsigned long modelId;
     std::string model_type_str;
@@ -34,6 +34,7 @@ static ERL_NIF_TERM new_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
     std::string optimizer_type_str;
     std::string optimizer_args_str;
     std::string loss_method_str;
+    std::string loss_args_str;
     std::string distributed_system_type_str;
     std::string distributed_system_args_str;
 
@@ -48,10 +49,12 @@ static ERL_NIF_TERM new_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
     nifpp::get_throws(env, argv[ARG_OPTIMIZER_TYPE], optimizer_type_str);
     nifpp::get_throws(env, argv[ARG_OPTIMIZER_ARGS], optimizer_args_str);
     nifpp::get_throws(env, argv[ARG_LOSS_METHOD], loss_method_str);
+    nifpp::get_throws(env, argv[ARG_LOSS_ARGS], loss_args_str);
     nifpp::get_throws(env, argv[ARG_DISTRIBUTED_SYSTEM_TYPE], distributed_system_type_str);
     nifpp::get_throws(env, argv[ARG_DISTRIBUTED_SYSTEM_ARGS], distributed_system_args_str);
 
-    std::shared_ptr<NerlWorkerOpenNN> new_nerl_worker_ptr =  create_nerlworker(model_type_str,model_args_str,learning_rate_str,epochs_str,optimizer_type_str,loss_method_str,distributed_system_type_str,layer_sizes_str,
+    std::shared_ptr<NerlWorkerOpenNN> new_nerl_worker_ptr =  create_nerlworker(model_type_str,model_args_str,learning_rate_str,epochs_str,optimizer_type_str,
+                                                                            loss_method_str,loss_args_str,distributed_system_type_str,layer_sizes_str,
     layer_types_str,layers_functionality_str,optimizer_args_str,distributed_system_args_str);
     // Create the singleton instance
     BridgeController& onnBrCtrl = BridgeController::GetInstance();
@@ -65,7 +68,7 @@ static ERL_NIF_TERM new_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
 static ERL_NIF_TERM test_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     enum{ARG_MODEL_ID,ARG_MODEL_TYPE, ARG_MODEL_ARGS , ARG_LAYERS_SIZES, ARG_LAYERS_TYPES, ARG_LAYERS_FUNCTIONALITY_CODES, ARG_LEARNING_RATE, ARG_EPOCHS, ARG_OPTIMIZER_TYPE,
-         ARG_OPTIMIZER_ARGS, ARG_LOSS_METHOD, ARG_DISTRIBUTED_SYSTEM_TYPE, ARG_DISTRIBUTED_SYSTEM_ARGS};
+         ARG_OPTIMIZER_ARGS, ARG_LOSS_METHOD, ARG_LOSS_ARGS, ARG_DISTRIBUTED_SYSTEM_TYPE, ARG_DISTRIBUTED_SYSTEM_ARGS};
 
     unsigned long modelId;
     std::string model_type_str;
@@ -78,6 +81,7 @@ static ERL_NIF_TERM test_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_
     std::string optimizer_type_str;
     std::string optimizer_args_str;
     std::string loss_method_str;
+    std::string loss_args_str;
     std::string distributed_system_type_str;
     std::string distributed_system_args_str;
 
@@ -92,10 +96,12 @@ static ERL_NIF_TERM test_nerlworker_nif(ErlNifEnv* env, int argc, const ERL_NIF_
     nifpp::get_throws(env, argv[ARG_OPTIMIZER_TYPE], optimizer_type_str);
     nifpp::get_throws(env, argv[ARG_OPTIMIZER_ARGS], optimizer_args_str);
     nifpp::get_throws(env, argv[ARG_LOSS_METHOD], loss_method_str);
+    nifpp::get_throws(env, argv[ARG_LOSS_ARGS], loss_args_str);
     nifpp::get_throws(env, argv[ARG_DISTRIBUTED_SYSTEM_TYPE], distributed_system_type_str);
     nifpp::get_throws(env, argv[ARG_DISTRIBUTED_SYSTEM_ARGS], distributed_system_args_str);
-    std::shared_ptr<NerlWorkerOpenNN> new_nerl_worker_ptr = create_nerlworker(model_type_str,model_args_str,learning_rate_str,epochs_str,optimizer_type_str,loss_method_str,distributed_system_type_str,layer_sizes_str,
-    layer_types_str,layers_functionality_str,optimizer_args_str,distributed_system_args_str);
+    std::shared_ptr<NerlWorkerOpenNN> new_nerl_worker_ptr = create_nerlworker(model_type_str,model_args_str,learning_rate_str,epochs_str,
+                                                                            optimizer_type_str,loss_method_str,loss_args_str,distributed_system_type_str,layer_sizes_str,
+                                                                            layer_types_str,layers_functionality_str,optimizer_args_str,distributed_system_args_str);
      // Create the singleton instance
     BridgeController& onnBrCtrl = BridgeController::GetInstance();
     // Put the model record to the map with modelId

src_cpp/opennnBridge/nerlWorkerOpenNN.cpp

@@ -9,9 +9,9 @@ namespace nerlnet
 
     NerlWorkerOpenNN::NerlWorkerOpenNN(int model_type, std::string &model_args_str , std::string &layer_sizes_str, std::string &layer_types_list, std::string &layers_functionality,
                     float learning_rate, int epochs, int optimizer_type, std::string &optimizer_args_str,
-                    int loss_method, int distributed_system_type, std::string &distributed_system_args_str) : NerlWorker(model_type, model_args_str , layer_sizes_str, layer_types_list, layers_functionality,
+                    int loss_method, std::string &loss_args_str, int distributed_system_type, std::string &distributed_system_args_str) : NerlWorker(model_type, model_args_str , layer_sizes_str, layer_types_list, layers_functionality,
                                                                                                                     learning_rate, epochs, optimizer_type, optimizer_args_str,
-                                                                                                                    loss_method, distributed_system_type, distributed_system_args_str)
+                                                                                                                    loss_method, loss_args_str, distributed_system_type, distributed_system_args_str)
     {
         _neural_network_ptr = std::make_shared<opennn::NeuralNetwork>();
         generate_opennn_neural_network();
@@ -169,11 +169,12 @@ namespace nerlnet
     **/
     void NerlWorkerOpenNN::generate_training_strategy()
     {
-  _training_strategy_ptr->set_neural_network_pointer(_neural_network_ptr.get()); // Neural network must be defined at this point
-    set_optimization_method(_optimizer_type,_learning_rate);
-    set_loss_method(_loss_method);
-    _training_strategy_ptr->set_maximum_epochs_number(_epochs); 
-    _training_strategy_ptr->set_display(TRAINING_STRATEGY_SET_DISPLAY_OFF); 
+      _training_strategy_ptr->set_neural_network_pointer(_neural_network_ptr.get()); // Neural network must be defined at this point
+      set_optimization_method(_optimizer_type,_learning_rate);
+      set_loss_method(_loss_method);
+      _training_strategy_ptr->get_loss_index_pointer()->set_regularization_method(parse_regularization_loss_args(_loss_args_str));
+      _training_strategy_ptr->set_maximum_epochs_number(_epochs); 
+      _training_strategy_ptr->set_display(TRAINING_STRATEGY_SET_DISPLAY_OFF); 
     }
 
     void NerlWorkerOpenNN::set_optimization_method(int optimizer_type,int learning_rate){
@@ -861,5 +862,34 @@ namespace nerlnet
             }
         }
     }
+
+
+    opennn::LossIndex::RegularizationMethod NerlWorkerOpenNN::parse_regularization_loss_args(const std::string &loss_args)
+    {
+        enum LossArgsEnum{L1=0,L2=1,NONE=2};
+        string L1_text = "reg=L1";
+        string L2_text = "reg=L2";
+        string NoRegularization_pattern_1 = "reg=None";
+        string NoRegularization_pattern_2 = "reg=NoRegularization";
+
+        int loss_reguarization_type = nerlnet_utilities::substr_in_string(loss_args,L1_text) ? L1 : 
+                            nerlnet_utilities::substr_in_string(loss_args,L2_text) ? L2 :
+                            nerlnet_utilities::substr_in_string(loss_args,NoRegularization_pattern_1) ? NONE :
+                            nerlnet_utilities::substr_in_string(loss_args,NoRegularization_pattern_2) ? NONE : NONE;
+
+        switch (loss_reguarization_type)
+        {
+        case L1:
+            return opennn::LossIndex::RegularizationMethod::L1;
+            break;
+        case L2:
+            return opennn::LossIndex::RegularizationMethod::L2;
+            break;
+        case NONE:
+        default:
+            return opennn::LossIndex::RegularizationMethod::NoRegularization;
+            break;
+        }
+    }
 
 } // namespace nerlnet

src_cpp/opennnBridge/nerlWorkerOpenNN.h

@@ -21,7 +21,7 @@ class NerlWorkerOpenNN : public NerlWorker
 
     NerlWorkerOpenNN(int model_type, std::string &model_args_str , std::string &layer_sizes_str, std::string &layer_types_list, std::string &layers_functionality,
                      float learning_rate, int epochs, int optimizer_type, std::string &optimizer_args_str,
-                     int loss_method, int distributed_system_type, std::string &distributed_system_args_str);
+                     int loss_method, std::string &loss_args_str, int distributed_system_type, std::string &distributed_system_args_str);
     ~NerlWorkerOpenNN();
 
     void generate_opennn_neural_network();
@@ -82,6 +82,7 @@ class NerlWorkerOpenNN : public NerlWorker
     opennn::Scaler translate_unscaling_method(int scaling_method);
     opennn::PoolingLayer::PoolingMethod translate_pooling_method(int pooling_method);
     opennn::ProbabilisticLayer::ActivationFunction translate_probabilistic_activation_function(int activation_function);
+    opennn::LossIndex::RegularizationMethod parse_regularization_loss_args(const std::string &loss_args_str);
 
     int translate_pooling_method_int(int pooling_method);
     int translate_model_type(int model_type, int &custom_model);

src_cpp/opennnBridge/openNNnif.h

@@ -164,8 +164,8 @@ static ErlNifFunc nif_funcs[] =
     {"nerltensor_sum_nif",3, nerltensor_sum_nif},
     {"nerltensor_scalar_multiplication_nif",3,nerltensor_scalar_multiplication_nif},
     // nerlworker functions
-    {"new_nerlworker_nif", 13, new_nerlworker_nif},
-    {"test_nerlworker_nif", 13, test_nerlworker_nif},
+    {"new_nerlworker_nif", 14, new_nerlworker_nif},
+    {"test_nerlworker_nif", 14, test_nerlworker_nif},
     {"update_nerlworker_train_params_nif", 6, update_nerlworker_train_params_nif},
     {"remove_nerlworker_nif", 1, remove_nerlworker_nif},
     {"get_distributed_system_train_labels_count_nif", 1, get_distributed_system_train_labels_count_nif}

src_erl/NerlnetApp/src/Bridge/layers_types_ag.hrl

@@ -1,5 +1,5 @@
 % This is an auto generated .hrl file
-% DC Fields Generated by Nerlplanner version: 1.0.3
+% DC Fields Generated by Nerlplanner version: 1.0.4
 
 -define(LAYERS_TYPE_DEFAULT_IDX,"0").
 -define(LAYERS_TYPE_SCALING_IDX,"1").

src_erl/NerlnetApp/src/Bridge/models_types_ag.hrl

@@ -1,5 +1,5 @@
 % This is an auto generated .hrl file
-% DC Fields Generated by Nerlplanner version: 1.0.3
+% DC Fields Generated by Nerlplanner version: 1.0.4
 
 -define(MODEL_TYPE_NN_IDX,"0").
 -define(MODEL_TYPE_APPROXIMATION_IDX,"1").

src_erl/NerlnetApp/src/Bridge/onnWorkers/nerlNIF.erl

@@ -20,7 +20,7 @@
 -export([nerltensor_scalar_multiplication_nif/3, nerltensor_scalar_multiplication_erl/2]).
 
 % nerlworker nif methods
--export([new_nerlworker_nif/13, remove_nerlworker_nif/1, test_nerlworker_nif/13,get_distributed_system_train_labels_count_nif/1]).
+-export([new_nerlworker_nif/14, remove_nerlworker_nif/1, test_nerlworker_nif/14,get_distributed_system_train_labels_count_nif/1]).
 
 init() ->
       NELNET_LIB_PATH = ?NERLNET_PATH++?BUILD_TYPE_RELEASE++"/"++?NERLNET_LIB,
@@ -221,15 +221,15 @@ nerltensor_scalar_multiplication_erl({NerlTensorErl, Type}, ScalarValue) ->
 %%%%%% NerlWorker NIF Methods %%%%%%
 
 new_nerlworker_nif(_ModelId,_ModelType, _ModelArgs , _LayersSizes, _LayersTypes, _LayersFunctionalityCodes, _LearningRate, _Epochs, _OptimizerType,
-_OptimizerArgs, _LossMethod, _DistributedSystemType, _DistributedSystemArgs) ->
+_OptimizerArgs, _LossMethod, _LossArgs, _DistributedSystemType, _DistributedSystemArgs) ->
       exit(nif_library_not_loaded).
 
 remove_nerlworker_nif(_ModelId) ->
       exit(nif_library_not_loaded).
 
 %% All of inputs must be binary strings! except for _ModelId which is an integer
 test_nerlworker_nif(_ModelId,_ModelType, _ModelArgs, _LayersSizes, _LayersTypes, _LayersFunctionalityCodes, _LearningRate, _Epochs, _OptimizerType,
-                _OptimizerArgs, _LossMethod, _DistributedSystemType, _DistributedSystemArgs) ->
+                _OptimizerArgs, _LossMethod, _LossArgs, _DistributedSystemType, _DistributedSystemArgs) ->
       exit(nif_library_not_loaded).
 
 %  input -  unsigned long modelId