From 8f6f9833b3e40b04e2cec7030af23ff7d402a402 Mon Sep 17 00:00:00 2001 From: Tom Benson <30674819+benson31@users.noreply.github.com> Date: Tue, 7 Sep 2021 14:32:53 -0400 Subject: [PATCH 01/37] Entrywise operators (#1955) * updates and cleanup to clamp operator * unary/binary math operators; cereal registration; unit tests for abs, add, subtract, multiply, cos, sin * Add more operators: activations and associated infrastructure. * remove old layers; entrywise loss layers to operators * add python classes for all the new operators * Remove complex numbers from the abs test when running under ROCm * Tiny bit of tidying * quick factoring to use the common macro --- include/lbann/layers/loss/entrywise.hpp | 76 --- include/lbann/layers/math/CMakeLists.txt | 2 - include/lbann/layers/math/binary.hpp | 142 ----- include/lbann/layers/math/math_builders.hpp | 49 -- include/lbann/layers/math/unary.hpp | 142 ----- include/lbann/layers/operator_layer_impl.hpp | 3 +- include/lbann/lbann.hpp | 5 +- include/lbann/operators/CMakeLists.txt | 1 + .../operators/activations/CMakeLists.txt | 7 + .../activations/activation_builders.hpp | 23 +- .../activations/activation_builders_impl.hpp | 36 ++ .../activations/activations.hpp | 45 +- include/lbann/operators/builder_macros.hpp | 88 +++ .../lbann/operators/declare_stateless_op.hpp | 125 ++++ .../lbann/operators/elementwise_operator.hpp | 8 +- include/lbann/operators/loss/CMakeLists.txt | 9 + .../lbann/operators/loss/entrywise.hpp | 34 +- .../lbann/operators/loss/loss_builders.hpp | 28 +- .../operators/loss/loss_builders_impl.hpp | 29 +- include/lbann/operators/math/CMakeLists.txt | 3 + include/lbann/operators/math/abs.hpp | 129 ++++ include/lbann/operators/math/binary.hpp | 63 ++ include/lbann/operators/math/clamp.hpp | 21 +- .../lbann/operators/math/math_builders.hpp | 67 ++- .../operators/math/math_builders_impl.hpp | 78 ++- include/lbann/operators/math/unary.hpp | 83 +++ include/lbann/operators/operator.hpp | 38 +- include/lbann/proto/datatype_helpers.hpp | 12 + include/lbann/proto/operator_factory_impl.hpp | 72 ++- include/lbann/utils/exception.hpp | 6 + include/lbann/utils/tensor.hpp | 6 +- python/lbann/core/operators.py | 531 ++++++++++++++++ .../ltfb/mutation_strategy.cpp | 20 +- src/layers/activations/CMakeLists.txt | 2 - src/layers/activations/activations.cpp | 168 ------ src/layers/activations/activations.cu | 173 ------ .../cereal_registration/CMakeLists.txt | 5 - .../activations/cereal_registration/selu.cpp | 44 -- .../cereal_registration/sigmoid.cpp | 44 -- src/layers/loss/CMakeLists.txt | 2 - .../loss/cereal_registration/CMakeLists.txt | 5 - src/layers/loss/entrywise.cpp | 255 -------- src/layers/math/CMakeLists.txt | 10 - src/layers/math/binary.cpp | 512 ---------------- src/layers/math/binary.cu | 545 ----------------- .../math/cereal_registration/CMakeLists.txt | 49 -- src/layers/math/cereal_registration/add.cpp | 44 -- src/layers/math/cereal_registration/asin.cpp | 44 -- src/layers/math/cereal_registration/asinh.cpp | 44 -- src/layers/math/cereal_registration/atan.cpp | 44 -- src/layers/math/cereal_registration/atanh.cpp | 44 -- src/layers/math/cereal_registration/ceil.cpp | 44 -- src/layers/math/cereal_registration/cosh.cpp | 44 -- .../math/cereal_registration/divide.cpp | 44 -- src/layers/math/cereal_registration/equal.cpp | 44 -- .../math/cereal_registration/erfinv.cpp | 44 -- src/layers/math/cereal_registration/expm1.cpp | 44 -- src/layers/math/cereal_registration/floor.cpp | 44 -- .../math/cereal_registration/greater.cpp | 44 -- .../cereal_registration/greater_equal.cpp | 44 -- src/layers/math/cereal_registration/less.cpp | 44 -- .../math/cereal_registration/less_equal.cpp | 44 -- src/layers/math/cereal_registration/log1p.cpp | 44 -- .../math/cereal_registration/logical_and.cpp | 44 -- .../math/cereal_registration/logical_not.cpp | 44 -- .../math/cereal_registration/logical_or.cpp | 44 -- .../math/cereal_registration/logical_xor.cpp | 44 -- src/layers/math/cereal_registration/max.cpp | 44 -- src/layers/math/cereal_registration/min.cpp | 44 -- src/layers/math/cereal_registration/mod.cpp | 44 -- .../math/cereal_registration/multiply.cpp | 44 -- .../math/cereal_registration/negative.cpp | 44 -- .../math/cereal_registration/not_equal.cpp | 44 -- src/layers/math/cereal_registration/pow.cpp | 44 -- .../math/cereal_registration/reciprocal.cpp | 44 -- src/layers/math/cereal_registration/round.cpp | 44 -- src/layers/math/cereal_registration/rsqrt.cpp | 44 -- .../math/cereal_registration/safe_divide.cpp | 44 -- .../cereal_registration/safe_reciprocal.cpp | 44 -- src/layers/math/cereal_registration/sign.cpp | 44 -- src/layers/math/cereal_registration/sinh.cpp | 44 -- src/layers/math/cereal_registration/sqrt.cpp | 44 -- .../math/cereal_registration/square.cpp | 44 -- .../squared_difference.cpp | 44 -- .../math/cereal_registration/subtract.cpp | 44 -- src/layers/math/cereal_registration/tanh.cpp | 44 -- src/layers/math/math_builders.cpp | 103 +--- src/layers/math/unary.cpp | 514 ---------------- src/layers/math/unary.cu | 481 --------------- src/models/unit_test/modify_test.cpp | 1 - src/operators/CMakeLists.txt | 2 + src/operators/activations/CMakeLists.txt | 18 + .../activations/activation_builders.cpp} | 26 +- src/operators/activations/activations.cpp | 204 +++++++ src/operators/activations/activations.cu | 197 ++++++ .../cereal_registration/CMakeLists.txt | 11 + .../cereal_registration/log_sigmoid.cpp | 31 + .../activations/cereal_registration/selu.cpp | 31 + .../cereal_registration/sigmoid.cpp | 31 + .../cereal_registration/softplus.cpp | 31 + .../cereal_registration/softsign.cpp | 31 + src/operators/loss/CMakeLists.txt | 18 + .../loss/cereal_registration/CMakeLists.txt | 11 + .../binary_cross_entropy.cpp | 31 + .../cereal_registration/boolean_accuracy.cpp | 31 + .../boolean_false_negative.cpp | 31 + .../boolean_false_positive.cpp | 31 + .../sigmoid_binary_cross_entropy.cpp | 31 + src/operators/loss/entrywise.cpp | 246 ++++++++ src/operators/loss/entrywise.cu | 243 ++++++++ .../loss/loss_builders.cpp} | 26 +- src/operators/math/CMakeLists.txt | 10 + src/operators/math/abs.cpp | 114 ++++ src/operators/math/abs.cu | 108 ++++ src/operators/math/binary.cpp | 543 +++++++++++++++++ src/operators/math/binary.cu | 510 ++++++++++++++++ .../math/cereal_registration/CMakeLists.txt | 49 ++ .../math/cereal_registration/abs.cpp | 59 ++ .../math/cereal_registration/acos.cpp | 31 + .../math/cereal_registration/acosh.cpp | 31 + .../math/cereal_registration/add.cpp | 31 + .../math/cereal_registration/asin.cpp | 31 + .../math/cereal_registration/asinh.cpp | 31 + .../math/cereal_registration/atan.cpp | 31 + .../math/cereal_registration/atanh.cpp | 31 + .../math/cereal_registration/ceil.cpp | 31 + .../math/cereal_registration/clamp.cpp | 2 +- .../math/cereal_registration/cos.cpp | 23 +- .../math/cereal_registration/cosh.cpp | 31 + .../math/cereal_registration/divide.cpp | 31 + .../math/cereal_registration/equal.cpp | 31 + .../math/cereal_registration/erf.cpp | 23 +- .../math/cereal_registration/erfinv.cpp | 31 + .../math/cereal_registration/exp.cpp | 23 +- .../math/cereal_registration/expm1.cpp | 31 + .../math/cereal_registration/floor.cpp | 31 + .../math/cereal_registration/greater.cpp | 31 + .../cereal_registration/greater_equal.cpp | 31 + .../math/cereal_registration/less.cpp | 31 + .../math/cereal_registration/less_equal.cpp | 31 + .../math/cereal_registration/log.cpp | 23 +- .../math/cereal_registration/log1p.cpp | 31 + .../math/cereal_registration/logical_and.cpp | 31 + .../math/cereal_registration/logical_not.cpp | 31 + .../math/cereal_registration/logical_or.cpp | 31 + .../math/cereal_registration/logical_xor.cpp | 31 + .../math/cereal_registration/max.cpp | 31 + .../math/cereal_registration/min.cpp | 31 + .../math/cereal_registration/mod.cpp | 31 + .../math/cereal_registration/multiply.cpp | 31 + .../math/cereal_registration/negative.cpp | 31 + .../math/cereal_registration/not_equal.cpp | 31 + .../math/cereal_registration/pow.cpp | 31 + .../math/cereal_registration/reciprocal.cpp | 31 + .../math/cereal_registration/round.cpp | 31 + .../math/cereal_registration/rsqrt.cpp | 31 + .../math/cereal_registration/safe_divide.cpp | 31 + .../cereal_registration/safe_reciprocal.cpp | 31 + .../math/cereal_registration/sign.cpp | 31 + .../math/cereal_registration/sin.cpp | 23 +- .../math/cereal_registration/sinh.cpp | 31 + .../math/cereal_registration/sqrt.cpp | 31 + .../math/cereal_registration/square.cpp | 31 + .../squared_difference.cpp | 31 + .../math/cereal_registration/subtract.cpp | 31 + .../math/cereal_registration/tan.cpp | 23 +- .../math/cereal_registration/tanh.cpp | 31 + src/operators/math/clamp.cpp | 78 +-- src/operators/math/common.cuh | 240 ++++++++ src/operators/math/common.hpp | 122 ++++ src/operators/math/math_builders.cpp | 69 ++- src/operators/math/unary.cpp | 567 ++++++++++++++++++ src/operators/math/unary.cu | 486 +++++++++++++++ src/operators/math/unit_test/CMakeLists.txt | 6 + .../math/unit_test/OperatorTraits.hpp | 100 +++ src/operators/math/unit_test/abs_test.cpp | 313 ++++++++++ src/operators/math/unit_test/add_test.cpp | 308 ++++++++++ src/operators/math/unit_test/clamp_test.cpp | 118 ++-- src/operators/math/unit_test/cos_test.cpp | 321 ++++++++++ .../math/unit_test/multiply_test.cpp | 310 ++++++++++ src/operators/math/unit_test/sin_test.cpp | 319 ++++++++++ .../math/unit_test/subtract_test.cpp | 312 ++++++++++ src/proto/datatype.proto | 2 + src/proto/factories/layer_factory.cpp | 63 +- src/proto/layers.proto | 139 +---- src/proto/operators.proto | 98 +++ unit_test/utilities/TestHelpers.hpp | 6 + 187 files changed, 9067 insertions(+), 5711 deletions(-) delete mode 100644 include/lbann/layers/loss/entrywise.hpp delete mode 100644 include/lbann/layers/math/binary.hpp delete mode 100644 include/lbann/layers/math/unary.hpp create mode 100644 include/lbann/operators/activations/CMakeLists.txt rename src/layers/math/cereal_registration/abs.cpp => include/lbann/operators/activations/activation_builders.hpp (66%) create mode 100644 include/lbann/operators/activations/activation_builders_impl.hpp rename include/lbann/{layers => operators}/activations/activations.hpp (63%) create mode 100644 include/lbann/operators/builder_macros.hpp create mode 100644 include/lbann/operators/declare_stateless_op.hpp create mode 100644 include/lbann/operators/loss/CMakeLists.txt rename src/layers/math/cereal_registration/acosh.cpp => include/lbann/operators/loss/entrywise.hpp (52%) rename src/layers/activations/cereal_registration/log_sigmoid.cpp => include/lbann/operators/loss/loss_builders.hpp (64%) rename src/layers/math/cereal_registration/acos.cpp => include/lbann/operators/loss/loss_builders_impl.hpp (58%) create mode 100644 include/lbann/operators/math/abs.hpp create mode 100644 include/lbann/operators/math/binary.hpp create mode 100644 include/lbann/operators/math/unary.hpp delete mode 100644 src/layers/activations/activations.cpp delete mode 100644 src/layers/activations/activations.cu delete mode 100644 src/layers/activations/cereal_registration/selu.cpp delete mode 100644 src/layers/activations/cereal_registration/sigmoid.cpp delete mode 100644 src/layers/loss/entrywise.cpp delete mode 100644 src/layers/math/binary.cpp delete mode 100644 src/layers/math/binary.cu delete mode 100644 src/layers/math/cereal_registration/add.cpp delete mode 100644 src/layers/math/cereal_registration/asin.cpp delete mode 100644 src/layers/math/cereal_registration/asinh.cpp delete mode 100644 src/layers/math/cereal_registration/atan.cpp delete mode 100644 src/layers/math/cereal_registration/atanh.cpp delete mode 100644 src/layers/math/cereal_registration/ceil.cpp delete mode 100644 src/layers/math/cereal_registration/cosh.cpp delete mode 100644 src/layers/math/cereal_registration/divide.cpp delete mode 100644 src/layers/math/cereal_registration/equal.cpp delete mode 100644 src/layers/math/cereal_registration/erfinv.cpp delete mode 100644 src/layers/math/cereal_registration/expm1.cpp delete mode 100644 src/layers/math/cereal_registration/floor.cpp delete mode 100644 src/layers/math/cereal_registration/greater.cpp delete mode 100644 src/layers/math/cereal_registration/greater_equal.cpp delete mode 100644 src/layers/math/cereal_registration/less.cpp delete mode 100644 src/layers/math/cereal_registration/less_equal.cpp delete mode 100644 src/layers/math/cereal_registration/log1p.cpp delete mode 100644 src/layers/math/cereal_registration/logical_and.cpp delete mode 100644 src/layers/math/cereal_registration/logical_not.cpp delete mode 100644 src/layers/math/cereal_registration/logical_or.cpp delete mode 100644 src/layers/math/cereal_registration/logical_xor.cpp delete mode 100644 src/layers/math/cereal_registration/max.cpp delete mode 100644 src/layers/math/cereal_registration/min.cpp delete mode 100644 src/layers/math/cereal_registration/mod.cpp delete mode 100644 src/layers/math/cereal_registration/multiply.cpp delete mode 100644 src/layers/math/cereal_registration/negative.cpp delete mode 100644 src/layers/math/cereal_registration/not_equal.cpp delete mode 100644 src/layers/math/cereal_registration/pow.cpp delete mode 100644 src/layers/math/cereal_registration/reciprocal.cpp delete mode 100644 src/layers/math/cereal_registration/round.cpp delete mode 100644 src/layers/math/cereal_registration/rsqrt.cpp delete mode 100644 src/layers/math/cereal_registration/safe_divide.cpp delete mode 100644 src/layers/math/cereal_registration/safe_reciprocal.cpp delete mode 100644 src/layers/math/cereal_registration/sign.cpp delete mode 100644 src/layers/math/cereal_registration/sinh.cpp delete mode 100644 src/layers/math/cereal_registration/sqrt.cpp delete mode 100644 src/layers/math/cereal_registration/square.cpp delete mode 100644 src/layers/math/cereal_registration/squared_difference.cpp delete mode 100644 src/layers/math/cereal_registration/subtract.cpp delete mode 100644 src/layers/math/cereal_registration/tanh.cpp delete mode 100644 src/layers/math/unary.cpp delete mode 100644 src/layers/math/unary.cu create mode 100644 src/operators/activations/CMakeLists.txt rename src/{layers/activations/cereal_registration/softsign.cpp => operators/activations/activation_builders.cpp} (63%) create mode 100644 src/operators/activations/activations.cpp create mode 100644 src/operators/activations/activations.cu create mode 100644 src/operators/activations/cereal_registration/CMakeLists.txt create mode 100644 src/operators/activations/cereal_registration/log_sigmoid.cpp create mode 100644 src/operators/activations/cereal_registration/selu.cpp create mode 100644 src/operators/activations/cereal_registration/sigmoid.cpp create mode 100644 src/operators/activations/cereal_registration/softplus.cpp create mode 100644 src/operators/activations/cereal_registration/softsign.cpp create mode 100644 src/operators/loss/CMakeLists.txt create mode 100644 src/operators/loss/cereal_registration/CMakeLists.txt create mode 100644 src/operators/loss/cereal_registration/binary_cross_entropy.cpp create mode 100644 src/operators/loss/cereal_registration/boolean_accuracy.cpp create mode 100644 src/operators/loss/cereal_registration/boolean_false_negative.cpp create mode 100644 src/operators/loss/cereal_registration/boolean_false_positive.cpp create mode 100644 src/operators/loss/cereal_registration/sigmoid_binary_cross_entropy.cpp create mode 100644 src/operators/loss/entrywise.cpp create mode 100644 src/operators/loss/entrywise.cu rename src/{layers/activations/cereal_registration/softplus.cpp => operators/loss/loss_builders.cpp} (63%) create mode 100644 src/operators/math/abs.cpp create mode 100644 src/operators/math/abs.cu create mode 100644 src/operators/math/binary.cpp create mode 100644 src/operators/math/binary.cu create mode 100644 src/operators/math/cereal_registration/abs.cpp create mode 100644 src/operators/math/cereal_registration/acos.cpp create mode 100644 src/operators/math/cereal_registration/acosh.cpp create mode 100644 src/operators/math/cereal_registration/add.cpp create mode 100644 src/operators/math/cereal_registration/asin.cpp create mode 100644 src/operators/math/cereal_registration/asinh.cpp create mode 100644 src/operators/math/cereal_registration/atan.cpp create mode 100644 src/operators/math/cereal_registration/atanh.cpp create mode 100644 src/operators/math/cereal_registration/ceil.cpp rename src/{layers => operators}/math/cereal_registration/cos.cpp (66%) create mode 100644 src/operators/math/cereal_registration/cosh.cpp create mode 100644 src/operators/math/cereal_registration/divide.cpp create mode 100644 src/operators/math/cereal_registration/equal.cpp rename src/{layers => operators}/math/cereal_registration/erf.cpp (66%) create mode 100644 src/operators/math/cereal_registration/erfinv.cpp rename src/{layers => operators}/math/cereal_registration/exp.cpp (66%) create mode 100644 src/operators/math/cereal_registration/expm1.cpp create mode 100644 src/operators/math/cereal_registration/floor.cpp create mode 100644 src/operators/math/cereal_registration/greater.cpp create mode 100644 src/operators/math/cereal_registration/greater_equal.cpp create mode 100644 src/operators/math/cereal_registration/less.cpp create mode 100644 src/operators/math/cereal_registration/less_equal.cpp rename src/{layers => operators}/math/cereal_registration/log.cpp (66%) create mode 100644 src/operators/math/cereal_registration/log1p.cpp create mode 100644 src/operators/math/cereal_registration/logical_and.cpp create mode 100644 src/operators/math/cereal_registration/logical_not.cpp create mode 100644 src/operators/math/cereal_registration/logical_or.cpp create mode 100644 src/operators/math/cereal_registration/logical_xor.cpp create mode 100644 src/operators/math/cereal_registration/max.cpp create mode 100644 src/operators/math/cereal_registration/min.cpp create mode 100644 src/operators/math/cereal_registration/mod.cpp create mode 100644 src/operators/math/cereal_registration/multiply.cpp create mode 100644 src/operators/math/cereal_registration/negative.cpp create mode 100644 src/operators/math/cereal_registration/not_equal.cpp create mode 100644 src/operators/math/cereal_registration/pow.cpp create mode 100644 src/operators/math/cereal_registration/reciprocal.cpp create mode 100644 src/operators/math/cereal_registration/round.cpp create mode 100644 src/operators/math/cereal_registration/rsqrt.cpp create mode 100644 src/operators/math/cereal_registration/safe_divide.cpp create mode 100644 src/operators/math/cereal_registration/safe_reciprocal.cpp create mode 100644 src/operators/math/cereal_registration/sign.cpp rename src/{layers => operators}/math/cereal_registration/sin.cpp (66%) create mode 100644 src/operators/math/cereal_registration/sinh.cpp create mode 100644 src/operators/math/cereal_registration/sqrt.cpp create mode 100644 src/operators/math/cereal_registration/square.cpp create mode 100644 src/operators/math/cereal_registration/squared_difference.cpp create mode 100644 src/operators/math/cereal_registration/subtract.cpp rename src/{layers => operators}/math/cereal_registration/tan.cpp (66%) create mode 100644 src/operators/math/cereal_registration/tanh.cpp create mode 100644 src/operators/math/common.cuh create mode 100644 src/operators/math/common.hpp create mode 100644 src/operators/math/unary.cpp create mode 100644 src/operators/math/unary.cu create mode 100644 src/operators/math/unit_test/OperatorTraits.hpp create mode 100644 src/operators/math/unit_test/abs_test.cpp create mode 100644 src/operators/math/unit_test/add_test.cpp create mode 100644 src/operators/math/unit_test/cos_test.cpp create mode 100644 src/operators/math/unit_test/multiply_test.cpp create mode 100644 src/operators/math/unit_test/sin_test.cpp create mode 100644 src/operators/math/unit_test/subtract_test.cpp diff --git a/include/lbann/layers/loss/entrywise.hpp b/include/lbann/layers/loss/entrywise.hpp deleted file mode 100644 index 6bb9ab1b15f..00000000000 --- a/include/lbann/layers/loss/entrywise.hpp +++ /dev/null @@ -1,76 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// - -#ifndef LBANN_LAYERS_LOSS_ENTRYWISE_HPP_INCLUDED -#define LBANN_LAYERS_LOSS_ENTRYWISE_HPP_INCLUDED - -#include "lbann/layers/math/binary.hpp" - -namespace lbann { - -#ifndef LBANN_ENTRYWISE_LAYER_INSTANTIATE -#define BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, DEVICE) \ - extern template class LAYER_NAME; \ - extern template class LAYER_NAME -#else -#define BINARY_ETI_DECL_MACRO_DEV(...) -#endif // LBANN_BINARY_LAYER_INSTANTIATE - -#ifdef LBANN_HAS_GPU -#define BINARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU); \ - BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::GPU) -#else -#define BINARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU) -#endif // LBANN_HAS_GPU - -// Convenience macro to define an entry-wise binary layer class -#define DEFINE_ENTRYWISE_BINARY_LAYER(layer_name, layer_string) \ - LBANN_DECLARE_ENTRYWISE_BINARY_LAYER(layer_name, layer_string); \ - BINARY_ETI_DECL_MACRO(layer_name, float); \ - BINARY_ETI_DECL_MACRO(layer_name, double) - -// Cross entropy loss -DEFINE_ENTRYWISE_BINARY_LAYER(binary_cross_entropy_layer, - "binary cross entropy"); -DEFINE_ENTRYWISE_BINARY_LAYER(sigmoid_binary_cross_entropy_layer, - "sigmoid binary cross entropy"); - -// Boolean loss functions -DEFINE_ENTRYWISE_BINARY_LAYER(boolean_accuracy_layer, "Boolean accuracy"); -DEFINE_ENTRYWISE_BINARY_LAYER(boolean_false_negative_layer, - "Boolean false negative rate"); -DEFINE_ENTRYWISE_BINARY_LAYER(boolean_false_positive_layer, - "Boolean false positive rate"); - -} // namespace lbann - -#undef DEFINE_ENTRYWISE_BINARY_LAYER -#undef BINARY_ETI_DECL_MACRO -#undef BINARY_ETI_DECL_MACRO_DEV - -#endif // LBANN_LAYERS_LOSS_ENTRYWISE_HPP_INCLUDED diff --git a/include/lbann/layers/math/CMakeLists.txt b/include/lbann/layers/math/CMakeLists.txt index 62239b4eb22..dedbae16d91 100644 --- a/include/lbann/layers/math/CMakeLists.txt +++ b/include/lbann/layers/math/CMakeLists.txt @@ -1,7 +1,5 @@ # Add the headers for this directory set_full_path(THIS_DIR_HEADERS - unary.hpp - binary.hpp matmul.hpp ) diff --git a/include/lbann/layers/math/binary.hpp b/include/lbann/layers/math/binary.hpp deleted file mode 100644 index 731d600db35..00000000000 --- a/include/lbann/layers/math/binary.hpp +++ /dev/null @@ -1,142 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// - -#ifndef LBANN_LAYERS_MATH_BINARY_HPP_INCLUDED -#define LBANN_LAYERS_MATH_BINARY_HPP_INCLUDED - -#include "lbann/layers/data_type_layer.hpp" - -namespace lbann { - -#define LBANN_DECLARE_ENTRYWISE_BINARY_LAYER(LAYER_NAME, LAYER_STRING) \ - template \ - class LAYER_NAME : public data_type_layer { \ - public: \ - LAYER_NAME(lbann_comm *comm) : data_type_layer(comm) { \ - this->m_expected_num_parent_layers = 2; \ - } \ - LAYER_NAME() : LAYER_NAME(nullptr) {} \ - LAYER_NAME* copy() const override { \ - return new LAYER_NAME(*this); \ - } \ - std::string get_type() const override { return LAYER_STRING; } \ - data_layout get_data_layout() const override { return Layout; } \ - El::Device get_device_allocation() const override { return Device; } \ - template \ - void serialize(ArchiveT& ar); \ - protected: \ - void setup_dims(DataReaderMetaData& dr_metadata) override { \ - data_type_layer::setup_dims(dr_metadata); \ - this->set_output_dims(this->get_input_dims()); \ - /* Check that input dimensions match */ \ - if (this->get_input_dims(0) != this->get_input_dims(1)) { \ - const auto& parents = this->get_parent_layers(); \ - std::stringstream err; \ - err << this->get_type() << " layer \"" << this->get_name() << "\" " \ - << "has input tensors with different dimensions ("; \ - for (int i = 0; i < this->get_num_parents(); ++i) { \ - const auto& dims = this->get_input_dims(i); \ - err << (i > 0 ? ", " : "") \ - << "layer \"" << parents[i]->get_name() << "\" outputs "; \ - for (size_t j = 0; j < dims.size(); ++j) { \ - err << (j > 0 ? " x " : "") << dims[j]; \ - } \ - } \ - err << ")"; \ - LBANN_ERROR(err.str()); \ - } \ - } \ - void fp_compute() override; \ - void bp_compute() override; \ - } - -// Convenience macros for ETI decls for binary layers - -#ifndef LBANN_BINARY_LAYER_INSTANTIATE -#define BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, DEVICE) \ - extern template class LAYER_NAME; \ - extern template class LAYER_NAME -#else -#define BINARY_ETI_DECL_MACRO_DEV(...) -#endif // LBANN_BINARY_LAYER_INSTANTIATE - -// Instnatiate both data and model parallel layers -#define BINARY_ETI_INST_MACRO_DEV_DT(LAYER_NAME, T, DEVICE) \ - template class LAYER_NAME; \ - template class LAYER_NAME - -// Instantiate a DEVICE for each allowed tensor data type -#define BINARY_ETI_INST_MACRO_DEV(LAYER_NAME, DEVICE) \ - BINARY_ETI_INST_MACRO_DEV_DT(LAYER_NAME, float, DEVICE); \ - BINARY_ETI_INST_MACRO_DEV_DT(LAYER_NAME, double, DEVICE) - -#ifdef LBANN_HAS_GPU -#define BINARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU); \ - BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::GPU) -#else -#define BINARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - BINARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU) -#endif // LBANN_HAS_GPU - -// Convenience macro to define an entry-wise binary layer class -#define DEFINE_ENTRYWISE_BINARY_LAYER(layer_name, layer_string) \ - LBANN_DECLARE_ENTRYWISE_BINARY_LAYER(layer_name, layer_string); \ - BINARY_ETI_DECL_MACRO(layer_name, float); \ - BINARY_ETI_DECL_MACRO(layer_name, double) - -// Arithmetic operations -DEFINE_ENTRYWISE_BINARY_LAYER(add_layer, "add"); -DEFINE_ENTRYWISE_BINARY_LAYER(subtract_layer, "subtract"); -DEFINE_ENTRYWISE_BINARY_LAYER(multiply_layer, "multiply"); -DEFINE_ENTRYWISE_BINARY_LAYER(divide_layer, "divide"); -DEFINE_ENTRYWISE_BINARY_LAYER(mod_layer, "modulo"); -DEFINE_ENTRYWISE_BINARY_LAYER(pow_layer, "power"); -DEFINE_ENTRYWISE_BINARY_LAYER(safe_divide_layer, "safe divide"); -DEFINE_ENTRYWISE_BINARY_LAYER(squared_difference_layer, "squared difference"); - -// Comparison operations -DEFINE_ENTRYWISE_BINARY_LAYER(max_layer, "maximum"); -DEFINE_ENTRYWISE_BINARY_LAYER(min_layer, "minimum"); -DEFINE_ENTRYWISE_BINARY_LAYER(equal_layer, "equal"); -DEFINE_ENTRYWISE_BINARY_LAYER(not_equal_layer, "not equal"); -DEFINE_ENTRYWISE_BINARY_LAYER(less_layer, "less than"); -DEFINE_ENTRYWISE_BINARY_LAYER(less_equal_layer, "less than or equal"); -DEFINE_ENTRYWISE_BINARY_LAYER(greater_layer, "greater than"); -DEFINE_ENTRYWISE_BINARY_LAYER(greater_equal_layer, "greater than or equal"); - -// Logical operations -DEFINE_ENTRYWISE_BINARY_LAYER(logical_and_layer, "logical and"); -DEFINE_ENTRYWISE_BINARY_LAYER(logical_or_layer, "logical or"); -DEFINE_ENTRYWISE_BINARY_LAYER(logical_xor_layer, "logical xor"); - -} // namespace lbann - -#undef DEFINE_ENTRYWISE_BINARY_LAYER -#undef BINARY_ETI_DECL_MACRO -#undef BINARY_ETI_DECL_MACRO_DEV - -#endif // LBANN_LAYERS_MATH_BINARY_HPP_INCLUDED diff --git a/include/lbann/layers/math/math_builders.hpp b/include/lbann/layers/math/math_builders.hpp index 6d81bad4bea..eb476b163cc 100644 --- a/include/lbann/layers/math/math_builders.hpp +++ b/include/lbann/layers/math/math_builders.hpp @@ -29,55 +29,6 @@ namespace lbann { -LBANN_DEFINE_LAYER_BUILDER(abs); -LBANN_DEFINE_LAYER_BUILDER(acos); -LBANN_DEFINE_LAYER_BUILDER(acosh); -LBANN_DEFINE_LAYER_BUILDER(add); -LBANN_DEFINE_LAYER_BUILDER(asin); -LBANN_DEFINE_LAYER_BUILDER(asinh); -LBANN_DEFINE_LAYER_BUILDER(atan); -LBANN_DEFINE_LAYER_BUILDER(atanh); -LBANN_DEFINE_LAYER_BUILDER(ceil); -LBANN_DEFINE_LAYER_BUILDER(cos); -LBANN_DEFINE_LAYER_BUILDER(cosh); -LBANN_DEFINE_LAYER_BUILDER(divide); -LBANN_DEFINE_LAYER_BUILDER(equal); -LBANN_DEFINE_LAYER_BUILDER(exp); -LBANN_DEFINE_LAYER_BUILDER(expm1); -LBANN_DEFINE_LAYER_BUILDER(floor); -LBANN_DEFINE_LAYER_BUILDER(greater); -LBANN_DEFINE_LAYER_BUILDER(greater_equal); -LBANN_DEFINE_LAYER_BUILDER(erf); -LBANN_DEFINE_LAYER_BUILDER(erfinv); -LBANN_DEFINE_LAYER_BUILDER(less); -LBANN_DEFINE_LAYER_BUILDER(less_equal); -LBANN_DEFINE_LAYER_BUILDER(log); -LBANN_DEFINE_LAYER_BUILDER(log1p); -LBANN_DEFINE_LAYER_BUILDER(logical_and); -LBANN_DEFINE_LAYER_BUILDER(logical_not); -LBANN_DEFINE_LAYER_BUILDER(logical_or); -LBANN_DEFINE_LAYER_BUILDER(logical_xor); LBANN_DEFINE_LAYER_BUILDER(matmul); -LBANN_DEFINE_LAYER_BUILDER(max); -LBANN_DEFINE_LAYER_BUILDER(min); -LBANN_DEFINE_LAYER_BUILDER(mod); -LBANN_DEFINE_LAYER_BUILDER(multiply); -LBANN_DEFINE_LAYER_BUILDER(negative); -LBANN_DEFINE_LAYER_BUILDER(not_equal); -LBANN_DEFINE_LAYER_BUILDER(pow); -LBANN_DEFINE_LAYER_BUILDER(reciprocal); -LBANN_DEFINE_LAYER_BUILDER(round); -LBANN_DEFINE_LAYER_BUILDER(rsqrt); -LBANN_DEFINE_LAYER_BUILDER(safe_divide); -LBANN_DEFINE_LAYER_BUILDER(safe_reciprocal); -LBANN_DEFINE_LAYER_BUILDER(sign); -LBANN_DEFINE_LAYER_BUILDER(sin); -LBANN_DEFINE_LAYER_BUILDER(sinh); -LBANN_DEFINE_LAYER_BUILDER(sqrt); -LBANN_DEFINE_LAYER_BUILDER(square); -LBANN_DEFINE_LAYER_BUILDER(squared_difference); -LBANN_DEFINE_LAYER_BUILDER(subtract); -LBANN_DEFINE_LAYER_BUILDER(tan); -LBANN_DEFINE_LAYER_BUILDER(tanh); }// namespace lbann diff --git a/include/lbann/layers/math/unary.hpp b/include/lbann/layers/math/unary.hpp deleted file mode 100644 index 6a7fc4f1382..00000000000 --- a/include/lbann/layers/math/unary.hpp +++ /dev/null @@ -1,142 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// - -#ifndef LBANN_LAYERS_MATH_UNARY_HPP_INCLUDED -#define LBANN_LAYERS_MATH_UNARY_HPP_INCLUDED - -#include "lbann/layers/data_type_layer.hpp" - -namespace lbann { - -#define LBANN_DECLARE_ENTRYWISE_UNARY_LAYER(LAYER_NAME, LAYER_STRING) \ - template \ - class LAYER_NAME : public data_type_layer { \ - public: \ - LAYER_NAME(lbann_comm *comm) : data_type_layer(comm) {} \ - LAYER_NAME() : LAYER_NAME(nullptr) {} \ - LAYER_NAME* copy() const override { \ - return new LAYER_NAME(*this); \ - } \ - std::string get_type() const override { return LAYER_STRING; } \ - data_layout get_data_layout() const override { return Layout; } \ - El::Device get_device_allocation() const override { return Device; } \ - template \ - void serialize(ArchiveT& ar); \ - protected: \ - void setup_dims(DataReaderMetaData& dr_metadata) override { \ - data_type_layer::setup_dims(dr_metadata); \ - this->set_output_dims(this->get_input_dims()); \ - } \ - void fp_compute() override; \ - void bp_compute() override; \ - } - -// Convenience macros for ETI decls for unary layers - -#ifndef LBANN_UNARY_LAYER_INSTANTIATE -#define UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, DEVICE) \ - extern template class LAYER_NAME; \ - extern template class LAYER_NAME -#else -#define UNARY_ETI_DECL_MACRO_DEV(...) -#endif // LBANN_UNARY_LAYER_INSTANTIATE - -#define UNARY_ETI_INST_MACRO_DEV_DT(LAYER_NAME, T, DEVICE) \ - template class LAYER_NAME; \ - template class LAYER_NAME - -#define UNARY_ETI_INST_MACRO_DEV(LAYER_NAME, DEVICE) \ - UNARY_ETI_INST_MACRO_DEV_DT(LAYER_NAME, float, DEVICE); \ - UNARY_ETI_INST_MACRO_DEV_DT(LAYER_NAME, double, DEVICE) - -#ifdef LBANN_HAS_GPU -#define UNARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU); \ - UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::GPU) -#else -#define UNARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU) -#endif // LBANN_HAS_GPU - -// Convenience macro to define an entry-wise unary layer class -#define DEFINE_ENTRYWISE_UNARY_LAYER(layer_name, layer_string) \ - LBANN_DECLARE_ENTRYWISE_UNARY_LAYER(layer_name, layer_string); \ - UNARY_ETI_DECL_MACRO(layer_name, float); \ - UNARY_ETI_DECL_MACRO(layer_name, double) - -// Logical operations -DEFINE_ENTRYWISE_UNARY_LAYER(logical_not_layer, "logical not"); - -// Sign operations -DEFINE_ENTRYWISE_UNARY_LAYER(abs_layer, "absolute value"); -DEFINE_ENTRYWISE_UNARY_LAYER(negative_layer, "negative"); -DEFINE_ENTRYWISE_UNARY_LAYER(sign_layer, "sign"); - -// Rounding operations -DEFINE_ENTRYWISE_UNARY_LAYER(round_layer, "round"); -DEFINE_ENTRYWISE_UNARY_LAYER(ceil_layer, "ceil"); -DEFINE_ENTRYWISE_UNARY_LAYER(floor_layer, "floor"); - -// Power operations -DEFINE_ENTRYWISE_UNARY_LAYER(reciprocal_layer, "reciprocal"); -DEFINE_ENTRYWISE_UNARY_LAYER(square_layer, "square"); -DEFINE_ENTRYWISE_UNARY_LAYER(sqrt_layer, "square root"); -DEFINE_ENTRYWISE_UNARY_LAYER(rsqrt_layer, "reciprocal square root"); -DEFINE_ENTRYWISE_UNARY_LAYER(safe_reciprocal_layer, "safe reciprocal"); - -// Exponential and logarithmic operations -DEFINE_ENTRYWISE_UNARY_LAYER(exp_layer, "exponential"); -DEFINE_ENTRYWISE_UNARY_LAYER(expm1_layer, "expm1"); -DEFINE_ENTRYWISE_UNARY_LAYER(log_layer, "natural logarithm"); -DEFINE_ENTRYWISE_UNARY_LAYER(log1p_layer, "log1p"); - -// Trigonometric operations -DEFINE_ENTRYWISE_UNARY_LAYER(cos_layer, "cosine"); -DEFINE_ENTRYWISE_UNARY_LAYER(sin_layer, "sine"); -DEFINE_ENTRYWISE_UNARY_LAYER(tan_layer, "tangent"); -DEFINE_ENTRYWISE_UNARY_LAYER(acos_layer, "arccosine"); -DEFINE_ENTRYWISE_UNARY_LAYER(asin_layer, "arcsine"); -DEFINE_ENTRYWISE_UNARY_LAYER(atan_layer, "arctangent"); - -// Hyperbolic operations -DEFINE_ENTRYWISE_UNARY_LAYER(cosh_layer, "hyperbolic cosine"); -DEFINE_ENTRYWISE_UNARY_LAYER(sinh_layer, "hyperbolic sine"); -DEFINE_ENTRYWISE_UNARY_LAYER(tanh_layer, "hyperbolic tangent"); -DEFINE_ENTRYWISE_UNARY_LAYER(acosh_layer, "hyperbolic arccosine"); -DEFINE_ENTRYWISE_UNARY_LAYER(asinh_layer, "hyperbolic arcsine"); -DEFINE_ENTRYWISE_UNARY_LAYER(atanh_layer, "hyperbolic arctangent"); - -// Error function -DEFINE_ENTRYWISE_UNARY_LAYER(erf_layer, "error function"); -DEFINE_ENTRYWISE_UNARY_LAYER(erfinv_layer, "inverse error function"); - -} // namespace lbann - -#undef DEFINE_ENTRYWISE_UNARY_LAYER -#undef UNARY_ETI_DECL_MACRO -#undef UNARY_ETI_DECL_MACRO_DEV - -#endif // LBANN_LAYERS_MATH_UNARY_HPP_INCLUDED diff --git a/include/lbann/layers/operator_layer_impl.hpp b/include/lbann/layers/operator_layer_impl.hpp index 49471e43406..23cf2f5a6c6 100644 --- a/include/lbann/layers/operator_layer_impl.hpp +++ b/include/lbann/layers/operator_layer_impl.hpp @@ -48,6 +48,7 @@ OperatorLayer::OperatorLayer(lbann_comm& comm, LBANN_ASSERT(op); m_ops.reserve(1); m_ops.emplace_back(std::move(op)); + this->m_expected_num_parent_layers = -1; // No limit on parents } template @@ -58,6 +59,7 @@ OperatorLayer::OperatorLayer( { LBANN_ASSERT(m_ops.size() == 1UL); // For starters. LBANN_ASSERT(m_ops[0]); + this->m_expected_num_parent_layers = -1; // No limit on parents } template @@ -261,5 +263,4 @@ namespace lbann { } // namespace lbann #endif // LBANN_INSTANTIATE_OPERATOR_LAYER - #endif // LBANN_LAYERS_OPERATOR_LAYER_IMPL_HPP_INCLUDED diff --git a/include/lbann/lbann.hpp b/include/lbann/lbann.hpp index fc1e06dd0fb..d67a143bbdd 100644 --- a/include/lbann/lbann.hpp +++ b/include/lbann/lbann.hpp @@ -41,7 +41,6 @@ #include "lbann/models/directed_acyclic_graph.hpp" /// Activation layers -#include "lbann/layers/activations/activations.hpp" #include "lbann/layers/activations/elu.hpp" #include "lbann/layers/activations/identity.hpp" #include "lbann/layers/activations/leaky_relu.hpp" @@ -63,7 +62,6 @@ /// Loss layers #include "lbann/layers/loss/categorical_accuracy.hpp" #include "lbann/layers/loss/cross_entropy.hpp" -#include "lbann/layers/loss/entrywise.hpp" #include "lbann/layers/loss/l1_norm.hpp" #include "lbann/layers/loss/l2_norm2.hpp" #include "lbann/layers/loss/mean_absolute_error.hpp" @@ -71,8 +69,7 @@ #include "lbann/layers/loss/top_k_categorical_accuracy.hpp" /// Math layers -#include "lbann/layers/math/unary.hpp" -#include "lbann/layers/math/binary.hpp" +#include "lbann/layers/math/matmul.hpp" /// Transform layers #include "lbann/layers/transform/reshape.hpp" diff --git a/include/lbann/operators/CMakeLists.txt b/include/lbann/operators/CMakeLists.txt index 09e88768f5d..2ad05f7cd98 100644 --- a/include/lbann/operators/CMakeLists.txt +++ b/include/lbann/operators/CMakeLists.txt @@ -4,6 +4,7 @@ set_full_path(THIS_DIR_HEADERS operator.hpp ) +add_subdirectory(activations) add_subdirectory(math) # Propagate the files up the tree diff --git a/include/lbann/operators/activations/CMakeLists.txt b/include/lbann/operators/activations/CMakeLists.txt new file mode 100644 index 00000000000..d0d8d6449d7 --- /dev/null +++ b/include/lbann/operators/activations/CMakeLists.txt @@ -0,0 +1,7 @@ +# Add the headers for this directory +set_full_path(THIS_DIR_HEADERS + activations.hpp + ) + +# Propagate the files up the tree +set(HEADERS "${HEADERS}" "${THIS_DIR_HEADERS}" PARENT_SCOPE) diff --git a/src/layers/math/cereal_registration/abs.cpp b/include/lbann/operators/activations/activation_builders.hpp similarity index 66% rename from src/layers/math/cereal_registration/abs.cpp rename to include/lbann/operators/activations/activation_builders.hpp index 8df83c1e677..b9ef26f44ee 100644 --- a/src/layers/math/cereal_registration/abs.cpp +++ b/include/lbann/operators/activations/activation_builders.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -23,22 +23,15 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" -#include + +#include "lbann/operators/builder_macros.hpp" namespace lbann { -template -template -void -abs_layer -::serialize(ArchiveT& ar) { - using DataTypeLayer = data_type_layer; - ar(::cereal::make_nvp("DataTypeLayer", - ::cereal::base_class(this))); -} +LBANN_DECLARE_OPERATOR_BUILDER(log_sigmoid); +LBANN_DECLARE_OPERATOR_BUILDER(selu); +LBANN_DECLARE_OPERATOR_BUILDER(sigmoid); +LBANN_DECLARE_OPERATOR_BUILDER(softplus); +LBANN_DECLARE_OPERATOR_BUILDER(softsign); } // namespace lbann - -#define LBANN_LAYER_NAME abs_layer -#include diff --git a/include/lbann/operators/activations/activation_builders_impl.hpp b/include/lbann/operators/activations/activation_builders_impl.hpp new file mode 100644 index 00000000000..c3315764f1d --- /dev/null +++ b/include/lbann/operators/activations/activation_builders_impl.hpp @@ -0,0 +1,36 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// + +#include "lbann/operators/activations/activation_builders.hpp" +#include "lbann/operators/activations/activations.hpp" + +#include "lbann/operators/builder_macros.hpp" + +LBANN_DEFINE_OPERATOR_BUILDER(log_sigmoid, LogSigmoid) +LBANN_DEFINE_OPERATOR_BUILDER(selu, Selu) +LBANN_DEFINE_OPERATOR_BUILDER(sigmoid, Sigmoid) +LBANN_DEFINE_OPERATOR_BUILDER(softplus, Softplus) +LBANN_DEFINE_OPERATOR_BUILDER(softsign, Softsign) diff --git a/include/lbann/layers/activations/activations.hpp b/include/lbann/operators/activations/activations.hpp similarity index 63% rename from include/lbann/layers/activations/activations.hpp rename to include/lbann/operators/activations/activations.hpp index 24d11fade27..d2a046df275 100644 --- a/include/lbann/layers/activations/activations.hpp +++ b/include/lbann/operators/activations/activations.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -24,45 +24,20 @@ // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#ifndef LBANN_LAYERS_ACTIVATIONS_ACTIVATIONS_HPP_INCLUDED -#define LBANN_LAYERS_ACTIVATIONS_ACTIVATIONS_HPP_INCLUDED +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_ACTIVATIONS_ACTIVATIONS_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_ACTIVATIONS_ACTIVATIONS_HPP_INCLUDED -#include "lbann/layers/math/unary.hpp" +#include namespace lbann { -// Convenience macros for ETI decls for unary layers - -#ifndef LBANN_ACTIVATIONS_LAYER_INSTANTIATE -#define UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, DEVICE) \ - extern template class LAYER_NAME; \ - extern template class LAYER_NAME -#else -#define UNARY_ETI_DECL_MACRO_DEV(...) -#endif // LBANN_UNARY_LAYER_INSTANTIATE - -#ifdef LBANN_HAS_GPU -#define UNARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU); \ - UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::GPU) -#else -#define UNARY_ETI_DECL_MACRO(LAYER_NAME, T) \ - UNARY_ETI_DECL_MACRO_DEV(LAYER_NAME, T, El::Device::CPU) -#endif // LBANN_HAS_GPU - -// Convenience macro to define an entry-wise unary layer class -#define DEFINE_ENTRYWISE_UNARY_LAYER(layer_name, layer_string) \ - LBANN_DECLARE_ENTRYWISE_UNARY_LAYER(layer_name, layer_string); \ - UNARY_ETI_DECL_MACRO(layer_name, float); \ - UNARY_ETI_DECL_MACRO(layer_name, double) - /** @class lbann::log_sigmoid_layer * @brief Logarithm of sigmoid function. * * @f[ \log(\sigma(x)) = -\log(1 + e^{-x}) @f] * See https://en.wikipedia.org/wiki/Sigmoid_function. */ -DEFINE_ENTRYWISE_UNARY_LAYER(log_sigmoid_layer, "log sigmoid"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(LogSigmoid, "log sigmoid"); /** @class lbann::selu_layer * @brief Scaled exponential rectified linear unit. @@ -81,7 +56,7 @@ DEFINE_ENTRYWISE_UNARY_LAYER(log_sigmoid_layer, "log sigmoid"); * Hochreiter. "Self-normalizing neural networks." In Advances in * Neural Information Processing Systems, pp. 971-980. 2017. */ -DEFINE_ENTRYWISE_UNARY_LAYER(selu_layer, "SELU"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Selu, "SELU"); /** @class lbann::sigmoid_layer * @brief Special case of logistic function. @@ -89,7 +64,7 @@ DEFINE_ENTRYWISE_UNARY_LAYER(selu_layer, "SELU"); * @f[ \sigma(x) = \frac{1}{1 + e^{-x}} @f] * See https://en.wikipedia.org/wiki/Sigmoid_function. */ -DEFINE_ENTRYWISE_UNARY_LAYER(sigmoid_layer, "sigmoid"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Sigmoid, "sigmoid"); // Sigmoid function output is strictly in (0,1) // Note: Output is in the range [eps,1-eps], where 'eps' is machine // epsilon. This avoids denormalized floats and helps mitigate some @@ -102,14 +77,14 @@ DEFINE_ENTRYWISE_UNARY_LAYER(sigmoid_layer, "sigmoid"); * @f[ \text{softplus}(x) = \log (e^x + 1) @f] * See https://en.wikipedia.org/wiki/Rectifier_(neural_networks) */ -DEFINE_ENTRYWISE_UNARY_LAYER(softplus_layer, "softplus"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Softplus, "softplus"); /** @class lbann::softsign_layer * @brief Smooth approximation to sign function. * * @f[ \text{softsign}(x) = \frac{x}{1 + |x|} @f] */ -DEFINE_ENTRYWISE_UNARY_LAYER(softsign_layer, "softsign"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Softsign, "softsign"); } // namespace lbann @@ -117,4 +92,4 @@ DEFINE_ENTRYWISE_UNARY_LAYER(softsign_layer, "softsign"); #undef UNARY_ETI_DECL_MACRO #undef UNARY_ETI_DECL_MACRO_DEV -#endif // LBANN_LAYERS_ACTIVATIONS_ACTIVATIONS_HPP_INCLUDED +#endif // LBANN_INCLUDE_LBANN_OPERATORS_ACTIVATIONS_ACTIVATIONS_HPP_INCLUDED diff --git a/include/lbann/operators/builder_macros.hpp b/include/lbann/operators/builder_macros.hpp new file mode 100644 index 00000000000..cf2a1520f3a --- /dev/null +++ b/include/lbann/operators/builder_macros.hpp @@ -0,0 +1,88 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_BUILDER_MACROS_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_BUILDER_MACROS_HPP_INCLUDED + +#include "lbann/operators/operator.hpp" +#include "lbann/proto/datatype_helpers.hpp" + +#include + +#include + +// Forward declaration +namespace lbann_data { +class Operator; +} // namespace lbann_data + +namespace lbann { +namespace details { + +template +void AssertConsistentTypeParameters(lbann_data::Operator const& op) +{ + LBANN_ASSERT(proto::ProtoDataType == op.input_datatype()); + LBANN_ASSERT(proto::ProtoDataType == op.output_datatype()); + LBANN_ASSERT(proto::ProtoDevice == + proto::resolve_default_device(op.device_allocation())); +} + +} // namespace details +} // namespace lbann + +/** @brief A utility macro fro adding a builder declaration for a single-type + * operator. + * @note Must be called inside lbann namespace. + */ +#define LBANN_DECLARE_OPERATOR_BUILDER(OP_NAME) \ + template \ + std::unique_ptr> build_##OP_NAME##_operator( \ + lbann_data::Operator const& op) + +/** @brief A utility macro for easily adding a default builder with + * dynamic type-checking assertions. + * + * Type-checking is only done with Debug builds. + * + * @note Must *NOT* be called inside lbann namespace. + */ +#define LBANN_DEFINE_OPERATOR_BUILDER(OP_LOWER, OP_NAME) \ + template \ + std::unique_ptr> \ + lbann::build_##OP_LOWER##_operator(lbann_data::Operator const& op) \ + { \ + details::AssertConsistentTypeParameters(op); \ + return std::make_unique>(); \ + } + +/** @brief A utility macro for easily adding ETI for operator builders + * @note Must *NOT* be called inside lbann namespace. + */ +#define LBANN_SINGLE_TYPE_OPERATOR_BUILDER_ETI(OPERATOR_NAME, T, D) \ + template std::unique_ptr> \ + lbann::build_##OPERATOR_NAME##_operator(lbann_data::Operator const&) + +#endif // LBANN_INCLUDE_LBANN_OPERATORS_BUILDER_MACROS_HPP_INCLUDED diff --git a/include/lbann/operators/declare_stateless_op.hpp b/include/lbann/operators/declare_stateless_op.hpp new file mode 100644 index 00000000000..3d9ab782dc4 --- /dev/null +++ b/include/lbann/operators/declare_stateless_op.hpp @@ -0,0 +1,125 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_DECLARE_STATELESS_OP_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_DECLARE_STATELESS_OP_HPP_INCLUDED + +#include "lbann/operators/elementwise_operator.hpp" +#include "lbann/operators/operator.hpp" +#include "lbann/utils/cloneable.hpp" + +#include + +// These are all single-type operators. + +#define LBANN_DECLARE_STATELESS_OPERATOR(OP_NAME, OP_STRING) \ + template \ + class OP_NAME##Operator final \ + : public Cloneable, Operator> \ + { \ + using BaseType = \ + Cloneable, Operator>; \ + using InputTensorType = typename BaseType::InputTensorType; \ + using OutputTensorType = typename BaseType::OutputTensorType; \ + using ConstInputTensorType = typename BaseType::ConstInputTensorType; \ + using ConstOutputTensorType = typename BaseType::ConstOutputTensorType; \ + \ + public: \ + OP_NAME##Operator() = default; \ + OP_NAME##Operator(OP_NAME##Operator&&) = default; \ + OP_NAME##Operator(OP_NAME##Operator const&) = default; \ + OP_NAME##Operator& operator=(OP_NAME##Operator&&) = default; \ + OP_NAME##Operator& operator=(OP_NAME##Operator const&) = default; \ + ~OP_NAME##Operator() = default; \ + std::string get_type() const final { return OP_STRING; } \ + template \ + void serialize(ArchiveT& ar) \ + { \ + using OperatorType = Operator; \ + ar(::cereal::make_nvp("Operator", \ + ::cereal::base_class(this))); \ + } \ + void fp_compute(std::vector const& inputs, \ + std::vector const& outputs) const final; \ + void bp_compute( \ + std::vector const& inputs, \ + std::vector const& gradient_wrt_outputs, \ + std::vector const& gradient_wrt_inputs) const final; \ + \ + private: \ + void set_proto_params(lbann_data::Operator& msg) const final \ + { \ + msg.mutable_parameters()->PackFrom(lbann_data::OP_NAME##Operator{}); \ + } \ + void do_fill_description(description&) const final {} \ + } + +#define LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(OP_NAME, OP_STRING) \ + template \ + class OP_NAME##Operator final \ + : public Cloneable, \ + ElementwiseOperator> \ + { \ + using BaseType = Cloneable, \ + ElementwiseOperator>; \ + using LocalInputTensorType = typename BaseType::LocalInputTensorType; \ + using LocalOutputTensorType = typename BaseType::LocalOutputTensorType; \ + using ConstLocalInputTensorType = \ + typename BaseType::ConstLocalInputTensorType; \ + using ConstLocalOutputTensorType = \ + typename BaseType::ConstLocalOutputTensorType; \ + \ + public: \ + OP_NAME##Operator() = default; \ + OP_NAME##Operator(OP_NAME##Operator&&) = default; \ + OP_NAME##Operator(OP_NAME##Operator const&) = default; \ + OP_NAME##Operator& operator=(OP_NAME##Operator&&) = default; \ + OP_NAME##Operator& operator=(OP_NAME##Operator const&) = default; \ + ~OP_NAME##Operator() = default; \ + std::string get_type() const final { return OP_STRING; } \ + template \ + void serialize(ArchiveT& ar) \ + { \ + using OperatorType = ElementwiseOperator; \ + ar(::cereal::make_nvp("ElementwiseOperator", \ + ::cereal::base_class(this))); \ + } \ + \ + private: \ + void \ + fp_compute_local(std::vector inputs, \ + std::vector outputs) const final; \ + void bp_compute_local( \ + std::vector inputs, \ + std::vector grads_wrt_outputs, \ + std::vector grads_wrt_inputs) const final; \ + void set_proto_params(lbann_data::Operator& msg) const final \ + { \ + msg.mutable_parameters()->PackFrom(lbann_data::OP_NAME##Operator{}); \ + } \ + void do_fill_description(description&) const final {} \ + } + +#endif // LBANN_INCLUDE_LBANN_OPERATORS_DECLARE_STATELESS_OP_HPP_INCLUDED diff --git a/include/lbann/operators/elementwise_operator.hpp b/include/lbann/operators/elementwise_operator.hpp index 8388debb6ec..d62836f70cc 100644 --- a/include/lbann/operators/elementwise_operator.hpp +++ b/include/lbann/operators/elementwise_operator.hpp @@ -120,10 +120,10 @@ class ElementwiseOperator * tensors, the gradient w.r.t. input tensors are * populated with the computed values. */ - void - bp_compute(std::vector const& inputs, - std::vector const& gradient_wrt_outputs, - std::vector const& gradient_wrt_inputs) const final + void bp_compute( + std::vector const& inputs, + std::vector const& gradient_wrt_outputs, + std::vector const& gradient_wrt_inputs) const final { return bp_compute_local(get_local_tensor_views(inputs), get_local_tensor_views(gradient_wrt_outputs), diff --git a/include/lbann/operators/loss/CMakeLists.txt b/include/lbann/operators/loss/CMakeLists.txt new file mode 100644 index 00000000000..a86e5df5f33 --- /dev/null +++ b/include/lbann/operators/loss/CMakeLists.txt @@ -0,0 +1,9 @@ +# Add the headers for this directory +set_full_path(THIS_DIR_HEADERS + entrywise.hpp + loss_builders.hpp + loss_builders_impl.hpp + ) + +# Propagate the files up the tree +set(HEADERS "${HEADERS}" "${THIS_DIR_HEADERS}" PARENT_SCOPE) diff --git a/src/layers/math/cereal_registration/acosh.cpp b/include/lbann/operators/loss/entrywise.hpp similarity index 52% rename from src/layers/math/cereal_registration/acosh.cpp rename to include/lbann/operators/loss/entrywise.hpp index f34f24df7bd..1714c12c429 100644 --- a/src/layers/math/cereal_registration/acosh.cpp +++ b/include/lbann/operators/loss/entrywise.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -23,22 +23,28 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" -#include + +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_LOSS_ENTRYWISE_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_LOSS_ENTRYWISE_HPP_INCLUDED + +#include "lbann/operators/declare_stateless_op.hpp" namespace lbann { -template -template -void -acosh_layer -::serialize(ArchiveT& ar) { - using DataTypeLayer = data_type_layer; - ar(::cereal::make_nvp("DataTypeLayer", - ::cereal::base_class(this))); -} +// Cross entropy loss +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(BinaryCrossEntropy, + "binary cross entropy"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(SigmoidBinaryCrossEntropy, + "sigmoid binary cross entropy"); + +// Boolean loss functions +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(BooleanAccuracy, + "Boolean accuracy"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(BooleanFalseNegative, + "Boolean false negative rate"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(BooleanFalsePositive, + "Boolean false positive rate"); } // namespace lbann -#define LBANN_LAYER_NAME acosh_layer -#include +#endif // LBANN_INCLUDE_LBANN_OPERATORS_LOSS_ENTRYWISE_HPP_INCLUDED diff --git a/src/layers/activations/cereal_registration/log_sigmoid.cpp b/include/lbann/operators/loss/loss_builders.hpp similarity index 64% rename from src/layers/activations/cereal_registration/log_sigmoid.cpp rename to include/lbann/operators/loss/loss_builders.hpp index 684df579d83..fcd37305a00 100644 --- a/src/layers/activations/cereal_registration/log_sigmoid.cpp +++ b/include/lbann/operators/loss/loss_builders.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -23,22 +23,20 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" -#include + +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_LOSS_LOSS_BUILDERS_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_LOSS_LOSS_BUILDERS_HPP_INCLUDED + +#include "lbann/operators/builder_macros.hpp" +#include "lbann/operators/operator.hpp" namespace lbann { -template -template -void -log_sigmoid_layer -::serialize(ArchiveT& ar) { - using DataTypeLayer = data_type_layer; - ar(::cereal::make_nvp("DataTypeLayer", - ::cereal::base_class(this))); -} +LBANN_DECLARE_OPERATOR_BUILDER(binary_cross_entropy); +LBANN_DECLARE_OPERATOR_BUILDER(boolean_accuracy); +LBANN_DECLARE_OPERATOR_BUILDER(boolean_false_negative); +LBANN_DECLARE_OPERATOR_BUILDER(boolean_false_positive); +LBANN_DECLARE_OPERATOR_BUILDER(sigmoid_binary_cross_entropy); } // namespace lbann - -#define LBANN_LAYER_NAME log_sigmoid_layer -#include "lbann/macros/register_layer_with_cereal.hpp" +#endif // LBANN_INCLUDE_LBANN_OPERATORS_LOSS_LOSS_BUILDERS_HPP_INCLUDED diff --git a/src/layers/math/cereal_registration/acos.cpp b/include/lbann/operators/loss/loss_builders_impl.hpp similarity index 58% rename from src/layers/math/cereal_registration/acos.cpp rename to include/lbann/operators/loss/loss_builders_impl.hpp index 77f8156cc94..81061d7fc2e 100644 --- a/src/layers/math/cereal_registration/acos.cpp +++ b/include/lbann/operators/loss/loss_builders_impl.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -23,22 +23,17 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" -#include +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_LOSS_LOSS_BUILDERS_IMPL_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_LOSS_LOSS_BUILDERS_IMPL_HPP_INCLUDED -namespace lbann { +#include "lbann/operators/loss/entrywise.hpp" +#include "lbann/operators/loss/loss_builders.hpp" -template -template -void -acos_layer -::serialize(ArchiveT& ar) { - using DataTypeLayer = data_type_layer; - ar(::cereal::make_nvp("DataTypeLayer", - ::cereal::base_class(this))); -} +LBANN_DEFINE_OPERATOR_BUILDER(binary_cross_entropy, BinaryCrossEntropy); +LBANN_DEFINE_OPERATOR_BUILDER(boolean_accuracy, BooleanAccuracy); +LBANN_DEFINE_OPERATOR_BUILDER(boolean_false_negative, BooleanFalseNegative); +LBANN_DEFINE_OPERATOR_BUILDER(boolean_false_positive, BooleanFalsePositive); +LBANN_DEFINE_OPERATOR_BUILDER(sigmoid_binary_cross_entropy, + SigmoidBinaryCrossEntropy); -} // namespace lbann - -#define LBANN_LAYER_NAME acos_layer -#include +#endif // LBANN_INCLUDE_LBANN_OPERATORS_LOSS_LOSS_BUILDERS_IMPL_HPP_INCLUDED diff --git a/include/lbann/operators/math/CMakeLists.txt b/include/lbann/operators/math/CMakeLists.txt index 990c1340ab2..5fc1e722c71 100644 --- a/include/lbann/operators/math/CMakeLists.txt +++ b/include/lbann/operators/math/CMakeLists.txt @@ -1,8 +1,11 @@ # Add the headers for this directory set_full_path(THIS_DIR_HEADERS + abs.hpp + binary.hpp clamp.hpp math_builders.hpp math_builders_impl.hpp + unary.hpp ) # Propagate the files up the tree diff --git a/include/lbann/operators/math/abs.hpp b/include/lbann/operators/math/abs.hpp new file mode 100644 index 00000000000..34f94fae7ce --- /dev/null +++ b/include/lbann/operators/math/abs.hpp @@ -0,0 +1,129 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// + +#ifndef LBANN_OPERATORS_MATH_ABS_HPP_INCLUDED +#define LBANN_OPERATORS_MATH_ABS_HPP_INCLUDED + +#include "lbann_config.hpp" + +#include "lbann/operators/elementwise_operator.hpp" +#include "lbann/proto/datatype_helpers.hpp" +#include "lbann/utils/cloneable.hpp" + +#include + +#include

+ +#include + +namespace lbann { + +/** @brief Entrywise absolute value. + * + * @f[ + * \text{abs}(x) = |x| + * @f] + */ +template +class AbsOperator final + : public Cloneable, + ElementwiseOperator, D>> +{ + /** @name Private Types */ + ///@{ + + using BaseType = Cloneable, + ElementwiseOperator, D>>; + + using LocalInputTensorType = typename BaseType::LocalInputTensorType; + using LocalOutputTensorType = typename BaseType::LocalOutputTensorType; + using ConstLocalInputTensorType = + typename BaseType::ConstLocalInputTensorType; + using ConstLocalOutputTensorType = + typename BaseType::ConstLocalOutputTensorType; + + ///@} + +public: + /** @name Lifecycle */ + ///@{ + + AbsOperator() = default; + AbsOperator(AbsOperator&&) = default; + AbsOperator(AbsOperator const&) = default; + AbsOperator& operator=(AbsOperator&&) = default; + AbsOperator& operator=(AbsOperator const&) = default; + ~AbsOperator() = default; + + ///@} + /** @name Queries */ + ///@{ + + std::string get_type() const final { return "abs"; } + + ///@} + /** @name Serialization */ + ///@{ + + template + void serialize(ArchiveT& ar) + { + using OperatorType = ElementwiseOperator, D>; + ar(::cereal::make_nvp("DataTypeOperator", + ::cereal::base_class(this))); + } + + ///@} + +private: + /** @brief Local forward compute function */ + virtual void + fp_compute_local(std::vector input, + std::vector output) const final; + + /** @brief Local backward compute function */ + void bp_compute_local( + std::vector input, + std::vector gradient_wrt_output, + std::vector gradient_wrt_input) const final; + + void set_proto_params(lbann_data::Operator& msg) const final + { + msg.mutable_parameters()->PackFrom(lbann_data::AbsOperator{}); + } + + void do_fill_description(description& desc) const final {} +}; // class AbsOperator + +#ifndef LBANN_ABS_OPERATOR_INSTANTIATE +#define PROTO_DEVICE(T, D) extern template class AbsOperator +#include "lbann/macros/instantiate_device.hpp" +#undef PROTO_DEVICE +#endif // LBANN_ABS_OPERATOR_INSTANTIATE + +} // namespace lbann + +#endif // LBANN_OPERATORS_MATH_ABS_HPP_INCLUDED diff --git a/include/lbann/operators/math/binary.hpp b/include/lbann/operators/math/binary.hpp new file mode 100644 index 00000000000..be193c95c1f --- /dev/null +++ b/include/lbann/operators/math/binary.hpp @@ -0,0 +1,63 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// + +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_MATH_BINARY_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_MATH_BINARY_HPP_INCLUDED + +#include "lbann/operators/declare_stateless_op.hpp" + +namespace lbann { + +// Arithmetic operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Add, "add"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Subtract, "subtract"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Multiply, "multiply"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Divide, "divide"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Mod, "modulo"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Pow, "power"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(SafeDivide, "safe divide"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(SquaredDifference, + "squared difference"); + +// Comparison operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Max, "maximum"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Min, "minimum"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Equal, "equal"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(NotEqual, "not equal"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Less, "less than"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(LessEqual, "less than or equal"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Greater, "greater than"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(GreaterEqual, + "greater than or equal"); + +// Logical operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(LogicalAnd, "logical and"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(LogicalOr, "logical or"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(LogicalXor, "logical xor"); + +} // namespace lbann + +#endif // LBANN_INCLUDE_LBANN_OPERATORS_MATH_BINARY_HPP_INCLUDED diff --git a/include/lbann/operators/math/clamp.hpp b/include/lbann/operators/math/clamp.hpp index 413892d034b..1fa71f7b855 100644 --- a/include/lbann/operators/math/clamp.hpp +++ b/include/lbann/operators/math/clamp.hpp @@ -30,15 +30,12 @@ #include "lbann_config.hpp" #include "lbann/operators/elementwise_operator.hpp" -#include "lbann/proto/datatype_helpers.hpp" #include "lbann/utils/cloneable.hpp" #include #include

-#include - namespace lbann { /** @brief Constrain values to a range. @@ -53,8 +50,9 @@ namespace lbann { * @f] */ template -class ClampOperator : public Cloneable, - ElementwiseOperator> +class ClampOperator final + : public Cloneable, + ElementwiseOperator> { #ifdef LBANN_HAS_GPU_FP16 using CompareType = @@ -111,7 +109,7 @@ class ClampOperator : public Cloneable, void serialize(ArchiveT& ar) { using OperatorType = ElementwiseOperator; - ar(::cereal::make_nvp("DataTypeOperator", + ar(::cereal::make_nvp("ElementwiseOperator", ::cereal::base_class(this)), CEREAL_NVP(m_min), CEREAL_NVP(m_max)); @@ -125,17 +123,16 @@ class ClampOperator : public Cloneable, private: /** @brief Local forward compute function */ - virtual void - fp_compute_local(std::vector input, - std::vector output) const override; + void fp_compute_local(std::vector input, + std::vector output) const final; /** @brief Local backward compute function */ void bp_compute_local( std::vector input, std::vector gradient_wrt_output, - std::vector gradient_wrt_input) const override; + std::vector gradient_wrt_input) const final; - void set_proto_params(lbann_data::Operator& msg) const override + void set_proto_params(lbann_data::Operator& msg) const final { lbann_data::ClampOperator clamp_msg; clamp_msg.set_min(m_min); @@ -143,7 +140,7 @@ class ClampOperator : public Cloneable, msg.mutable_parameters()->PackFrom(clamp_msg); } - void do_fill_description(description& desc) const override + void do_fill_description(description& desc) const final { std::ostringstream oss; oss << "[" << m_min << "," << m_max << "]"; diff --git a/include/lbann/operators/math/math_builders.hpp b/include/lbann/operators/math/math_builders.hpp index 873cda58943..7a232d5927f 100644 --- a/include/lbann/operators/math/math_builders.hpp +++ b/include/lbann/operators/math/math_builders.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -23,19 +23,68 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_MATH_MATH_BUILDERS_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_MATH_MATH_BUILDERS_HPP_INCLUDED #include "lbann/operators/operator.hpp" -#include - -// Forward declaration -namespace lbann_data { -class Operator; -} // namespace lbann_data +#include "lbann/operators/builder_macros.hpp" namespace lbann { template -std::unique_ptr> -build_clamp_operator(lbann_data::Operator const& op); +std::unique_ptr, D>> +build_abs_operator(lbann_data::Operator const& op); + +LBANN_DECLARE_OPERATOR_BUILDER(acos); +LBANN_DECLARE_OPERATOR_BUILDER(acosh); +LBANN_DECLARE_OPERATOR_BUILDER(add); +LBANN_DECLARE_OPERATOR_BUILDER(asin); +LBANN_DECLARE_OPERATOR_BUILDER(asinh); +LBANN_DECLARE_OPERATOR_BUILDER(atan); +LBANN_DECLARE_OPERATOR_BUILDER(atanh); +LBANN_DECLARE_OPERATOR_BUILDER(ceil); +LBANN_DECLARE_OPERATOR_BUILDER(clamp); +LBANN_DECLARE_OPERATOR_BUILDER(cos); +LBANN_DECLARE_OPERATOR_BUILDER(cosh); +LBANN_DECLARE_OPERATOR_BUILDER(divide); +LBANN_DECLARE_OPERATOR_BUILDER(equal); +LBANN_DECLARE_OPERATOR_BUILDER(erf); +LBANN_DECLARE_OPERATOR_BUILDER(erfinv); +LBANN_DECLARE_OPERATOR_BUILDER(exp); +LBANN_DECLARE_OPERATOR_BUILDER(expm1); +LBANN_DECLARE_OPERATOR_BUILDER(floor); +LBANN_DECLARE_OPERATOR_BUILDER(greater); +LBANN_DECLARE_OPERATOR_BUILDER(greater_equal); +LBANN_DECLARE_OPERATOR_BUILDER(less); +LBANN_DECLARE_OPERATOR_BUILDER(less_equal); +LBANN_DECLARE_OPERATOR_BUILDER(log); +LBANN_DECLARE_OPERATOR_BUILDER(log1p); +LBANN_DECLARE_OPERATOR_BUILDER(logical_and); +LBANN_DECLARE_OPERATOR_BUILDER(logical_not); +LBANN_DECLARE_OPERATOR_BUILDER(logical_or); +LBANN_DECLARE_OPERATOR_BUILDER(logical_xor); +LBANN_DECLARE_OPERATOR_BUILDER(max); +LBANN_DECLARE_OPERATOR_BUILDER(min); +LBANN_DECLARE_OPERATOR_BUILDER(mod); +LBANN_DECLARE_OPERATOR_BUILDER(multiply); +LBANN_DECLARE_OPERATOR_BUILDER(negative); +LBANN_DECLARE_OPERATOR_BUILDER(not_equal); +LBANN_DECLARE_OPERATOR_BUILDER(pow); +LBANN_DECLARE_OPERATOR_BUILDER(reciprocal); +LBANN_DECLARE_OPERATOR_BUILDER(round); +LBANN_DECLARE_OPERATOR_BUILDER(rsqrt); +LBANN_DECLARE_OPERATOR_BUILDER(safe_divide); +LBANN_DECLARE_OPERATOR_BUILDER(safe_reciprocal); +LBANN_DECLARE_OPERATOR_BUILDER(sign); +LBANN_DECLARE_OPERATOR_BUILDER(sin); +LBANN_DECLARE_OPERATOR_BUILDER(sinh); +LBANN_DECLARE_OPERATOR_BUILDER(sqrt); +LBANN_DECLARE_OPERATOR_BUILDER(square); +LBANN_DECLARE_OPERATOR_BUILDER(squared_difference); +LBANN_DECLARE_OPERATOR_BUILDER(subtract); +LBANN_DECLARE_OPERATOR_BUILDER(tan); +LBANN_DECLARE_OPERATOR_BUILDER(tanh); } // namespace lbann + +#endif // LBANN_INCLUDE_LBANN_OPERATORS_MATH_MATH_BUILDERS_HPP_INCLUDED diff --git a/include/lbann/operators/math/math_builders_impl.hpp b/include/lbann/operators/math/math_builders_impl.hpp index eca30524bc0..0f69d3417f2 100644 --- a/include/lbann/operators/math/math_builders_impl.hpp +++ b/include/lbann/operators/math/math_builders_impl.hpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. // Produced at the Lawrence Livermore National Laboratory. // Written by the LBANN Research Team (B. Van Essen, et al.) listed in // the CONTRIBUTORS file. @@ -23,26 +23,82 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_MATH_MATH_BUILDERS_IMPL_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_MATH_MATH_BUILDERS_IMPL_HPP_INCLUDED -#include "lbann/operators/math/clamp.hpp" #include "lbann/operators/math/math_builders.hpp" -#include "lbann/operators/operator.hpp" -#include "lbann/proto/datatype_helpers.hpp" -#include +#include "lbann/operators/math/abs.hpp" +#include "lbann/operators/math/binary.hpp" +#include "lbann/operators/math/clamp.hpp" +#include "lbann/operators/math/unary.hpp" -#include +#include "lbann/proto/datatype_helpers.hpp" template std::unique_ptr> lbann::build_clamp_operator(lbann_data::Operator const& op) { - LBANN_ASSERT(proto::ProtoDataType == op.input_datatype()); - LBANN_ASSERT(proto::ProtoDataType == op.output_datatype()); - LBANN_ASSERT(proto::ProtoDevice == - proto::resolve_default_device(op.device_allocation())); - + details::AssertConsistentTypeParameters(op); lbann_data::ClampOperator params; LBANN_ASSERT(op.parameters().UnpackTo(¶ms)); return std::make_unique>(params.min(), params.max()); } + +template +std::unique_ptr, D>> +lbann::build_abs_operator(lbann_data::Operator const& op) +{ + details::AssertConsistentTypeParameters, D>(op); + return std::make_unique>(); +} + +LBANN_DEFINE_OPERATOR_BUILDER(acos, Acos) +LBANN_DEFINE_OPERATOR_BUILDER(acosh, Acosh) +LBANN_DEFINE_OPERATOR_BUILDER(add, Add) +LBANN_DEFINE_OPERATOR_BUILDER(asin, Asin) +LBANN_DEFINE_OPERATOR_BUILDER(asinh, Asinh) +LBANN_DEFINE_OPERATOR_BUILDER(atan, Atan) +LBANN_DEFINE_OPERATOR_BUILDER(atanh, Atanh) +LBANN_DEFINE_OPERATOR_BUILDER(ceil, Ceil) +LBANN_DEFINE_OPERATOR_BUILDER(cos, Cos) +LBANN_DEFINE_OPERATOR_BUILDER(cosh, Cosh) +LBANN_DEFINE_OPERATOR_BUILDER(divide, Divide) +LBANN_DEFINE_OPERATOR_BUILDER(equal, Equal) +LBANN_DEFINE_OPERATOR_BUILDER(erf, Erf) +LBANN_DEFINE_OPERATOR_BUILDER(erfinv, ErfInv) +LBANN_DEFINE_OPERATOR_BUILDER(exp, Exp) +LBANN_DEFINE_OPERATOR_BUILDER(expm1, Expm1) +LBANN_DEFINE_OPERATOR_BUILDER(floor, Floor) +LBANN_DEFINE_OPERATOR_BUILDER(greater, Greater) +LBANN_DEFINE_OPERATOR_BUILDER(greater_equal, GreaterEqual) +LBANN_DEFINE_OPERATOR_BUILDER(less, Less) +LBANN_DEFINE_OPERATOR_BUILDER(less_equal, LessEqual) +LBANN_DEFINE_OPERATOR_BUILDER(log, Log) +LBANN_DEFINE_OPERATOR_BUILDER(log1p, Log1p) +LBANN_DEFINE_OPERATOR_BUILDER(logical_and, LogicalAnd) +LBANN_DEFINE_OPERATOR_BUILDER(logical_not, LogicalNot) +LBANN_DEFINE_OPERATOR_BUILDER(logical_or, LogicalOr) +LBANN_DEFINE_OPERATOR_BUILDER(logical_xor, LogicalXor) +LBANN_DEFINE_OPERATOR_BUILDER(max, Max) +LBANN_DEFINE_OPERATOR_BUILDER(min, Min) +LBANN_DEFINE_OPERATOR_BUILDER(mod, Mod) +LBANN_DEFINE_OPERATOR_BUILDER(multiply, Multiply) +LBANN_DEFINE_OPERATOR_BUILDER(negative, Negative) +LBANN_DEFINE_OPERATOR_BUILDER(not_equal, NotEqual) +LBANN_DEFINE_OPERATOR_BUILDER(pow, Pow) +LBANN_DEFINE_OPERATOR_BUILDER(reciprocal, Reciprocal) +LBANN_DEFINE_OPERATOR_BUILDER(round, Round) +LBANN_DEFINE_OPERATOR_BUILDER(rsqrt, Rsqrt) +LBANN_DEFINE_OPERATOR_BUILDER(safe_divide, SafeDivide) +LBANN_DEFINE_OPERATOR_BUILDER(safe_reciprocal, SafeReciprocal) +LBANN_DEFINE_OPERATOR_BUILDER(sign, Sign) +LBANN_DEFINE_OPERATOR_BUILDER(sin, Sin) +LBANN_DEFINE_OPERATOR_BUILDER(sinh, Sinh) +LBANN_DEFINE_OPERATOR_BUILDER(sqrt, Sqrt) +LBANN_DEFINE_OPERATOR_BUILDER(square, Square) +LBANN_DEFINE_OPERATOR_BUILDER(squared_difference, SquaredDifference) +LBANN_DEFINE_OPERATOR_BUILDER(subtract, Subtract) +LBANN_DEFINE_OPERATOR_BUILDER(tan, Tan) +LBANN_DEFINE_OPERATOR_BUILDER(tanh, Tanh) +#endif // LBANN_INCLUDE_LBANN_OPERATORS_MATH_MATH_BUILDERS_IMPL_HPP_INCLUDED diff --git a/include/lbann/operators/math/unary.hpp b/include/lbann/operators/math/unary.hpp new file mode 100644 index 00000000000..3495fc1786e --- /dev/null +++ b/include/lbann/operators/math/unary.hpp @@ -0,0 +1,83 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2021, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// + +#ifndef LBANN_INCLUDE_LBANN_OPERATORS_MATH_UNARY_HPP_INCLUDED +#define LBANN_INCLUDE_LBANN_OPERATORS_MATH_UNARY_HPP_INCLUDED + +#include "lbann/operators/declare_stateless_op.hpp" + +namespace lbann { + +// These are all single-type operators. + +// Logical operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(LogicalNot, "logical not"); + +// Sign operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Negative, "negative"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Sign, "sign"); + +// Rounding operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Round, "round"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Ceil, "ceil"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Floor, "floor"); + +// Power operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Reciprocal, "reciprocal"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Square, "square"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Sqrt, "square root"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Rsqrt, "reciprocal square root"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(SafeReciprocal, "safe reciprocal"); + +// Exponential and logarithmic operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Exp, "exponential"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Expm1, "expm1"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Log, "natural logarithm"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Log1p, "log1p"); + +// Trigonometric operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Cos, "cosine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Sin, "sine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Tan, "tangent"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Acos, "arccosine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Asin, "arcsine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Atan, "arctangent"); + +// Hyperbolic operations +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Cosh, "hyperbolic cosine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Sinh, "hyperbolic sine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Tanh, "hyperbolic tangent"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Acosh, "hyperbolic arccosine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Asinh, "hyperbolic arcsine"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Atanh, "hyperbolic arctangent"); + +// Error function +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(Erf, "error function"); +LBANN_DECLARE_STATELESS_ELEMENTWISE_OPERATOR(ErfInv, "inverse error function"); + +} // namespace lbann + +#endif // LBANN_INCLUDE_LBANN_OPERATORS_MATH_UNARY_HPP_INCLUDED diff --git a/include/lbann/operators/operator.hpp b/include/lbann/operators/operator.hpp index 7270e14f717..cdc1b5e6867 100644 --- a/include/lbann/operators/operator.hpp +++ b/include/lbann/operators/operator.hpp @@ -46,13 +46,6 @@ #include #include -/** @brief A utility macro for easily adding ETI for operator builders - * @note Must be called inside lbann namespace. - */ -#define LBANN_SINGLE_TYPE_OPERATOR_BUILDER_ETI(OPERATOR_NAME, T, D) \ - template std::unique_ptr> \ - build_##OPERATOR_NAME##_operator(lbann_data::Operator const&) - namespace lbann { using supported_operator_data_type = h2::meta::TL< @@ -63,7 +56,9 @@ using supported_operator_data_type = h2::meta::TL< cpu_fp16, #endif float, - double>; + double, + El::Complex, + El::Complex>; /** @brief Neural network tensor operation. * @@ -132,13 +127,11 @@ class Operator : public AbstractCloneableBase>, ///@{ template - void serialize(ArchiveT& ar){}; + void serialize(ArchiveT& ar); ///@} - - // =========================================================== - // Forward prop compute function - // =========================================================== + /** @name Computational interface */ + ///@{ /** @brief Apply operator's forward operation. * @details Given the input tensors, the output tensors are @@ -148,10 +141,6 @@ class Operator : public AbstractCloneableBase>, fp_compute(std::vector const& inputs, std::vector const& outputs) const = 0; - // =========================================================== - // Back prop compute function - // =========================================================== - /** @brief Compute operator's "backward" operation * @details Given the inputs, outputs, and gradient w.r.t. output * tensors, the gradient w.r.t. input tensors are @@ -160,7 +149,8 @@ class Operator : public AbstractCloneableBase>, virtual void bp_compute(std::vector const& inputs, std::vector const& gradient_wrt_outputs, - std::vector const& gradient_wrt_inputs) const {}; + std::vector const& gradient_wrt_inputs) const; + ///@} protected: Operator(Operator&& other) noexcept = default; @@ -205,5 +195,17 @@ Description Operator::get_description() const return desc; } +template +void Operator::bp_compute( + std::vector const&, + std::vector const&, + std::vector const&) const +{} + +template +template +void Operator::serialize(ArchiveT& ar) +{} + } // namespace lbann #endif // LBANN_OPERATORS_OPERATOR_HPP_INCLUDED diff --git a/include/lbann/proto/datatype_helpers.hpp b/include/lbann/proto/datatype_helpers.hpp index b0bf7a0ec6e..949589f67b9 100644 --- a/include/lbann/proto/datatype_helpers.hpp +++ b/include/lbann/proto/datatype_helpers.hpp @@ -49,6 +49,18 @@ struct TypeToProtoDataType static constexpr auto value = lbann_data::DOUBLE; }; +template <> +struct TypeToProtoDataType> +{ + static constexpr auto value = lbann_data::COMPLEX_FLOAT; +}; + +template <> +struct TypeToProtoDataType> +{ + static constexpr auto value = lbann_data::COMPLEX_DOUBLE; +}; + #ifdef LBANN_HAS_HALF template <> struct TypeToProtoDataType diff --git a/include/lbann/proto/operator_factory_impl.hpp b/include/lbann/proto/operator_factory_impl.hpp index 56dd439c4c4..88ab5cb13be 100644 --- a/include/lbann/proto/operator_factory_impl.hpp +++ b/include/lbann/proto/operator_factory_impl.hpp @@ -29,6 +29,8 @@ #include "lbann/proto/factories.hpp" #include "lbann/proto/operator_factory.hpp" +#include "lbann/operators/activations/activation_builders.hpp" +#include "lbann/operators/loss/loss_builders.hpp" #include "lbann/operators/math/math_builders.hpp" #include "lbann/operators/operator.hpp" #include "lbann/proto/datatype_helpers.hpp" @@ -44,9 +46,77 @@ OperatorFactory build_default_factory() { OperatorFactory factory; +#define LBANN_REGISTER_BUILDER(OP_NAME, OP_LOWER) \ + factory.register_builder(#OP_NAME "Operator", \ + build_##OP_LOWER##_operator) + if constexpr (std::is_same_v) { - factory.register_builder("ClampOperator", build_clamp_operator); + LBANN_REGISTER_BUILDER(Acos, acos); + LBANN_REGISTER_BUILDER(Acosh, acosh); + LBANN_REGISTER_BUILDER(Add, add); + LBANN_REGISTER_BUILDER(Asin, asin); + LBANN_REGISTER_BUILDER(Asinh, asinh); + LBANN_REGISTER_BUILDER(Atan, atan); + LBANN_REGISTER_BUILDER(Atanh, atanh); + LBANN_REGISTER_BUILDER(BinaryCrossEntropy, binary_cross_entropy); + LBANN_REGISTER_BUILDER(BooleanAccuracy, boolean_accuracy); + LBANN_REGISTER_BUILDER(BooleanFalseNegative, boolean_false_negative); + LBANN_REGISTER_BUILDER(BooleanFalsePositive, boolean_false_positive); + LBANN_REGISTER_BUILDER(Ceil, ceil); + LBANN_REGISTER_BUILDER(Clamp, clamp); + LBANN_REGISTER_BUILDER(Cos, cos); + LBANN_REGISTER_BUILDER(Cosh, cosh); + LBANN_REGISTER_BUILDER(Divide, divide); + LBANN_REGISTER_BUILDER(Equal, equal); + LBANN_REGISTER_BUILDER(Erf, erf); + LBANN_REGISTER_BUILDER(ErfInv, erfinv); + LBANN_REGISTER_BUILDER(Exp, exp); + LBANN_REGISTER_BUILDER(Expm1, expm1); + LBANN_REGISTER_BUILDER(Floor, floor); + LBANN_REGISTER_BUILDER(Greater, greater); + LBANN_REGISTER_BUILDER(GreaterEqual, greater_equal); + LBANN_REGISTER_BUILDER(Less, less); + LBANN_REGISTER_BUILDER(LessEqual, less_equal); + LBANN_REGISTER_BUILDER(Log, log); + LBANN_REGISTER_BUILDER(Log1p, log1p); + LBANN_REGISTER_BUILDER(LogSigmoid, log_sigmoid); + LBANN_REGISTER_BUILDER(LogicalAnd, logical_and); + LBANN_REGISTER_BUILDER(LogicalNot, logical_not); + LBANN_REGISTER_BUILDER(LogicalOr, logical_or); + LBANN_REGISTER_BUILDER(LogicalXor, logical_xor); + LBANN_REGISTER_BUILDER(Max, max); + LBANN_REGISTER_BUILDER(Min, min); + LBANN_REGISTER_BUILDER(Mod, mod); + LBANN_REGISTER_BUILDER(Multiply, multiply); + LBANN_REGISTER_BUILDER(Negative, negative); + LBANN_REGISTER_BUILDER(NotEqual, not_equal); + LBANN_REGISTER_BUILDER(Pow, pow); + LBANN_REGISTER_BUILDER(Reciprocal, reciprocal); + LBANN_REGISTER_BUILDER(Round, round); + LBANN_REGISTER_BUILDER(Rsqrt, rsqrt); + LBANN_REGISTER_BUILDER(SafeDivide, safe_divide); + LBANN_REGISTER_BUILDER(SafeReciprocal, safe_reciprocal); + LBANN_REGISTER_BUILDER(Selu, selu); + LBANN_REGISTER_BUILDER(Sigmoid, sigmoid); + LBANN_REGISTER_BUILDER(SigmoidBinaryCrossEntropy, + sigmoid_binary_cross_entropy); + LBANN_REGISTER_BUILDER(Sign, sign); + LBANN_REGISTER_BUILDER(Sin, sin); + LBANN_REGISTER_BUILDER(Sinh, sinh); + LBANN_REGISTER_BUILDER(Softplus, softplus); + LBANN_REGISTER_BUILDER(Softsign, softsign); + LBANN_REGISTER_BUILDER(Sqrt, sqrt); + LBANN_REGISTER_BUILDER(Square, square); + LBANN_REGISTER_BUILDER(SquaredDifference, squared_difference); + LBANN_REGISTER_BUILDER(Subtract, subtract); + LBANN_REGISTER_BUILDER(Tan, tan); + LBANN_REGISTER_BUILDER(Tanh, tanh); + } + + if constexpr (std::is_same_v>) { + factory.register_builder("AbsOperator", build_abs_operator); } + return factory; } diff --git a/include/lbann/utils/exception.hpp b/include/lbann/utils/exception.hpp index c74f4eaea36..5c664bea89a 100644 --- a/include/lbann/utils/exception.hpp +++ b/include/lbann/utils/exception.hpp @@ -78,6 +78,12 @@ if (!(cond)) \ LBANN_ERROR("The assertion " #cond " failed.") +#ifdef LBANN_DEBUG +#define LBANN_ASSERT_DEBUG(cond) LBANN_ASSERT(cond) +#else +#define LBANN_ASSERT_DEBUG(cond) +#endif + #define LBANN_ASSERT_WARNING(cond) \ if (!(cond)) \ LBANN_WARNING("The assertion " #cond " failed.") diff --git a/include/lbann/utils/tensor.hpp b/include/lbann/utils/tensor.hpp index 22cef53b582..bab2330233c 100644 --- a/include/lbann/utils/tensor.hpp +++ b/include/lbann/utils/tensor.hpp @@ -101,7 +101,8 @@ std::vector verify_and_get_dims(MatrixT const& A, auto const A_width = A.Width(); if (dims[0] == El::To(A_width)) { #ifdef LBANN_DEBUG - LBANN_ASSERT(get_linear_size(dims) == A_height * A_width); + LBANN_ASSERT(get_linear_size(dims) == + static_cast(A_height * A_width)); #endif return dims; } @@ -282,7 +283,8 @@ template class ConstDistTensorView : public details::MatrixAsTensorView const> { - using base_type = details::MatrixAsTensorView const>; + using base_type = + details::MatrixAsTensorView const>; public: template diff --git a/python/lbann/core/operators.py b/python/lbann/core/operators.py index badb9a211d9..e21f92c23eb 100644 --- a/python/lbann/core/operators.py +++ b/python/lbann/core/operators.py @@ -64,3 +64,534 @@ def do_export_proto(self): params.min = self.min params.max = self.max return params + +class Abs(Operator): + """Apply the Abs operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AbsOperator() + return params + +class Acos(Operator): + """Apply the Acos operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AcosOperator() + return params + +class Acosh(Operator): + """Apply the Acosh operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AcoshOperator() + return params + +class Add(Operator): + """Apply the Add operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AddOperator() + return params + +class Asin(Operator): + """Apply the Asin operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AsinOperator() + return params + +class Asinh(Operator): + """Apply the Asinh operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AsinhOperator() + return params + +class Atan(Operator): + """Apply the Atan operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AtanOperator() + return params + +class Atanh(Operator): + """Apply the Atanh operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.AtanhOperator() + return params + +class BinaryCrossEntropy(Operator): + """Apply the BinaryCrossEntropy operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.BinaryCrossEntropyOperator() + return params + +class BooleanAccuracy(Operator): + """Apply the BooleanAccuracy operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.BooleanAccuracyOperator() + return params + +class BooleanFalseNegative(Operator): + """Apply the BooleanFalseNegative operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.BooleanFalseNegativeOperator() + return params + +class BooleanFalsePositive(Operator): + """Apply the BooleanFalsePositive operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.BooleanFalsePositiveOperator() + return params + +class Ceil(Operator): + """Apply the Ceil operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.CeilOperator() + return params + +class Cos(Operator): + """Apply the Cos operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.CosOperator() + return params + +class Cosh(Operator): + """Apply the Cosh operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.CoshOperator() + return params + +class Divide(Operator): + """Apply the Divide operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.DivideOperator() + return params + +class Equal(Operator): + """Apply the Equal operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.EqualOperator() + return params + +class Erf(Operator): + """Apply the Erf operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.ErfOperator() + return params + +class ErfInv(Operator): + """Apply the ErfInv operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.ErfInvOperator() + return params + +class Exp(Operator): + """Apply the Exp operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.ExpOperator() + return params + +class Expm1(Operator): + """Apply the Expm1 operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.Expm1Operator() + return params + +class Floor(Operator): + """Apply the Floor operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.FloorOperator() + return params + +class Greater(Operator): + """Apply the Greater operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.GreaterOperator() + return params + +class GreaterEqual(Operator): + """Apply the GreaterEqual operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.GreaterEqualOperator() + return params + +class Less(Operator): + """Apply the Less operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LessOperator() + return params + +class LessEqual(Operator): + """Apply the LessEqual operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LessEqualOperator() + return params + +class Log(Operator): + """Apply the Log operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LogOperator() + return params + +class Log1p(Operator): + """Apply the Log1p operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.Log1pOperator() + return params + +class LogSigmoid(Operator): + """Apply the LogSigmoid operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LogSigmoidOperator() + return params + +class LogicalAnd(Operator): + """Apply the LogicalAnd operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LogicalAndOperator() + return params + +class LogicalNot(Operator): + """Apply the LogicalNot operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LogicalNotOperator() + return params + +class LogicalOr(Operator): + """Apply the LogicalOr operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LogicalOrOperator() + return params + +class LogicalXor(Operator): + """Apply the LogicalXor operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.LogicalXorOperator() + return params + +class Max(Operator): + """Apply the Max operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.MaxOperator() + return params + +class Min(Operator): + """Apply the Min operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.MinOperator() + return params + +class Mod(Operator): + """Apply the Mod operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.ModOperator() + return params + +class Multiply(Operator): + """Apply the Multiply operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.MultiplyOperator() + return params + +class Negative(Operator): + """Apply the Negative operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.NegativeOperator() + return params + +class NotEqual(Operator): + """Apply the NotEqual operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.NotEqualOperator() + return params + +class Pow(Operator): + """Apply the Pow operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.PowOperator() + return params + +class Reciprocal(Operator): + """Apply the Reciprocal operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.ReciprocalOperator() + return params + +class Round(Operator): + """Apply the Round operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.RoundOperator() + return params + +class Rsqrt(Operator): + """Apply the Rsqrt operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.RsqrtOperator() + return params + +class SafeDivide(Operator): + """Apply the SafeDivide operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SafeDivideOperator() + return params + +class SafeReciprocal(Operator): + """Apply the SafeReciprocal operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SafeReciprocalOperator() + return params + +class Selu(Operator): + """Apply the Selu operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SeluOperator() + return params + +class Sigmoid(Operator): + """Apply the Sigmoid operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SigmoidOperator() + return params + +class SigmoidBinaryCrossEntropy(Operator): + """Apply the SigmoidBinaryCrossEntropy operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SigmoidBinaryCrossEntropyOperator() + return params + +class Sign(Operator): + """Apply the Sign operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SignOperator() + return params + +class Sin(Operator): + """Apply the Sin operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SinOperator() + return params + +class Sinh(Operator): + """Apply the Sinh operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SinhOperator() + return params + +class Softplus(Operator): + """Apply the Softplus operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SoftplusOperator() + return params + +class Softsign(Operator): + """Apply the Softsign operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SoftsignOperator() + return params + +class Sqrt(Operator): + """Apply the Sqrt operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SqrtOperator() + return params + +class Square(Operator): + """Apply the Square operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SquareOperator() + return params + +class SquaredDifference(Operator): + """Apply the SquaredDifference operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SquaredDifferenceOperator() + return params + +class Subtract(Operator): + """Apply the Subtract operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.SubtractOperator() + return params + +class Tan(Operator): + """Apply the Tan operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.TanOperator() + return params + +class Tanh(Operator): + """Apply the Tanh operator entrywise.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def do_export_proto(self): + params = OpProto.TanhOperator() + return params diff --git a/src/execution_algorithms/ltfb/mutation_strategy.cpp b/src/execution_algorithms/ltfb/mutation_strategy.cpp index afa29753652..7f5e3997cb2 100644 --- a/src/execution_algorithms/ltfb/mutation_strategy.cpp +++ b/src/execution_algorithms/ltfb/mutation_strategy.cpp @@ -28,18 +28,23 @@ #include "lbann/comm_impl.hpp" -#include "lbann/layers/activations/activations.hpp" #include "lbann/layers/activations/elu.hpp" #include "lbann/layers/activations/leaky_relu.hpp" #include "lbann/layers/activations/log_softmax.hpp" #include "lbann/layers/activations/relu.hpp" #include "lbann/layers/activations/softmax.hpp" -#include "lbann/layers/math/unary.hpp" +#include "lbann/layers/operator_layer.hpp" + +#include "lbann/operators/activations/activations.hpp" // SigmoidOperator +#include "lbann/operators/math/unary.hpp" // TanhOperator #include "lbann/layers/learning/convolution.hpp" #include "lbann/models/model.hpp" +#include "lbann/operators/math/unary.hpp" #include "lbann/utils/random.hpp" +#include "lbann_config.hpp" +#include #ifdef LBANN_HAS_GPU constexpr El::Device Dev = El::Device::GPU; @@ -70,9 +75,10 @@ make_new_activation_layer(lbann_comm& comm, &comm); } else if (new_type == "tanh") { - layer = - std::make_unique>( - &comm); + layer = std::make_unique< + OperatorLayer>( + comm, + std::make_unique>()); } else if (new_type == "softmax") { layer = std::make_unique< @@ -96,7 +102,9 @@ make_new_activation_layer(lbann_comm& comm, } else if (new_type == "sigmoid") { layer = std::make_unique< - sigmoid_layer>(&comm); + OperatorLayer>( + comm, + std::make_unique>()); } else { LBANN_ERROR("Unknown new layer type: ", new_type); diff --git a/src/layers/activations/CMakeLists.txt b/src/layers/activations/CMakeLists.txt index 6bb4bc14beb..fc5492afc70 100644 --- a/src/layers/activations/CMakeLists.txt +++ b/src/layers/activations/CMakeLists.txt @@ -1,6 +1,5 @@ # Add the source files for this directory set_full_path(THIS_DIR_SOURCES - activations.cpp elu.cpp identity.cpp relu.cpp @@ -13,7 +12,6 @@ set_full_path(THIS_DIR_SOURCES if (LBANN_HAS_GPU) # Add the CUDA source files for this directory set_full_path(THIS_DIR_CU_SOURCES - activations.cu elu.cu relu.cu leaky_relu.cu diff --git a/src/layers/activations/activations.cpp b/src/layers/activations/activations.cpp deleted file mode 100644 index f461355b933..00000000000 --- a/src/layers/activations/activations.cpp +++ /dev/null @@ -1,168 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// - -#define LBANN_ACTIVATIONS_LAYER_INSTANTIATE -#include "lbann/layers/activations/activations.hpp" -#include "lbann/utils/entrywise_operator.hpp" - -namespace lbann { - -namespace { - -// ========================================================= -// Operator objects for entry-wise unary layers -// ========================================================= -// Note: Unary operator corresponds to forward prop step -// (\f$ y = f(x) \f$) and binary operator corresponds to -// back prop step -// (\f$ \frac{dL}{dx} = \frac{dL}{dy} f'(x) \f$). - -/** Log sigmoid operator. */ -template -struct log_sigmoid_op { - inline TensorDataType operator()(const TensorDataType& x) const { - using std::log1p; - if (x >= El::TypeTraits::Zero()) { - return -log1p(El::Exp(-x)); - } else { - return x - log1p(El::Exp(x)); - } - } - inline TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - return dy / (El::TypeTraits::One() + El::Exp(x)); - } -}; - -/** SELU operator. */ -template -struct selu_op { - inline TensorDataType operator()(const TensorDataType& x) const { - using std::expm1; - static const auto alpha = TensorDataType(1.6732632423543772848170429916717); - static const auto scale = TensorDataType(1.0507009873554804934193349852946); - static const auto zero = TensorDataType(0.); - return (x > zero ? - scale * x : - scale * alpha * expm1(x)); - } - inline TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - static const auto alpha = TensorDataType(1.6732632423543772848170429916717); - static const auto scale = TensorDataType(1.0507009873554804934193349852946); - static const auto zero = TensorDataType(0.); - return (x > zero ? - dy * scale : - dy * scale * alpha * El::Exp(x)); - } -}; - -/** Sigmoid operator. */ -template -struct sigmoid_op { - TensorDataType eps = std::numeric_limits::epsilon(); - inline TensorDataType operator()(const TensorDataType& x) const { - static const auto one = El::TypeTraits::One(); - const auto& y = one / (one + El::Exp(-x)); -#ifdef LBANN_ENABLE_SIGMOID_CUTOFF - if (y <= eps) { return eps; } - else if (y >= one - eps) { return one - eps; } -#endif // LBANN_ENABLE_SIGMOID_CUTOFF - return y; - } - inline TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - static const auto one = El::TypeTraits::One(); - const auto& y = one / (one + El::Exp(-x)); -#ifdef LBANN_ENABLE_SIGMOID_CUTOFF - if (y <= eps || y >= one - eps) { return El::TypeTraits::Zero(); } -#endif // LBANN_ENABLE_SIGMOID_CUTOFF - return dy * y * (one - y); - } -}; - -/** Softplus operator. */ -template -struct softplus_op { - inline TensorDataType operator()(const TensorDataType& x) const { - using std::log1p; - if (x > El::TypeTraits::Zero()) { - return log1p(El::Exp(-x)) + x; - } else { - return log1p(El::Exp(x)); - } - } - inline TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - return dy / (El::TypeTraits::One() + El::Exp(-x)); - } -}; - -/** Softsign operator. */ -template -struct softsign_op { - inline TensorDataType operator()(const TensorDataType& x) const { - using std::fabs; - return x / (El::TypeTraits::One() + fabs(x)); - } - inline TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - using std::fabs; - const auto& denom = El::TypeTraits::One() + fabs(x); - return dy / (denom * denom); - } -}; - -} // namespace - -// Template instantiation -#define DEFINE_COMPUTE_OPS(layer, op) \ - template \ - void layer::fp_compute() { \ - apply_entrywise_unary_operator( \ - this->get_prev_activations(), \ - this->get_activations()); \ - } \ - template \ - void layer::bp_compute() { \ - apply_entrywise_binary_operator( \ - this->get_prev_activations(), \ - this->get_prev_error_signals(), \ - this->get_error_signals()); \ - } - -DEFINE_COMPUTE_OPS(log_sigmoid_layer, log_sigmoid_op) -DEFINE_COMPUTE_OPS(selu_layer, selu_op) -DEFINE_COMPUTE_OPS(sigmoid_layer, sigmoid_op) -DEFINE_COMPUTE_OPS(softplus_layer, softplus_op) -DEFINE_COMPUTE_OPS(softsign_layer, softsign_op) - -#define PROTO(T) \ - UNARY_ETI_INST_MACRO_DEV_DT(log_sigmoid_layer, T, El::Device::CPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(selu_layer, T, El::Device::CPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(sigmoid_layer, T, El::Device::CPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(softplus_layer, T, El::Device::CPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(softsign_layer, T, El::Device::CPU) - -#define LBANN_INSTANTIATE_CPU_HALF -#include "lbann/macros/instantiate.hpp" - -} // namespace lbann diff --git a/src/layers/activations/activations.cu b/src/layers/activations/activations.cu deleted file mode 100644 index 3b17fd1bbc7..00000000000 --- a/src/layers/activations/activations.cu +++ /dev/null @@ -1,173 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// - -#define LBANN_ACTIVATIONS_LAYER_INSTANTIATE -#include "lbann/layers/activations/activations.hpp" -#include "lbann/utils/gpu/helpers.hpp" - -namespace lbann { - -namespace { - -// ========================================================= -// Operator objects for entry-wise unary layers -// ========================================================= -// Note: Unary operator corresponds to forward prop step -// (\f$ y = f(x) \f$) and binary operator corresponds to -// back prop step -// (\f$ \frac{dL}{dx} = \frac{dL}{dy} f'(x) \f$). - -/** Log sigmoid operator. */ -template -struct log_sigmoid_op { - inline __device__ TensorDataType operator()(const TensorDataType& x) const { - if (x >= TensorDataType(0.0)) { - return -gpu_lib::log1p(gpu_lib::exp(-x)); - } else { - return x - gpu_lib::log1p(gpu_lib::exp(x)); - } - } - inline __device__ TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - return dy / (TensorDataType(1.0) + gpu_lib::exp(x)); - } -}; - -/** ReLU operator. */ -template -struct relu_op { - inline __device__ TensorDataType operator()(const TensorDataType& x) const { - return gpu_lib::max(x, TensorDataType(0.0)); - } - inline __device__ TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - return x > TensorDataType(0.0) ? dy : TensorDataType(0.0); - } -}; - -/** SELU operator. */ -template -struct selu_op { - inline __device__ TensorDataType operator()(const TensorDataType& x) const { - const TensorDataType alpha = 1.6732632423543772848170429916717; - const TensorDataType scale = 1.0507009873554804934193349852946; - return (x > TensorDataType(0.0) ? - scale * x : - scale * alpha * gpu_lib::expm1(x)); - } - inline __device__ TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - const TensorDataType alpha = 1.6732632423543772848170429916717; - const TensorDataType scale = 1.0507009873554804934193349852946; - return (x > TensorDataType(0.0) ? - dy * scale : - dy * scale * alpha * gpu_lib::exp(x)); - } -}; - -/** Sigmoid operator. */ -template -struct sigmoid_op { - inline __device__ TensorDataType operator()(const TensorDataType& x) const { - const TensorDataType one = 1.; - const auto& y = one / (one + gpu_lib::exp(-x)); -#ifdef LBANN_ENABLE_SIGMOID_CUTOFF - const auto eps = gpu_lib::epsilon(); - if (y <= eps) { return eps; } - else if (y >= one - eps) { return one - eps; } -#endif // LBANN_ENABLE_SIGMOID_CUTOFF - return y; - } - inline __device__ TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - const TensorDataType one = 1.; - const auto& y = one / (one + gpu_lib::exp(-x)); -#ifdef LBANN_ENABLE_SIGMOID_CUTOFF - const auto eps = gpu_lib::epsilon(); - if (y <= eps || y >= one - eps) { return TensorDataType(0.0); } -#endif // LBANN_ENABLE_SIGMOID_CUTOFF - return dy * y * (one - y); - } -}; - -/** Softplus operator. */ -template -struct softplus_op { - inline __device__ TensorDataType operator()(const TensorDataType& x) const { - if (x > TensorDataType(0.0)) { - return gpu_lib::log1p(gpu_lib::exp(-x)) + x; - } else { - return gpu_lib::log1p(gpu_lib::exp(x)); - } - } - inline __device__ TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - return dy / (TensorDataType(1.0) + gpu_lib::exp(-x)); - } -}; - -/** Softsign operator. */ -template -struct softsign_op { - inline __device__ TensorDataType operator()(const TensorDataType& x) const { - return x / (TensorDataType(1.0) + gpu_lib::abs(x)); - } - inline __device__ TensorDataType operator()(const TensorDataType& x, const TensorDataType& dy) const { - const auto& denom = TensorDataType(1.0) + gpu_lib::abs(x); - return dy / (denom * denom); - } -}; - -} // namespace - -// Template instantiation -#define DEFINE_COMPUTE_OPS(layer, op) \ - template \ - void layer::fp_compute() { \ - gpu_lib::apply_entrywise_unary_operator( \ - this->get_prev_activations(), \ - this->get_activations()); \ - } \ - template \ - void layer::bp_compute() { \ - gpu_lib::apply_entrywise_binary_operator( \ - this->get_prev_activations(), \ - this->get_prev_error_signals(), \ - this->get_error_signals()); \ - } - -DEFINE_COMPUTE_OPS(log_sigmoid_layer, log_sigmoid_op) -DEFINE_COMPUTE_OPS(selu_layer, selu_op) -DEFINE_COMPUTE_OPS(sigmoid_layer, sigmoid_op) -DEFINE_COMPUTE_OPS(softplus_layer, softplus_op) -DEFINE_COMPUTE_OPS(softsign_layer, softsign_op) - -#define PROTO(T) \ - UNARY_ETI_INST_MACRO_DEV_DT(log_sigmoid_layer, T, El::Device::GPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(selu_layer, T, El::Device::GPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(sigmoid_layer, T, El::Device::GPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(softplus_layer, T, El::Device::GPU); \ - UNARY_ETI_INST_MACRO_DEV_DT(softsign_layer, T, El::Device::GPU) - -#define LBANN_INSTANTIATE_GPU_HALF -#include "lbann/macros/instantiate.hpp" - -} // namespace lbann diff --git a/src/layers/activations/cereal_registration/CMakeLists.txt b/src/layers/activations/cereal_registration/CMakeLists.txt index 30bf1df37d4..8d6fdfbd5bd 100644 --- a/src/layers/activations/cereal_registration/CMakeLists.txt +++ b/src/layers/activations/cereal_registration/CMakeLists.txt @@ -3,14 +3,9 @@ set_full_path(THIS_DIR_SOURCES elu.cpp identity.cpp leaky_relu.cpp - log_sigmoid.cpp log_softmax.cpp relu.cpp - selu.cpp - sigmoid.cpp softmax.cpp - softplus.cpp - softsign.cpp ) # Propagate the files up the tree diff --git a/src/layers/activations/cereal_registration/selu.cpp b/src/layers/activations/cereal_registration/selu.cpp deleted file mode 100644 index f26e6b99aa1..00000000000 --- a/src/layers/activations/cereal_registration/selu.cpp +++ /dev/null @@ -1,44 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" -#include - -namespace lbann { - -template -template -void -selu_layer -::serialize(ArchiveT& ar) { - using DataTypeLayer = data_type_layer; - ar(::cereal::make_nvp("DataTypeLayer", - ::cereal::base_class(this))); -} - -} // namespace lbann - -#define LBANN_LAYER_NAME selu_layer -#include "lbann/macros/register_layer_with_cereal.hpp" diff --git a/src/layers/activations/cereal_registration/sigmoid.cpp b/src/layers/activations/cereal_registration/sigmoid.cpp deleted file mode 100644 index 6575c4f5a5f..00000000000 --- a/src/layers/activations/cereal_registration/sigmoid.cpp +++ /dev/null @@ -1,44 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" -#include - -namespace lbann { - -template -template -void -sigmoid_layer -::serialize(ArchiveT& ar) { - using DataTypeLayer = data_type_layer; - ar(::cereal::make_nvp("DataTypeLayer", - ::cereal::base_class(this))); -} - -} // namespace lbann - -#define LBANN_LAYER_NAME sigmoid_layer -#include "lbann/macros/register_layer_with_cereal.hpp" diff --git a/src/layers/loss/CMakeLists.txt b/src/layers/loss/CMakeLists.txt index 6df4694b5fc..2e8bc59dd63 100644 --- a/src/layers/loss/CMakeLists.txt +++ b/src/layers/loss/CMakeLists.txt @@ -2,7 +2,6 @@ set_full_path(THIS_DIR_SOURCES categorical_accuracy.cpp cross_entropy.cpp - entrywise.cpp l1_norm.cpp l2_norm2.cpp mean_absolute_error.cpp @@ -15,7 +14,6 @@ if (LBANN_HAS_GPU) set_full_path(THIS_DIR_CU_SOURCES categorical_accuracy.cu cross_entropy.cu - entrywise.cu l1_norm.cu l2_norm2.cu mean_absolute_error.cu diff --git a/src/layers/loss/cereal_registration/CMakeLists.txt b/src/layers/loss/cereal_registration/CMakeLists.txt index 6029f110644..5fdecbabd21 100644 --- a/src/layers/loss/cereal_registration/CMakeLists.txt +++ b/src/layers/loss/cereal_registration/CMakeLists.txt @@ -1,16 +1,11 @@ # Add the source files for this directory set_full_path(THIS_DIR_SOURCES - binary_cross_entropy.cpp - boolean_accuracy.cpp - boolean_false_negative.cpp - boolean_false_positive.cpp categorical_accuracy.cpp cross_entropy.cpp l1_norm.cpp l2_norm2.cpp mean_absolute_error.cpp mean_squared_error.cpp - sigmoid_binary_cross_entropy.cpp top_k_categorical_accuracy.cpp ) diff --git a/src/layers/loss/entrywise.cpp b/src/layers/loss/entrywise.cpp deleted file mode 100644 index 1fabcf24c14..00000000000 --- a/src/layers/loss/entrywise.cpp +++ /dev/null @@ -1,255 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -// Produced at the Lawrence Livermore National Laboratory. -// Written by the LBANN Research Team (B. Van Essen, et al.) listed in -// the CONTRIBUTORS file. -// -// LLNL-CODE-697807. -// All rights reserved. -// -// This file is part of LBANN: Livermore Big Artificial Neural Network -// Toolkit. For details, see http://software.llnl.gov/LBANN or -// https://github.com/LLNL/LBANN. -// -// Licensed under the Apache License, Version 2.0 (the "Licensee"); you -// may not use this file except in compliance with the License. You may -// obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the license. -//////////////////////////////////////////////////////////////////////////////// - -#define LBANN_ENTRYWISE_LAYER_INSTANTIATE -#include "lbann/layers/loss/entrywise.hpp" -#include "lbann/utils/entrywise_operator.hpp" -#include "lbann/utils/numerical_traits.hpp" - -namespace lbann { - -namespace { - -/** Apply a binary backprop operator to CPU data. - * The input and output data must be on CPU and must have the same - * dimensions. Given a binary function \f$ y = f(x_1,x_2) \f$, the - * corresponding BinaryBackPropOperator is a 5-ary function with the - * arguments \f$ x_1 \f$, \f$ x_2 \f$, \f$ dL/dy \f$, \f$ dL/dx_1\f$, - * \f$ dL/dx_2 \f$. The last two arguments should be overwritten when - * the BinaryBackPropOperator is called. - */ -template