Skip to content

Commit

Permalink
Cherry-picked USE_HDF5 from Android branch
Browse files Browse the repository at this point in the history
  • Loading branch information
jerryz123 committed Apr 16, 2018
1 parent 3066720 commit 3318a46
Show file tree
Hide file tree
Showing 16 changed files with 79 additions and 1 deletion.
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ caffe_option(USE_LMDB "Build with lmdb" ON)
caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)
caffe_option(USE_OPENMP "Link with OpenMP (when your BLAS wants OpenMP and you get linker errors)" OFF)

# This code is taken from https://github.com/sh1r0/caffe-android-lib
caffe_option(USE_HDF5 "Build with hdf5" ON)

# ---[ Dependencies
include(cmake/Dependencies.cmake)

Expand Down
12 changes: 11 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -178,11 +178,13 @@ ifneq ($(CPU_ONLY), 1)
LIBRARIES := cudart cublas curand
endif

LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5
LIBRARIES += glog gflags protobuf boost_system boost_filesystem m

# handle IO dependencies
USE_LEVELDB ?= 1
USE_LMDB ?= 1
# This code is taken from https://github.com/sh1r0/caffe-android-lib
USE_HDF5 ?= 1
USE_OPENCV ?= 1

ifeq ($(USE_LEVELDB), 1)
Expand All @@ -191,6 +193,10 @@ endif
ifeq ($(USE_LMDB), 1)
LIBRARIES += lmdb
endif
# This code is taken from https://github.com/sh1r0/caffe-android-lib
ifeq ($(USE_HDF5), 1)
LIBRARIES += hdf5_hl hdf5
endif
ifeq ($(USE_OPENCV), 1)
LIBRARIES += opencv_core opencv_highgui opencv_imgproc

Expand Down Expand Up @@ -347,6 +353,10 @@ ifeq ($(ALLOW_LMDB_NOLOCK), 1)
COMMON_FLAGS += -DALLOW_LMDB_NOLOCK
endif
endif
# This code is taken from https://github.com/sh1r0/caffe-android-lib
ifeq ($(USE_HDF5), 1)
COMMON_FLAGS += -DUSE_HDF5
endif

# CPU-only configuration
ifeq ($(CPU_ONLY), 1)
Expand Down
2 changes: 2 additions & 0 deletions Makefile.config.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
# USE_OPENCV := 0
# USE_LEVELDB := 0
# USE_LMDB := 0
# This code is taken from https://github.com/sh1r0/caffe-android-lib
# USE_HDF5 := 0

# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary)
# You should not set this flag if you will be reading LMDBs with any
Expand Down
12 changes: 12 additions & 0 deletions cmake/ConfigGen.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@ function(caffe_generate_export_configs)
set(HAVE_CUDA FALSE)
endif()

set(HDF5_IMPORTED OFF)
foreach(_lib ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
if(TARGET ${_lib})
set(HDF5_IMPORTED ON)
endif()
endforeach()

# This code is taken from https://github.com/sh1r0/caffe-android-lib
if(USE_HDF5)
list(APPEND Caffe_DEFINITIONS -DUSE_HDF5)
endif()

if(NOT HAVE_CUDNN)
set(HAVE_CUDNN FALSE)
endif()
Expand Down
8 changes: 8 additions & 0 deletions cmake/Dependencies.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ find_package(HDF5 COMPONENTS HL REQUIRED)
list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${HDF5_INCLUDE_DIRS})
list(APPEND Caffe_LINKER_LIBS PUBLIC ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})

# This code is taken from https://github.com/sh1r0/caffe-android-lib
if(USE_HDF5)
find_package(HDF5 COMPONENTS HL REQUIRED)
include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
add_definitions(-DUSE_HDF5)
endif()

# ---[ LMDB
if(USE_LMDB)
find_package(LMDB REQUIRED)
Expand Down
2 changes: 2 additions & 0 deletions cmake/Summary.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ function(caffe_print_configuration_summary)
caffe_status(" USE_LMDB : ${USE_LMDB}")
caffe_status(" USE_NCCL : ${USE_NCCL}")
caffe_status(" ALLOW_LMDB_NOLOCK : ${ALLOW_LMDB_NOLOCK}")
# This code is taken from https://github.com/sh1r0/caffe-android-lib
caffe_status(" USE_HDF5 : ${USE_HDF5}")
caffe_status("")
caffe_status("Dependencies:")
caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")
Expand Down
2 changes: 2 additions & 0 deletions include/caffe/util/hdf5.hpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#ifndef CAFFE_UTIL_HDF5_H_
#define CAFFE_UTIL_HDF5_H_

Expand Down Expand Up @@ -37,3 +38,4 @@ string hdf5_get_name_by_idx(hid_t loc_id, int idx);
} // namespace caffe

#endif // CAFFE_UTIL_HDF5_H_
#endif // USE_HDF5
2 changes: 2 additions & 0 deletions src/caffe/layers/hdf5_data_layer.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
/*
TODO:
- load file in a separate thread ("prefetch")
Expand Down Expand Up @@ -184,3 +185,4 @@ INSTANTIATE_CLASS(HDF5DataLayer);
REGISTER_LAYER_CLASS(HDF5Data);

} // namespace caffe
#endif // USE_HDF5
2 changes: 2 additions & 0 deletions src/caffe/layers/hdf5_data_layer.cu
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
/*
TODO:
- only load parts of the file, in accordance with a prototxt param "max_mem"
Expand Down Expand Up @@ -34,3 +35,4 @@ void HDF5DataLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
INSTANTIATE_LAYER_GPU_FUNCS(HDF5DataLayer);

} // namespace caffe
#endif // USE_HDF5
2 changes: 2 additions & 0 deletions src/caffe/layers/hdf5_output_layer.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#include <vector>

#include "hdf5.h"
Expand Down Expand Up @@ -72,3 +73,4 @@ INSTANTIATE_CLASS(HDF5OutputLayer);
REGISTER_LAYER_CLASS(HDF5Output);

} // namespace caffe
#endif // USE_HDF5
2 changes: 2 additions & 0 deletions src/caffe/layers/hdf5_output_layer.cu
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#include <vector>

#include "hdf5.h"
Expand Down Expand Up @@ -37,3 +38,4 @@ void HDF5OutputLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
INSTANTIATE_LAYER_GPU_FUNCS(HDF5OutputLayer);

} // namespace caffe
#endif // USE_HDF5
13 changes: 13 additions & 0 deletions src/caffe/net.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
#include <utility>
#include <vector>

#ifdef USE_HDF5
#include "hdf5.h"
#endif // USE_HDF5

#include "caffe/common.hpp"
#include "caffe/layer.hpp"
Expand Down Expand Up @@ -786,6 +788,7 @@ void Net<Dtype>::CopyTrainedLayersFromBinaryProto(

template <typename Dtype>
void Net<Dtype>::CopyTrainedLayersFromHDF5(const string trained_filename) {
#ifdef USE_HDF5
hid_t file_hid = H5Fopen(trained_filename.c_str(), H5F_ACC_RDONLY,
H5P_DEFAULT);
CHECK_GE(file_hid, 0) << "Couldn't open " << trained_filename;
Expand Down Expand Up @@ -832,6 +835,10 @@ void Net<Dtype>::CopyTrainedLayersFromHDF5(const string trained_filename) {
}
H5Gclose(data_hid);
H5Fclose(file_hid);
#else
LOG(FATAL) << "CopyTrainedLayersFromHDF5 requires hdf5;"
<< " compile with USE_HDF5.";
#endif // USE_HDF5
}

template <typename Dtype>
Expand All @@ -848,6 +855,8 @@ void Net<Dtype>::ToProto(NetParameter* param, bool write_diff) const {

template <typename Dtype>
void Net<Dtype>::ToHDF5(const string& filename, bool write_diff) const {
// This code is taken from https://github.com/sh1r0/caffe-android-lib
#ifdef USE_HDF5
hid_t file_hid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT);
CHECK_GE(file_hid, 0)
Expand Down Expand Up @@ -901,6 +910,10 @@ void Net<Dtype>::ToHDF5(const string& filename, bool write_diff) const {
H5Gclose(diff_hid);
}
H5Fclose(file_hid);
// This code is taken from https://github.com/sh1r0/caffe-android-lib
#else
LOG(FATAL) << "ToHDF5 requires hdf5; compile with USE_HDF5.";
#endif // USE_HDF5
}

template <typename Dtype>
Expand Down
12 changes: 12 additions & 0 deletions src/caffe/solvers/sgd_solver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ void SGDSolver<Dtype>::SnapshotSolverStateToBinaryProto(
template <typename Dtype>
void SGDSolver<Dtype>::SnapshotSolverStateToHDF5(
const string& model_filename) {
// This code is taken from https://github.com/sh1r0/caffe-android-lib
#ifdef USE_HDF5
string snapshot_filename =
Solver<Dtype>::SnapshotFilename(".solverstate.h5");
LOG(INFO) << "Snapshotting solver state to HDF5 file " << snapshot_filename;
Expand All @@ -306,6 +308,11 @@ void SGDSolver<Dtype>::SnapshotSolverStateToHDF5(
}
H5Gclose(history_hid);
H5Fclose(file_hid);
// This code is taken from https://github.com/sh1r0/caffe-android-lib
#else
LOG(FATAL) << "SnapshotSolverStateToHDF5 requires hdf5;"
<< " compile with USE_HDF5.";
#endif // USE_HDF5
}

template <typename Dtype>
Expand All @@ -330,6 +337,7 @@ void SGDSolver<Dtype>::RestoreSolverStateFromBinaryProto(

template <typename Dtype>
void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) {
#ifdef USE_HDF5
hid_t file_hid = H5Fopen(state_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
CHECK_GE(file_hid, 0) << "Couldn't open solver state file " << state_file;
this->iter_ = hdf5_load_int(file_hid, "iter");
Expand All @@ -351,6 +359,10 @@ void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) {
}
H5Gclose(history_hid);
H5Fclose(file_hid);
#else
LOG(FATAL) << "RestoreSolverStateFromHDF5 requires hdf5;"
<< " compile with USE_HDF5.";
#endif // USE_HDF5
}

INSTANTIATE_CLASS(SGDSolver);
Expand Down
2 changes: 2 additions & 0 deletions src/caffe/test/test_hdf5_output_layer.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#include <string>
#include <vector>

Expand Down Expand Up @@ -120,3 +121,4 @@ TYPED_TEST(HDF5OutputLayerTest, TestForward) {
}

} // namespace caffe
#endif // USE_HDF5
2 changes: 2 additions & 0 deletions src/caffe/test/test_hdf5data_layer.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#include <string>
#include <vector>

Expand Down Expand Up @@ -163,3 +164,4 @@ TYPED_TEST(HDF5DataLayerTest, TestSkip) {
}

} // namespace caffe
#endif // USE_HDF5
2 changes: 2 additions & 0 deletions src/caffe/util/hdf5.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#include "caffe/util/hdf5.hpp"

#include <string>
Expand Down Expand Up @@ -207,3 +208,4 @@ string hdf5_get_name_by_idx(hid_t loc_id, int idx) {
}

} // namespace caffe
#endif // USE_HDF5

0 comments on commit 3318a46

Please sign in to comment.