diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d5a1572 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.swp +build diff --git a/BinaryData.h b/BinaryData.h index a60c07d..15a94eb 100644 --- a/BinaryData.h +++ b/BinaryData.h @@ -2,38 +2,47 @@ #define BINARYDATA #include +#include +#include -typedef unsigned char byte +namespace libdvid { + +typedef unsigned char byte; class BinaryData; typedef boost::shared_ptr BinaryDataPtr; class BinaryData { public: - static BinaryDataPtr create_binary_data(byte* data_) + static BinaryDataPtr create_binary_data(const char* data_) { return BinaryDataPtr(new BinaryData(data_)); } - char * get_raw() + static BinaryDataPtr create_binary_data(std::ifstream& fin) { - return data; + return BinaryDataPtr(new BinaryData(fin)); } - ~BinaryData() + + byte * get_raw() { - delete data; + return (byte *)(data.c_str()); } - + std::string& get_data() + { + return data; + } + ~BinaryData() {} private: - BinaryData(byte data_) : data(data_) {} - byte data; + BinaryData(const char* data_) : data(data_) {} + BinaryData(std::ifstream& fin) + { + data.assign( (std::istreambuf_iterator(fin) ), + (std::istreambuf_iterator() ) ); + } + std::string data; }; - - - - - - +} #endif diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..ff5069c --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,92 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +project (libdvidcpp) + +include (ExternalProject) + +set (RUN_ENVIRONMENT "Workstation" CACHE TYPE STRING) +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set (CMAKE_CXX_LINK_FLAGS "-O3") +set (CMAKE_CXX_FLAGS_RELEASE "-O3") +set (CMAKE_CXX_FLAGS_DEBUG "-ggdb") +set (CMAKE_DEBUG_POSTFIX "-g") + + +################################################################################ +# Check if BUILDEM_DIR has already been assigned. If not, create a default. +set (BUILDEM_DIR "None" CACHE TYPE STRING) + +if (${BUILDEM_DIR} STREQUAL "None") + message ("WARNING: To use Buildem, Buildem directory (for all downloads & builds) should be specified via -DBUILDEM_DIR= on cmake command line.") + message ("Builds will be placed here: ${CMAKE_SOURCE_DIR}/bin") +else() + message ("FlyEM downloads and builds will be placed here: ${BUILDEM_DIR}") +endif () +############################################################################### + +if (${BUILDEM_DIR} STREQUAL "None") + set (BUILDLOC ${CMAKE_SOURCE_DIR}) +else() + set (BUILDLOC ${BUILDEM_DIR}) +endif() + +if (NOT ${BUILDEM_DIR} STREQUAL "None") + ############################################################################### + # Download and install buildem, if it isn't already in BUILDEM_DIR. + set (BUILDEM_REPO_DIR ${BUILDEM_DIR}/src/buildem) + if (NOT EXISTS ${BUILDEM_REPO_DIR}/python.cmake) + message ("Installing buildem repo...") + ExternalProject_Add(buildem + PREFIX ${BUILDEM_DIR} + GIT_REPOSITORY http://github.com/janelia-flyem/buildem.git + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "" + ) + message ("\n**********************************************************\n") + message ("\nAfter running make, you must re-run the cmake command once") + message ("buildem has been downloaded!\n") + message ("\n***********************************************************\n") + return() + endif() + ############################################################################### + + # Use modules from the downloaded buildem + set (CMAKE_MODULE_PATH ${BUILDEM_REPO_DIR}) + message("Using cmake modules from ${BUILDEM_REPO_DIR}") + + include (jsoncpp) + include (cppnetlib) +else () + FIND_PACKAGE(cppnetlib 0.11.0 REQUIRED) + include_directories (BEFORE ${CPPNETLIB_INCLUDE_DIRS}) + set (LIBDVID_EXT_LIBS ${CPPNETLIB_LIBRARIES}) +endif (NOT ${BUILDEM_DIR} STREQUAL "None") + +# set bin and lib directories +set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${BUILDLOC}/bin) +if (NOT EXISTS ${BUILDLOC}/bin) + file (MAKE_DIRECTORY ${BUILDLOC}/bin) +endif() + +set (CMAKE_LIBRARY_OUTPUT_DIRECTORY ${BUILDLOC}/lib) +set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${BUILDLOC}/lib) +if (NOT EXISTS ${BUILDLOC}/lib) + file (MAKE_DIRECTORY ${BUILDLOC}/lib) +endif() + +# Compile libdvidcpp library components +add_library (libdvidcpp SHARED DVIDNode.cpp DVIDServer.cpp) + +target_link_libraries (libdvidcpp ${LIBDVID_EXT_LIBS}) + +if (NOT ${BUILDEM_DIR} STREQUAL "None") + add_dependencies (libdvidcpp ${LIBDVID_DEPS}) +endif() + + diff --git a/DVIDException.h b/DVIDException.h index b773a01..b347b4e 100644 --- a/DVIDException.h +++ b/DVIDException.h @@ -2,20 +2,20 @@ #define DVIDEXCEPTION_H #include "Utilities.h" -#include +#include namespace libdvid { class DVIDException : public ErrMsg { public: - DVIDException(int status_) : + DVIDException(std::string msg_, int status_) : ErrMsg(msg_), status(status_) {} std::string get_msg() { std::stringstream sstr; - sstr << status << ": " << ErrMsg::get_msg(); - return sstr.string(); + sstr << "DVID Error (" << status << "): " << ErrMsg::get_msg(); + return sstr.str(); } private: int status; diff --git a/DVIDNode.cpp b/DVIDNode.cpp index 64baf43..68bef53 100644 --- a/DVIDNode.cpp +++ b/DVIDNode.cpp @@ -1 +1,198 @@ +#include "DVIDNode.h" #include "DVIDException.h" +#include +#include +#include + +using std::ifstream; using std::set; using std::stringstream; +Json::Reader json_reader; + +using namespace boost::network; +using namespace boost::network::http; + +namespace libdvid { + +DVIDNode::DVIDNode(DVIDServer web_addr_, UUID uuid_) : + web_addr(web_addr_), uuid(uuid_) +{ + client::request requestobj(web_addr.get_uri_root() + "node/" + uuid + "/info"); + requestobj << header("Connection", "close"); + client request_client; + client::response respdata = request_client.get(requestobj); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } +} + +void DVIDNode::create_keyvalue(std::string keyvalue) +{ + client::request requestobj(web_addr.get_uri_root() + "dataset/" + uuid + + "/new/keyvalue" + keyvalue ); + requestobj << header("Connection", "close"); + client request_client; + + std::string data("{}"); + client::response respdata = request_client.post(requestobj, + "application/json", data); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } +} + +void DVIDNode::put(std::string keyvalue, std::string key, BinaryDataPtr value) +{ + client::request requestobj(web_addr.get_uri_root() + "node/" + uuid + + "/" + keyvalue + "/" + key); + requestobj << header("Connection", "close"); + client request_client; + + client::response respdata = request_client.post(requestobj, + "application/octet-stream", value->get_data()); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } +} + +void DVIDNode::put(std::string keyvalue, std::string key, ifstream& fin) +{ + BinaryDataPtr data = BinaryData::create_binary_data(fin); + put(keyvalue, key, data); +} + + +void DVIDNode::put(std::string keyvalue, std::string key, Json::Value& data) +{ + stringstream datastr; + datastr << data; + BinaryDataPtr bdata = BinaryData::create_binary_data(datastr.str().c_str()); + put(keyvalue, key, bdata); +} + + +void DVIDNode::get(std::string keyvalue, std::string key, BinaryDataPtr& value) +{ + client::request requestobj(web_addr.get_uri_root() + "node/" + uuid + + "/" + keyvalue + "/" + key); + requestobj << header("Connection", "close"); + client request_client; + client::response respdata = request_client.get(requestobj); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } + std::string data = body(respdata); + // ?! allow intialization to happen in constructor + value = BinaryData::create_binary_data(data.c_str()); +} + +void DVIDNode::get(std::string keyvalue, std::string key, Json::Value& data) +{ + BinaryDataPtr binary; + get(keyvalue, key, binary); + + Json::Reader json_reader; + if (!json_reader.parse(binary->get_data(), data)) { + throw ErrMsg("Could not decode JSON"); + } +} + +void DVIDNode::get_gray_slice(std::string datatype_instance, tuple start, + tuple sizes, tuple channels, DVIDGrayPtr& gray) +{ + std::string volume; + retrieve_volume(datatype_instance, start, sizes, channels, volume); + gray = DVIDVoxels::get_dvid_voxels(volume); +} + +void DVIDNode::get_label_slice(std::string datatype_instance, tuple start, + tuple sizes, tuple channels, DVIDLabelPtr& labels) +{ + std::string volume; + retrieve_volume(datatype_instance, start, sizes, channels, volume); + labels = DVIDVoxels::get_dvid_voxels(volume); +} + +void DVIDNode::write_label_slice(std::string datatype_instance, tuple start, + tuple sizes, tuple channels, BinaryDataPtr data) +{ + client::request requestobj(construct_volume_uri( + datatype_instance, start, sizes, channels)); + requestobj << header("Connection", "close"); + client request_client; + client::response respdata = request_client.post(requestobj, + "application/octet-stream", data->get_data()); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } +} + +std::string DVIDNode::construct_volume_uri(std::string datatype_inst, tuple start, tuple sizes, tuple channels) +{ + std::string uri = web_addr.get_uri_root() + "node/" + uuid + "/" + + datatype_inst + "/raw/"; + + if (start.size() < 3) { + throw ErrMsg("libdvid does not support 2D datatype instances"); + } + if (channels.size() == 0) { + throw ErrMsg("must specify more than one channel"); + } + if (sizes.size() != channels.size()) { + throw ErrMsg("number of size dimensions does not match the number of channels"); + } + stringstream sstr; + sstr << uri; + sstr << channels[0]; + + // retrieve at least a 3D volume + set used_channels; + for (int i = 0; i < channels.size(); ++i) { + used_channels.insert(channels[i]); + } + int channel_id = 0; + for (int i = channels.size(); i < 3; ++i) { + while (used_channels.find(channel_id) != used_channels.end()) { + ++channel_id; + } + channels.push_back(channel_id); + } + + for (int i = 1; i < channels.size(); ++i) { + sstr << "_" << channels[i]; + } + + // retrieve at least a 3D volume + for (int i = sizes.size(); i < 3; ++i) { + sizes.push_back(1); + } + sstr << "/" << sizes[0]; + for (int i = 1; i < sizes.size(); ++i) { + sstr << "_" << sizes[i]; + } + sstr << "/" << start[0]; + for (int i = 1; i < start.size(); ++i) { + sstr << "_" << start[i]; + } + + return sstr.str(); +} + +void DVIDNode::retrieve_volume(std::string datatype_inst, tuple start, tuple sizes, tuple channels, std::string& volume) +{ + client::request requestobj(construct_volume_uri(datatype_inst, start, sizes, channels)); + requestobj << header("Connection", "close"); + client request_client; + client::response respdata = request_client.get(requestobj); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } + volume = body(respdata); +} + +} + diff --git a/DVIDNode.h b/DVIDNode.h index bcb092c..38a3771 100644 --- a/DVIDNode.h +++ b/DVIDNode.h @@ -1,11 +1,13 @@ #ifndef DVIDNODE_H #define DVIDNODE_H -#include -#include #include "BinaryData.h" #include "DVIDServer.h" #include "DVIDVoxels.h" +#include "Utilities.h" +#include +#include +#include namespace libdvid { @@ -13,19 +15,17 @@ typedef std::string UUID; class DVIDNode { public: - // ?! check that node is available -- user server interface - DVIDNode(DVIDServer web_addr_, UUID uuid_) : - web_addr(web_addr_), uuid(uuid_) {} + // check that node is available + DVIDNode(DVIDServer web_addr_, UUID uuid_); // throw error if start point is 2D - void get_gray_slice(std::string name, Coords start, Size size, - Channels channels, DVIDGrayPtr& gray); - void get_label_slice(std::string name, Coords start, Size size, - Channels channels, DVIDLabelPtr& labels); + void get_gray_slice(std::string datatype_instance, tuple start, + tuple sizes, tuple channels, DVIDGrayPtr& gray); + void get_label_slice(std::string datatype_instance, tuple start, + tuple sizes, tuple channels, DVIDLabelPtr& labels); - void write_label_slice(std::string name, Coords start, Size size, - Channels channels); - + void write_label_slice(std::string datatype_instance, tuple start, + tuple sizes, tuple channels, BinaryDataPtr data); // Key-Value Interface @@ -33,7 +33,7 @@ class DVIDNode { void create_keyvalue(std::string keyvalue); void put(std::string keyvalue, std::string key, BinaryDataPtr value); - void put(std::string keyvalue, std::string key, ifstream& fin); + void put(std::string keyvalue, std::string key, std::ifstream& fin); void put(std::string keyvalue, std::string key, Json::Value& data); void get(std::string keyvalue, std::string key, BinaryDataPtr& value); @@ -43,6 +43,9 @@ class DVIDNode { UUID uuid; DVIDServer web_addr; + std::string construct_volume_uri(std::string datatype_inst, tuple start, tuple sizes, tuple channels); + void retrieve_volume(std::string datatype_inst, tuple start, tuple sizes, tuple channels, std::string& volume); + // ?! maybe add node meta data ?? -- but probably make it on demand }; diff --git a/DVIDServer.cpp b/DVIDServer.cpp index 73347ad..554c341 100644 --- a/DVIDServer.cpp +++ b/DVIDServer.cpp @@ -1,14 +1,22 @@ #include "DVIDServer.h" +#include "DVIDException.h" +#include -using std::string; +using namespace boost::network; +using namespace boost::network::http; namespace libdvid { -DVIDServer::DVIDServer(string addr_) : addr(addr_) +DVIDServer::DVIDServer(std::string addr_) : addr(addr_) { - - - + client::request requestobj(get_uri_root() + "info"); + requestobj << header("Connection", "close"); + client request_client; + client::response respdata = request_client.get(requestobj); + int status_code = status(respdata); + if (status_code != 200) { + throw DVIDException(body(respdata), status_code); + } } } diff --git a/DVIDServer.h b/DVIDServer.h index f9638dc..a5ff1f4 100644 --- a/DVIDServer.h +++ b/DVIDServer.h @@ -12,6 +12,10 @@ class DVIDServer { { return addr; } + std::string get_uri_root() + { + addr + "/api/"; + } private: std::string addr; diff --git a/DVIDVoxels.h b/DVIDVoxels.h index 7f97d12..5a0d563 100644 --- a/DVIDVoxels.h +++ b/DVIDVoxels.h @@ -2,6 +2,9 @@ #define DVIDVOXELS_H #include +#include + +namespace libdvid { template class DVIDVoxels { @@ -10,9 +13,14 @@ class DVIDVoxels { { return boost::shared_ptr >(new DVIDVoxels(array_)); } + static boost::shared_ptr > get_dvid_voxels(std::string& data_str) + { + return boost::shared_ptr >(new DVIDVoxels(data_str)); + } + ~DVIDVoxels() { - delete array; + delete []array; } T* get_raw() { @@ -21,9 +29,23 @@ class DVIDVoxels { private: DVIDVoxels(T* array_) : array(array_) {} + DVIDVoxels(std::string& data_str) + { + const char * byte_array = data_str.c_str(); + int incr = sizeof(T); + array = new T[data_str.size()]; + for (int i = 0, pos = 0; i < data_str.size(); i+=incr, pos++) { + array[pos] = T(byte_array[i]); + } + } + T* array; }; typedef boost::shared_ptr > DVIDGrayPtr; typedef boost::shared_ptr > DVIDLabelPtr; + +} + +#endif diff --git a/Utilities.h b/Utilities.h index b32a12e..63a5062 100644 --- a/Utilities.h +++ b/Utilities.h @@ -1,9 +1,11 @@ #ifndef UTILITIES_H #define UTILITIES_H -namespace libdvid { - #include +#include +#include + +namespace libdvid { class ErrMsg { public: @@ -15,8 +17,7 @@ class ErrMsg { private: std::string msg; -} - +}; std::ostream& operator<<(std::ostream& os, ErrMsg& err) { @@ -24,6 +25,9 @@ std::ostream& operator<<(std::ostream& os, ErrMsg& err) return os; } +// e.g., dim1, dim2, dim3; x, y, z +typedef std::vector tuple; + } #endif diff --git a/todo b/todo index 0ddba79..a6f4d13 100644 --- a/todo +++ b/todo @@ -1,11 +1,16 @@ --make dvid server and check for existence (info?) --make dvid node and check for existence --create coords, channels, and size utilities --make DVIDVoxels generic type (initialization from string) -- just use c_str casted to template -- prevent 2D start, convert all 2D slices to 3D (2D x 1) --implement get/put interface (especially with JSON) --deal with binary data object since it is really a string -- just create string from char *? -- what about unnecessary copying -- figure this stuff out --create buildem setup --make an example taking dvid server, uuid, and optional +X-make dvid server and check for existence (info?) + +X-make dvid node and check for existence +X-create coords, channels, and size utilities +X-make DVIDVoxels generic type (initialization from string) -- just use c_str casted to template -- prevent 2D start, convert all 2D slices to 3D (2D x 1) + +X-implement get/put interface (especially with JSON) +X-deal with binary data object since it is really a string -- just create string from char *? -- what about unnecessary copying -- figure this stuff out + +-create buildem setup (buildem library module, standalone build, handle includes) +-make an example taking dvid server, uuid, and option +-integration tests?; extensive debug +-documentation