Merge pull request #222 from European-XFEL/dev

0.8.4 release

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "thirdparty/xsimd"]
 	path = thirdparty/xsimd
 	url = https://github.com/QuantStack/xsimd.git
+[submodule "thirdparty/xtensor-blas"]
+	path = thirdparty/xtensor-blas
+	url = https://github.com/xtensor-stack/xtensor-blas.git

.travis.yml

@@ -86,19 +86,21 @@ before_install:
 install:
   - conda install -y python=$PYTHON_VERSION
   - echo $PYTHON_VERSION
-  - conda install -y cmake -c conda-forge
+  - conda install -y cmake libstdcxx-ng numpy -c anaconda
+  - conda install -y -c conda-forge tbb-devel
+  - export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
   - pip install -e .[test]
 before_script:
   - /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1400x900x24
   - sleep 3
 script:
   # test parallel version
-  - python setup.py build_ext --with-tests
+  - python setup.py build_ext --inplace --with-tests
   - python setup.py test -v
   - python setup.py benchmark -v
 
   # test series version
   - export BUILD_SERIAL_FOAM=1
-  - python setup.py build_ext --with-tests
+  - python setup.py build_ext --inplace --with-tests
   - python setup.py test -v
   - python setup.py benchmark -v

CMakeLists.txt

@@ -4,14 +4,7 @@
 # All rights reserved.                                            #
 ###################################################################
 
-cmake_minimum_required(VERSION 3.12)
-
-if(${CMAKE_VERSION} VERSION_LESS 3.13)
-    cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
-else()
-    cmake_policy(VERSION 3.13)
-endif()
-
+cmake_minimum_required(VERSION 3.13)
 project(extra-foam LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 14)
@@ -25,12 +18,20 @@ message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
-OPTION(FOAM_WITH_TBB "Build extra-foam (xtensor is not included) with intel TBB" ON)
+# options
+# =======
+
+OPTION(FOAM_USE_TBB "Build extra-foam (xtensor is not included) with intel TBB" OFF)
 
-OPTION(FOAM_WITH_XSIMD "Build extra-foam (xtensor is not included) with XSIMD" ON)
+OPTION(FOAM_USE_XSIMD "Build extra-foam (xtensor is not included) with XSIMD" OFF)
 
 OPTION(BUILD_FOAM_TESTS "Build c++ unit test" OFF)
 
+OPTION(BUILD_FOAM_PYTHON "Build FOAM Python bindings" OFF)
+
+# dependencies
+# ============
+
 set(thirdparty_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/thirdparty)
 
 function(setup_external_project NAME)
@@ -54,49 +55,50 @@ function(setup_external_project NAME)
 endfunction()
 
 # tbb
-if(FOAM_WITH_TBB OR XTENSOR_USE_TBB)
-    set(tbb_REQUIRED_VERSION 2019_U9)
-
-    configure_file(cmake/downloadTbb.cmake.in ${thirdparty_BINARY_DIR}/tbb-stage/CMakeLists.txt)
-    setup_external_project( tbb )
-
-    include(${thirdparty_BINARY_DIR}/tbb-src/cmake/TBBBuild.cmake)
-    tbb_build(TBB_ROOT ${thirdparty_BINARY_DIR}/tbb-src
-              CONFIG_DIR TBB_DIR
-              MAKE_ARGS tbb_build_dir=${thirdparty_BINARY_DIR}/tbb-build
-                        tbb_build_prefix=tbb_cmake_build)
-    find_package(TBB CONFIG REQUIRED COMPONENTS tbb tbbmalloc)
-    message(STATUS "Found TBB: ${TBB_VERSION_MAJOR}.${TBB_VERSION_MINOR}")
-
-    get_target_property(TBB_INCLUDE_DIRS TBB::tbb INTERFACE_INCLUDE_DIRECTORIES)
-    get_target_property(TBB_LIBRARIES TBB::tbb IMPORTED_LOCATION_RELEASE)
+if(FOAM_USE_TBB OR XTENSOR_USE_TBB)
+    set(TBB_REQUIRED_VERSION 2020.1)
+    string(REPLACE "." "_U" TBB_REQUIRED_VERSION_STR ${TBB_REQUIRED_VERSION})
+
+    find_package(TBB QUIET)
+    if(TBB_FOUND AND ${TBB_VERSION_MAJOR}.${TBB_VERSION_MINOR} VERSION_EQUAL TBB_REQUIRED_VERSION)
+        set(TBB_LIBRARY ${TBB_LIBRARIES_RELEASE})
+    else()
+        message(STATUS "Downloading TBB ${TBB_REQUIRED_VERSION_STR}")
+
+        configure_file(cmake/downloadTbb.cmake.in ${thirdparty_BINARY_DIR}/tbb-stage/CMakeLists.txt)
+        setup_external_project( tbb )
+
+        include(${thirdparty_BINARY_DIR}/tbb-src/cmake/TBBBuild.cmake)
+        tbb_build(TBB_ROOT ${thirdparty_BINARY_DIR}/tbb-src
+                  CONFIG_DIR TBB_DIR
+                  MAKE_ARGS tbb_build_dir=${thirdparty_BINARY_DIR}/tbb-build
+                            tbb_build_prefix=tbb_cmake_build)
+        find_package(TBB ${TBB_REQUIRED_VERSION} EXACT CONFIG REQUIRED COMPONENTS tbb tbbmalloc)
+
+        get_target_property(TBB_INCLUDE_DIRS TBB::tbb INTERFACE_INCLUDE_DIRECTORIES)
+        get_target_property(TBB_LIBRARY TBB::tbb IMPORTED_LOCATION_RELEASE)
+    endif()
 
-    get_filename_component(TBB_LIB_NAME ${TBB_LIBRARIES} NAME)
-    configure_file(${TBB_LIBRARIES} ${CMAKE_CURRENT_BINARY_DIR}/${TBB_LIB_NAME} COPYONLY)
+    message(STATUS "TBB FOUND ${TBB_VERSION_MAJOR}.${TBB_VERSION_MINOR}: ${TBB_INCLUDE_DIRS}, ${TBB_LIBRARY}")
 
-    if(FOAM_WITH_TBB)
+    if(FOAM_USE_TBB)
         message(STATUS "Build extra-foam with intel TBB")
-        add_compile_definitions(FOAM_WITH_TBB)
+        add_compile_definitions(FOAM_USE_TBB)
     endif()
 endif()
 
 # xsimd
-if(FOAM_WITH_XSIMD OR XTENSOR_USE_XSIMD)
+if(FOAM_USE_XSIMD OR XTENSOR_USE_XSIMD)
     add_subdirectory(thirdparty/xsimd)
     set(xsimd_DIR ${PROJECT_BINARY_DIR}/thirdparty/xsimd)
 
-    if(FOAM_WITH_XSIMD)
+    if(FOAM_USE_XSIMD)
         message(STATUS "Build extra-foam with XSIMD")
-        add_compile_definitions(FOAM_WITH_XSIMD)
+        add_compile_definitions(FOAM_USE_XSIMD)
     endif()
 
 endif()
 
-# pybind11
-set(PYBIND11_CPP_STANDARD -std=c++14)
-add_subdirectory(thirdparty/pybind11)
-find_package(pybind11 REQUIRED)
-
 # xtl
 add_subdirectory(thirdparty/xtl)
 set(xtl_DIR ${CMAKE_BINARY_DIR}/thirdparty/xtl)
@@ -105,14 +107,95 @@ set(xtl_DIR ${CMAKE_BINARY_DIR}/thirdparty/xtl)
 add_subdirectory(thirdparty/xtensor)
 set(xtensor_DIR ${CMAKE_BINARY_DIR}/thirdparty/xtensor)
 
-# xtensor-python
-add_subdirectory(thirdparty/xtensor-python)
+# xtensor-blas
+find_package(BLAS REQUIRED)
+find_package(LAPACK REQUIRED)
+message(STATUS "BLAS VENDOR:    " ${BLA_VENDOR})
+message(STATUS "BLAS LIBRARIES: " ${BLAS_LIBRARIES})
+add_subdirectory(thirdparty/xtensor-blas)
+
+# EXtra-foam
+set(FOAM_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/extra_foam/include)
+
+set(FOAM_HEADERS
+    ${FOAM_INCLUDE_DIR}/f_canny.hpp
+    ${FOAM_INCLUDE_DIR}/f_geometry.hpp
+    ${FOAM_INCLUDE_DIR}/f_geometry_1m.hpp
+    ${FOAM_INCLUDE_DIR}/f_helpers.hpp
+    ${FOAM_INCLUDE_DIR}/f_imageproc.hpp
+    ${FOAM_INCLUDE_DIR}/f_smooth.hpp
+    ${FOAM_INCLUDE_DIR}/f_statistics.hpp
+    ${FOAM_INCLUDE_DIR}/f_traits.hpp
+    ${FOAM_INCLUDE_DIR}/f_utilities.hpp
+    )
+
+add_library(${PROJECT_NAME} INTERFACE)
 
-# extra-foam
-set(FOAM_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src/extra_foam/include)
+target_include_directories(${PROJECT_NAME}
+    INTERFACE
+        $<BUILD_INTERFACE:${FOAM_INCLUDE_DIR}>
+        $<INSTALL_INTERFACE:include>)
 
-add_subdirectory(src/extra_foam)
+target_link_libraries(${PROJECT_NAME}
+    INTERFACE
+        xtensor
+        xtensor-blas
+        $<$<BOOL:${FOAM_USE_XSIMD}>:xsimd>)
 
 if(BUILD_FOAM_TESTS)
     add_subdirectory(test)
 endif()
+
+# ============
+# Installation
+# ============
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+
+install(FILES ${FOAM_HEADERS}
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})
+
+install(TARGETS ${PROJECT_NAME}
+        EXPORT ${PROJECT_NAME}-targets)
+
+# Makes the project importable from the build directory
+export(EXPORT ${PROJECT_NAME}-targets
+       FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake")
+
+configure_package_config_file(
+    ${PROJECT_NAME}Config.cmake.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
+)
+
+write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+    VERSION ${PROJECT_NAME}_VERSION
+    COMPATIBILITY AnyNewerVersion
+)
+
+install(EXPORT ${PROJECT_NAME}-targets
+        FILE ${PROJECT_NAME}Targets.cmake
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+
+# ============
+# FOAM_PYTHON
+# ============
+
+if(BUILD_FOAM_PYTHON)
+    # pybind11
+    set(PYBIND11_CPP_STANDARD -std=c++14)
+    add_subdirectory(thirdparty/pybind11)
+    find_package(pybind11 REQUIRED)
+
+    # xtensor-python
+    add_subdirectory(thirdparty/xtensor-python)
+
+    # extra-foam Python
+    add_subdirectory(src/extra_foam)
+endif()

README.md

@@ -1,8 +1,15 @@
 EXtra-foam
 ==========
 
+[![Lates Release](https://img.shields.io/github/v/release/European-XFEL/EXtra-foam)](https://github.com/European-XFEL/EXtra-foam/releases)
+[![License](https://img.shields.io/github/license/European-XFEL/EXtra-foam)](https://github.com/European-XFEL/EXtra-foam/releases)
 [![Build Status](https://travis-ci.org/European-XFEL/EXtra-foam.svg?branch=master)](https://travis-ci.org/European-XFEL/EXtra-foam)
+[![Documentation](https://img.shields.io/readthedocs/extra-foam)](https://extra-foam.readthedocs.io/en/latest/)
+[![Documentation](https://img.shields.io/badge/documentation-online-blue)](https://extra-foam.readthedocs.io/en/latest/)
 
+![Language](https://img.shields.io/badge/language-python-blue)
+![Language](https://img.shields.io/badge/language-c++-red)
+[![Qt 5](https://img.shields.io/badge/Qt-5-brightgreen)](https://doc.qt.io/qt-5/)
 
 *EXtra-foam* (previously known as *[karaboFAI](https://in.xfel.eu/readthedocs/docs/karabofai/en/latest/)*) is a 
 framework that provides super fast on-line (real-time) and off-line data analysis and visualization for 

benchmarks/benchmark_geometry.py

@@ -12,8 +12,12 @@
 
 import numpy as np
 
+from extra_foam.config import config
 
-_data_sources = [(np.uint16, 'raw'), (np.float32, 'calibrated')]
+_IMAGE_DTYPE = config['SOURCE_PROC_IMAGE_DTYPE']
+_RAW_IMAGE_DTYPE = config['SOURCE_RAW_IMAGE_DTYPE']
+
+_data_sources = [(_RAW_IMAGE_DTYPE, 'raw'), (_IMAGE_DTYPE, 'calibrated')]
 
 _geom_path = osp.join(osp.dirname(osp.abspath(__file__)), "../extra_foam/geometries")
 
@@ -27,12 +31,23 @@ def _benchmark_1m_imp(geom_fast_cls, geom_cls, geom_file, quad_positions=None):
                            geom_fast_cls.module_shape[0],
                            geom_fast_cls.module_shape[1]), dtype=from_dtype)
 
+        # assemble with geometry and quad position in EXtra-geom
+
+        if quad_positions is not None:
+            geom = geom_cls.from_h5_file_and_quad_positions(geom_file, quad_positions)
+        else:
+            geom = geom_cls.from_crystfel_geom(geom_file)
+        assembled = geom.output_array_for_position_fast((n_pulses,), dtype=_IMAGE_DTYPE)
+        t0 = time.perf_counter()
+        geom.position_all_modules(modules, out=assembled)
+        dt_geom = time.perf_counter() - t0
+
         # stack only
 
         geom = geom_fast_cls()
-        out = np.full((n_pulses, *geom.assembledShape()), np.nan, dtype=np.float32)
+        assembled = np.full((n_pulses, *geom.assembledShape()), np.nan, dtype=_IMAGE_DTYPE)
         t0 = time.perf_counter()
-        geom.position_all_modules(modules, out)
+        geom.position_all_modules(modules, assembled)
         dt_foam_stack = time.perf_counter() - t0
 
         # assemble with geometry and quad position
@@ -41,26 +56,23 @@ def _benchmark_1m_imp(geom_fast_cls, geom_cls, geom_file, quad_positions=None):
             geom = geom_fast_cls.from_h5_file_and_quad_positions(geom_file, quad_positions)
         else:
             geom = geom_fast_cls.from_crystfel_geom(geom_file)
-        out = np.full((n_pulses, *geom.assembledShape()), np.nan, dtype=np.float32)
+        assembled = np.full((n_pulses, *geom.assembledShape()), np.nan, dtype=_IMAGE_DTYPE)
         t0 = time.perf_counter()
-        geom.position_all_modules(modules, out)
+        geom.position_all_modules(modules, assembled)
         dt_foam = time.perf_counter() - t0
 
-        # assemble with geometry and quad position in EXtra-geom
-
-        if quad_positions is not None:
-            geom = geom_cls.from_h5_file_and_quad_positions(geom_file, quad_positions)
-        else:
-            geom = geom_cls.from_crystfel_geom(geom_file)
-        out = geom.output_array_for_position_fast((n_pulses,))
-        t0 = time.perf_counter()
-        geom.position_all_modules(modules, out=out)
-        dt_geom = time.perf_counter() - t0
-
         print(f"\nposition all modules for {geom_cls.__name__} (from {from_str} data) - \n"
               f"  dt (foam stack only): {dt_foam_stack:.4f}, dt (foam): {dt_foam:.4f}, "
               f"dt (geom): {dt_geom:.4f}")
 
+        if modules.dtype == _IMAGE_DTYPE:
+            t0 = time.perf_counter()
+            geom.dismantle_all_modules(assembled, modules)
+            dt_foam_dismantle = time.perf_counter() - t0
+
+            print(f"\ndismantle all modules for {geom_cls.__name__} (from {from_str} data) - \n"
+                  f"  dt (foam): {dt_foam_dismantle:.4f}")
+
 
 def benchmark_dssc_1m():
     from extra_foam.geometries import DSSC_1MGeometryFast
@@ -101,6 +113,46 @@ def benchmark_agipd_1m():
     _benchmark_1m_imp(AGIPD_1MGeometryFast, AGIPD_1MGeometry, geom_file)
 
 
+def benchmark_jungfrau():
+    from extra_foam.geometries import JungFrauGeometryFast
+
+    for from_dtype, from_str in _data_sources:
+        n_row, n_col = 3, 2
+        geom = JungFrauGeometryFast(n_row, n_col)
+        n_pulses = 16
+        modules = np.ones((n_pulses, n_row * n_col, *geom.module_shape), dtype=from_dtype)
+
+        assembled = geom.output_array_for_position_fast((n_pulses,), _IMAGE_DTYPE)
+
+        t0 = time.perf_counter()
+        geom.position_all_modules(modules, assembled)
+        dt_assemble = time.perf_counter() - t0
+
+        print(f"\nposition all modules for JungFrauGeometry (from {from_str} data) - \n"
+              f"  dt (foam stack only): {dt_assemble:.4f}")
+
+        if modules.dtype == _IMAGE_DTYPE:
+            t0 = time.perf_counter()
+            geom.dismantle_all_modules(assembled, modules)
+            dt_dismantle = time.perf_counter() - t0
+
+            print(f"\ndismantle all modules for JungFrauGeometry (from {from_str} data) - \n"
+                  f"  dt (foam stack only): {dt_dismantle:.4f}")
+
+    module = np.ones((n_pulses, *geom.module_shape), dtype=_IMAGE_DTYPE)
+    t0 = time.perf_counter()
+    JungFrauGeometryFast.maskModule(module)
+    dt_mask_cpp = time.perf_counter() - t0
+
+    module = np.ones((n_pulses, *geom.module_shape), dtype=_IMAGE_DTYPE)
+    t0 = time.perf_counter()
+    JungFrauGeometryFast.mask_module_py(module)
+    dt_mask_py = time.perf_counter() - t0
+
+    print(f"\nMask single module for JungFrauGeometry - \n"
+          f"  dt (cpp): {dt_mask_cpp:.4f}, dt (py): {dt_mask_py:.4f}")
+
+
 if __name__ == "__main__":
     print("*" * 80)
     print("Benchmark geometry")
@@ -111,3 +163,5 @@ def benchmark_agipd_1m():
     benchmark_lpd_1m()
 
     benchmark_agipd_1m()
+
+    benchmark_jungfrau()