use tiles::hybrid_node_idx

to store node coordinates on disk

.gitignore

@@ -14,3 +14,7 @@ CMakeSettings.json
 /deps
 /.clang-tidy
 .pkg.mutex
+*.ppr
+*.ppr.art
+*.ppr.ert
+*.stats.csv

.pkg

@@ -29,7 +29,7 @@
 [utl]
   [email protected]:motis-project/utl.git
   branch=master
-  commit=3d3b9f2d45fd6e864cfda0b32a924e8d445c8b69
+  commit=6360fac4f35f4088decb71189adc1337cd28dfaa
 [zlib]
   [email protected]:motis-project/zlib.git
   branch=master
@@ -50,3 +50,7 @@
   [email protected]:motis-project/fmt.git
   branch=master
   commit=d19c9bfb59e792b7a39d1a347b76d5702e02130a
+[tiles]
+  [email protected]:motis-project/tiles.git
+  branch=master
+  commit=abbf28537c2ac0271dc8e4e29ed47b0359265723

.pkg.lock

@@ -1,16 +1,29 @@
-1928867614217922333
+12906991495586833437
 cista a6218e9cd9da9e1c7a70c1188e2faf9e082b7f1c
 zlib fe8e13ffca867612951bc6baf114e5ac8b00f305
 boost be5235eb2258d2ec19e32546ab767a62311d9b46
 conf aab49490c6b77027938b74265f386144364aa266
 cpptoml 2133029ec819e8398e96fa679993b269f21ff9f2
-fmt d19c9bfb59e792b7a39d1a347b76d5702e02130a
+fmt edb385ac526c24bc917ec4a41bb0edb28f0ca59e
 libosmium 6e6d6b3081cc8bdf25dda89730e25c36eb995516
 mimalloc 2a557cafb2e9e7c872358a83a63c62a7e14330b3
 libressl 390253a44ceef00eb620c38606588414326e9f23
 net 44674d2f3917e20b7019a0f7254d332522c36fb7
 protozero 8c9f3fa97c2cfdceef86d0b61818ae98e9328f29
 rapidjson e4a599d2b5dec065b1ea2af5d8dac52baa9df5f5
-unordered_dense 4f380fb1d64f1843ca2c993ed39a8342a8747e5d
 Catch2 47d56f28a9801911c048d011b375e5631dbb658f
-utl 3d3b9f2d45fd6e864cfda0b32a924e8d445c8b69
+LuaJIT 4638e9198beb2f14bd1c90b42aff744469eed404
+clipper 904f0e6644c7f01c176443613be8f7788d59c658
+concurrentqueue ef2227ada6f76cd65e517b7a6e902964443c0ffc
+expat b8c26c40f1900899b95c795705e0252fc0c1350c
+doctest 70e8f76437b76dd5e9c0a2eb9b907df190ab71a0
+geo 5bf5179faf5f21d543866cca85fe090fe3664cc5
+lmdb 39d8127e5697b1323a67e61c3ad8f087384c7429
+miniz 1edbdece9d71dc65c6ff405572ee37cbdcef7af4
+res c500c261531a2b8b8d475fc9d42c075f28f72aaf
+pbf-sdf-fonts 91b369e4eb8a618e0a83b0c04b1b08632ea872c4
+sol2 fdb0f8a60e48aa737f0a8d73edede48627f0c984
+utl 6360fac4f35f4088decb71189adc1337cd28dfaa
+variant 3c7fc8266bb46046b42c2dc2663f9f505f0cec28
+tiles abbf28537c2ac0271dc8e4e29ed47b0359265723
+unordered_dense 4f380fb1d64f1843ca2c993ed39a8342a8747e5d

CMakeLists.txt

@@ -196,6 +196,7 @@ target_link_libraries(ppr-preprocessing
   cista
   unordered_dense
   ppr-common
+  tiles-import-library
 )
 target_compile_features(ppr-preprocessing PUBLIC cxx_std_20)
 set_target_properties(ppr-preprocessing PROPERTIES CXX_EXTENSIONS OFF)

include/ppr/cmd/preprocess/prog_options.h

@@ -33,6 +33,7 @@ class prog_options : public conf::configuration {
     param(verify_graph_, "verify-graph", "Verify generated graph file");
     param(print_timing_overview_, "timings", "Print timing overview");
     param(print_memory_usage_, "mem", "Print memory usage");
+    param(tmp_dir_, "tmp_dir", "temporary directoy");
   }
 
   options get_options() const {
@@ -51,6 +52,7 @@ class prog_options : public conf::configuration {
     return opt;
   }
 
+  std::filesystem::path tmp_dir_{"./ppr_tmp"};
   std::string osm_file_{"germany-latest.osm.pbf"};
   std::string graph_file_{"routing-graph.ppr"};
   std::vector<std::string> dem_files_;

include/ppr/preprocessing/options.h

@@ -1,12 +1,15 @@
 #pragma once
 
+#include <filesystem>
 #include <string>
 #include <thread>
 #include <vector>
 
 namespace ppr::preprocessing {
 
 struct options {
+  std::filesystem::path tmp_dir_;
+
   std::string osm_file_;
   std::vector<std::string> dem_files_;
 

include/ppr/preprocessing/osm_graph/extractor.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <filesystem>
 #include <string>
 
 #include "ppr/preprocessing/logging.h"
@@ -8,6 +9,7 @@
 
 namespace ppr::preprocessing {
 
-osm_graph extract(std::string const& osm_file, logging& log, statistics& stats);
+osm_graph extract(std::filesystem::path const& tmp_dname,
+                  std::string const& osm_file, logging& log, statistics& stats);
 
 }  // namespace ppr::preprocessing

src/preprocessing/osm_graph/builder.cc

@@ -12,7 +12,7 @@ namespace ppr::preprocessing {
 
 osm_graph build_osm_graph(options const& opt, logging& log, statistics& stats) {
   auto const t_start = timing_now();
-  auto og = extract(opt.osm_file_, log, stats);
+  auto og = extract(opt.tmp_dir_, opt.osm_file_, log, stats);
   auto const t_after_extract = timing_now();
   stats.osm_.d_extract_ = ms_between(t_start, t_after_extract);
 

src/preprocessing/osm_graph/extractor.cc

@@ -17,6 +17,10 @@
 #include "osmium/relations/relations_manager.hpp"
 #include "osmium/visitor.hpp"
 
+#include "tiles/osm/hybrid_node_idx.h"
+#include "tiles/osm/tmp_file.h"
+#include "tiles/util_parallel.h"
+
 #include "ppr/common/timing.h"
 #include "ppr/preprocessing/logging.h"
 #include "ppr/preprocessing/names.h"
@@ -32,6 +36,7 @@
 
 namespace bg = boost::geometry;
 namespace bgi = boost::geometry::index;
+namespace fs = std::filesystem;
 
 using namespace ppr::preprocessing::osm;
 
@@ -294,8 +299,8 @@ struct multipolygon_way_manager
   ankerl::unordered_dense::set<osmium::object_id_type>& ways_;
 };
 
-osm_graph extract(std::string const& osm_file, logging& log,
-                  statistics& stats) {
+osm_graph extract(fs::path const& tmp_dname, std::string const& osm_file,
+                  logging& log, statistics& stats) {
   auto const t_start = timing_now();
   auto const infile = osmium::io::File(osm_file);
   stats.osm_input_size_ = boost::filesystem::file_size(osm_file);
@@ -313,17 +318,29 @@ osm_graph extract(std::string const& osm_file, logging& log,
   ankerl::unordered_dense::set<osmium::object_id_type> multipolygon_ways;
   multipolygon_way_manager mp_way_manager{filter, multipolygon_ways};
 
+  auto const node_idx_file =
+      tiles::tmp_file{(tmp_dname / "idx.bin").generic_string()};
+  auto const node_dat_file =
+      tiles::tmp_file{(tmp_dname / "dat.bin").generic_string()};
+  auto node_idx =
+      tiles::hybrid_node_idx{node_idx_file.fileno(), node_dat_file.fileno()};
+
   {
-    osmium::io::Reader reader{infile, osmium::osm_entity_bits::relation};
+    auto node_idx_builder = tiles::hybrid_node_idx_builder{node_idx};
+
+    osmium::io::Reader reader{infile, osmium::osm_entity_bits::relation,
+                              osmium::io::read_meta::no};
     step_progress progress{log, pp_step::OSM_EXTRACT_RELATIONS,
                            reader.file_size()};
     while (auto buffer = reader.read()) {
       progress.set(reader.offset());
-      osmium::apply(buffer, mp_manager, mp_way_manager);
+      osmium::apply(buffer, node_idx_builder, mp_manager, mp_way_manager);
     }
     reader.close();
     mp_manager.prepare_for_lookup();
     mp_way_manager.prepare_for_lookup();
+
+    node_idx_builder.finish();
   }
 
   stats.osm_.extract_.d_relations_pass_ =
@@ -343,11 +360,11 @@ osm_graph extract(std::string const& osm_file, logging& log,
     step_progress progress{log, pp_step::OSM_EXTRACT_MAIN, reader.file_size()};
     while (auto buffer = reader.read()) {
       progress.set(reader.offset());
-      osmium::apply(
-          buffer, location_handler, handler,
-          mp_manager.handler([&handler](osmium::memory::Buffer&& buffer) {
-            osmium::apply(buffer, handler);
-          }));
+      osmium::apply(buffer, location_handler, handler,
+                    mp_manager.handler([&](osmium::memory::Buffer&& buffer) {
+                      tiles::update_locations(node_idx, buffer);
+                      osmium::apply(buffer, handler);
+                    }));
     }
     reader.close();
   }