diff --git a/core/include/tangram/data/clientDataSource.h b/core/include/tangram/data/clientDataSource.h index 284a55a7a5..6b89ffe0fe 100644 --- a/core/include/tangram/data/clientDataSource.h +++ b/core/include/tangram/data/clientDataSource.h @@ -50,11 +50,17 @@ class ClientDataSource : public TileSource { void addPointFeature(Properties&& properties, LngLat coordinates); - void addPolylineFeature(Properties&& properties, PolylineBuilder&& polyline); + uint64_t addPolylineFeature(Properties&& properties, PolylineBuilder&& polyline); void addPolygonFeature(Properties&& properties, PolygonBuilder && polygon); + void updatePolylineFeature(uint64_t id, const Coordinates& coordinates); + + void updatePolylineFeature(uint64_t id, const Properties&& properties); + + void removePolylineFeature(uint64_t id); + // Transform added feature data into tiles. void generateTiles(); diff --git a/core/include/tangram/data/tileSource.h b/core/include/tangram/data/tileSource.h index d7233c365e..53febf557b 100644 --- a/core/include/tangram/data/tileSource.h +++ b/core/include/tangram/data/tileSource.h @@ -54,6 +54,8 @@ class TileSource : public std::enable_shared_from_this { struct DataSource { virtual ~DataSource() {} + virtual TileID getFallbackTileID(const TileID& _tileID, int32_t _maxZoom, int32_t _zoomBias) = 0; + virtual bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) = 0; /* Stops any running I/O tasks pertaining to @_tile */ @@ -92,6 +94,8 @@ class TileSource : public std::enable_shared_from_this { */ virtual const char* mimeType() const; + TileID getFallbackTileID(const TileID& _tileID); + /* Fetches data for the map tile specified by @_tileID * * LoadTile starts an asynchronous I/O task to retrieve the data for a tile. When diff --git a/core/src/data/clientDataSource.cpp b/core/src/data/clientDataSource.cpp index 1590c9767c..788573374d 100644 --- a/core/src/data/clientDataSource.cpp +++ b/core/src/data/clientDataSource.cpp @@ -38,6 +38,7 @@ struct ClientDataSource::Storage { std::unique_ptr tiles; geometry::feature_collection features; std::vector properties; + std::map polylineIds; }; struct ClientDataSource::PolylineBuilderData : mapbox::geometry::line_string { @@ -260,14 +261,21 @@ void ClientDataSource::addPointFeature(Properties&& properties, LngLat coordinat m_store->properties.emplace_back(properties); } -void ClientDataSource::addPolylineFeature(Properties&& properties, PolylineBuilder&& polyline) { +uint64_t ClientDataSource::addPolylineFeature(Properties&& properties, PolylineBuilder&& polyline) { std::lock_guard lock(m_mutexStore); - uint64_t id = m_store->features.size(); + uint64_t i = m_store->features.size(); + uint64_t id = m_store->polylineIds.size(); + + properties.set("id", std::to_string(id)); + auto geom = std::move(polyline.data); - m_store->features.emplace_back(*geom, id); + m_store->features.emplace_back(*geom, i); m_store->properties.emplace_back(properties); + m_store->polylineIds.insert(std::pair(id, i)); + + return id; } void ClientDataSource::addPolygonFeature(Properties&& properties, PolygonBuilder&& polygon) { @@ -280,6 +288,61 @@ void ClientDataSource::addPolygonFeature(Properties&& properties, PolygonBuilder m_store->properties.emplace_back(properties); } +void ClientDataSource::updatePolylineFeature(uint64_t id, const Coordinates& coordinates) { + + std::lock_guard lock(m_mutexStore); + + std::map::iterator findIt = m_store->polylineIds.find(id); + + if (findIt != m_store->polylineIds.end() && findIt->second < m_store->features.size()) { + uint64_t i = findIt->second; + + geometry::line_string geom; + for (auto &p : coordinates) { + geom.emplace_back(p.longitude, p.latitude); + } + + m_store->features[i] = mapbox::geometry::feature(geom, i); + } +} + +void ClientDataSource::updatePolylineFeature(uint64_t id, const Properties&& properties) { + + std::lock_guard lock(m_mutexStore); + + std::map::iterator findIt = m_store->polylineIds.find(id); + + if (findIt != m_store->polylineIds.end() && findIt->second < m_store->properties.size()) { + uint64_t i = findIt->second; + + for (int j = 0; j < properties.items().size(); ++j) { + m_store->properties[i].set(properties.items()[j].key, + properties.getString(properties.items()[j].key)); + } + } +} + +void ClientDataSource::removePolylineFeature(uint64_t id) { + + std::lock_guard lock(m_mutexStore); + + std::map::iterator findIt = m_store->polylineIds.find(id); + + if (findIt != m_store->polylineIds.end() && findIt->second < m_store->features.size()) { + uint64_t i = findIt->second; + + m_store->features.erase(std::next(m_store->features.begin(), i)); + m_store->properties.erase(std::next(m_store->properties.begin(), i)); + + for (std::map::iterator it = m_store->polylineIds.begin(); it != m_store->polylineIds.end(); ++it) { + if (it->second > i) { + it->second = it->second - 1; + m_store->features[it->second].id = it->second; + } + } + } +} + struct add_geometry { static constexpr double extent = 4096.0; diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index 6997003bf0..b579849dd2 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -132,6 +132,24 @@ MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name, std MBTilesDataSource::~MBTilesDataSource() { } +TileID MBTilesDataSource::getFallbackTileID(const TileID& _tileID, int32_t _maxZoom, int32_t _zoomBias) { + + TileID tileID(_tileID); + + while (!hasTileData(tileID) && tileID.z > 0) { + tileID = tileID.getParent(_zoomBias); + } + + if (tileID.z == _maxZoom) { + tileID = _tileID; + } + else { + tileID.s = _tileID.s; + } + + return tileID; +} + bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { if (m_offlineMode) { @@ -196,12 +214,12 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTask if (m_cacheMode) { m_worker->enqueue([this, _task](){ - auto& task = static_cast(*_task); + auto& task = static_cast(*_task); - LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); + LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); - storeTileData(_task->tileId(), *task.rawTileData); - }); + storeTileData(_task->tileId(), *task.rawTileData); + }); } _cb.func(_task); @@ -422,8 +440,50 @@ void MBTilesDataSource::initSchema(SQLite::Database& db, std::string _name, std: } } +bool MBTilesDataSource::hasTileData(const TileID& _tileId) { + + TileID tileId = TileID(_tileId); + tileId.s = 0; + + auto search = m_HasTileDataCache.find(tileId); + + if (search != m_HasTileDataCache.end()) { + return search->second; + } + + auto& stmt = m_queries->getTileData; + try { + // Google TMS to WMTS + // https://github.com/mapbox/node-mbtiles/blob/ + // 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149 + int z = _tileId.z; + int y = (1 << z) - 1 - _tileId.y; + + stmt.bind(1, z); + stmt.bind(2, _tileId.x); + stmt.bind(3, y); + + if (stmt.executeStep()) { + stmt.reset(); + m_HasTileDataCache[tileId] = true; + return true; + } + + } catch (std::exception& e) { + LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what()); + } + try { + stmt.reset(); + } catch (...) {} + + m_HasTileDataCache[tileId] = false; + return false; +} + bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _data) { + int largestLength = 0; + auto& stmt = m_queries->getTileData; try { // Google TMS to WMTS @@ -441,20 +501,24 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d const char* blob = (const char*) column.getBlob(); const int length = column.getBytes(); - if ((m_schemaOptions.compression == Compression::undefined) || - (m_schemaOptions.compression == Compression::deflate)) { - - if (zlib::inflate(blob, length, _data) != 0) { - if (m_schemaOptions.compression == Compression::undefined) { - _data.resize(length); - memcpy(_data.data(), blob, length); - } else { - LOGW("Invalid deflate compression"); + if (length > largestLength) { + if ((m_schemaOptions.compression == Compression::undefined) || + (m_schemaOptions.compression == Compression::deflate)) { + + if (zlib::inflate(blob, length, _data) != 0) { + if (m_schemaOptions.compression == Compression::undefined) { + _data.resize(length); + memcpy(_data.data(), blob, length); + } else { + LOGW("Invalid deflate compression"); + } } + } else { + _data.resize(length); + memcpy(_data.data(), blob, length); } - } else { - _data.resize(length); - memcpy(_data.data(), blob, length); + + largestLength = length; } stmt.reset(); diff --git a/core/src/data/mbtilesDataSource.h b/core/src/data/mbtilesDataSource.h index 2a9f65ef8a..c24656b4a3 100644 --- a/core/src/data/mbtilesDataSource.h +++ b/core/src/data/mbtilesDataSource.h @@ -1,5 +1,6 @@ #pragma once +#include #include "data/tileSource.h" namespace SQLite { @@ -22,11 +23,14 @@ class MBTilesDataSource : public TileSource::DataSource { ~MBTilesDataSource(); + TileID getFallbackTileID(const TileID& _tileID, int32_t _maxZoom, int32_t _zoomBias) override; + bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) override; void clear() override {} private: + bool hasTileData(const TileID& _tileId); bool getTileData(const TileID& _tileId, std::vector& _data); void storeTileData(const TileID& _tileId, const std::vector& _data); bool loadNextSource(std::shared_ptr _task, TileTaskCb _cb); @@ -52,6 +56,9 @@ class MBTilesDataSource : public TileSource::DataSource { std::unique_ptr m_queries; std::unique_ptr m_worker; + // Cached has tile data + std::map m_HasTileDataCache; + // Platform reference Platform& m_platform; diff --git a/core/src/data/memoryCacheDataSource.cpp b/core/src/data/memoryCacheDataSource.cpp index cf7de63dd6..253bc9372d 100644 --- a/core/src/data/memoryCacheDataSource.cpp +++ b/core/src/data/memoryCacheDataSource.cpp @@ -22,8 +22,17 @@ struct RawCache { CacheMap m_cacheMap; CacheList m_cacheList; - int m_usage = 0; - int m_maxUsage = 0; + size_t m_usage = 0; + size_t m_maxUsage = 0; + + bool has(const TileID& _tileID) { + + if (m_maxUsage <= 0) { return false; } + + std::lock_guard lock(m_mutex); + + return m_cacheMap.find(_tileID) != m_cacheMap.end(); + } bool get(BinaryTileTask& _task) { @@ -93,6 +102,10 @@ void MemoryCacheDataSource::setCacheSize(size_t _cacheSize) { m_cache->m_maxUsage = _cacheSize; } +bool MemoryCacheDataSource::hasCache(const TileID& _tileID) { + return m_cache->has(_tileID); +} + bool MemoryCacheDataSource::cacheGet(BinaryTileTask& _task) { return m_cache->get(_task); } @@ -101,6 +114,43 @@ void MemoryCacheDataSource::cachePut(const TileID& _tileID, std::shared_ptrput(_tileID, _rawDataRef); } +TileID MemoryCacheDataSource::getFallbackTileID(const TileID& _tileID, int32_t _maxZoom, int32_t _zoomBias) { + + TileID tileID(_tileID); + bool isCached = false; + + while (!(isCached = hasCache(tileID)) && tileID.z > 0) { + tileID = tileID.getParent(_zoomBias); + } + + if (tileID.z == _maxZoom) { + tileID = _tileID; + } + else { + tileID.s = _tileID.s; + } + + if (next) { + TileID nextTileID = next->getFallbackTileID(_tileID, _maxZoom, _zoomBias); + + if (nextTileID.z == _maxZoom) { + nextTileID = _tileID; + } + else { + nextTileID.s = _tileID.s; + } + + if (isCached) { + return (tileID.z > nextTileID.z) ? tileID : nextTileID; + } + else { + return nextTileID; + } + } + + return tileID; +} + bool MemoryCacheDataSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { auto& task = static_cast(*_task); diff --git a/core/src/data/memoryCacheDataSource.h b/core/src/data/memoryCacheDataSource.h index ba19e98c49..b02deff9fc 100644 --- a/core/src/data/memoryCacheDataSource.h +++ b/core/src/data/memoryCacheDataSource.h @@ -10,6 +10,8 @@ class MemoryCacheDataSource : public TileSource::DataSource { MemoryCacheDataSource(); ~MemoryCacheDataSource(); + TileID getFallbackTileID(const TileID& _tileID, int32_t _maxZoom, int32_t _zoomBias) override; + bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) override; void clear() override; @@ -20,6 +22,7 @@ class MemoryCacheDataSource : public TileSource::DataSource { void setCacheSize(size_t _cacheSize); private: + bool hasCache(const TileID& _tileID); bool cacheGet(BinaryTileTask& _task); void cachePut(const TileID& _tileID, std::shared_ptr> _rawDataRef); diff --git a/core/src/data/networkDataSource.h b/core/src/data/networkDataSource.h index 69d1b865d9..6fc6b88ed4 100644 --- a/core/src/data/networkDataSource.h +++ b/core/src/data/networkDataSource.h @@ -15,6 +15,8 @@ class NetworkDataSource : public TileSource::DataSource { NetworkDataSource(Platform& _platform, const std::string& _urlTemplate, std::vector&& _urlSubdomains, bool _isTms); + TileID getFallbackTileID(const TileID& _tileID, int32_t _maxZoom, int32_t _zoomBias) override { return _tileID; } + bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) override; void cancelLoadingTile(TileTask& _task) override; diff --git a/core/src/data/rasterSource.cpp b/core/src/data/rasterSource.cpp index 689e9e31f5..19e402cf62 100644 --- a/core/src/data/rasterSource.cpp +++ b/core/src/data/rasterSource.cpp @@ -150,19 +150,24 @@ std::shared_ptr RasterSource::parse(const TileTask& _task) const { void RasterSource::addRasterTask(TileTask& _task) { - TileID subTileID = _task.tileId(); + TileID fallbackTileID = getFallbackTileID(_task.tileId()); - // apply apt downsampling for raster tiles depending on difference - // in zoomBias (which also takes zoom offset into account) - auto zoomDiff = m_zoomOptions.zoomBias - _task.source()->zoomBias(); + if (fallbackTileID == _task.tileId()) { + // apply apt downsampling for raster tiles depending on difference + // in zoomBias (which also takes zoom offset into account) + auto zoomDiff = m_zoomOptions.zoomBias - _task.source()->zoomBias(); - if (zoomDiff > 0) { - subTileID = subTileID.zoomBiasAdjusted(zoomDiff).withMaxSourceZoom(m_zoomOptions.maxZoom); - } else { - subTileID = subTileID.withMaxSourceZoom(m_zoomOptions.maxZoom); + if (zoomDiff > 0) { + fallbackTileID = fallbackTileID.zoomBiasAdjusted(zoomDiff).withMaxSourceZoom(m_zoomOptions.maxZoom); + } else { + fallbackTileID = fallbackTileID.withMaxSourceZoom(m_zoomOptions.maxZoom); + } + } + else if (fallbackTileID.s > (int8_t)_task.source()->maxZoom()) { + fallbackTileID.s = (int8_t)_task.source()->maxZoom(); } - auto rasterTask = createRasterTask(subTileID, true); + auto rasterTask = createRasterTask(fallbackTileID, true); _task.subTasks().push_back(rasterTask); } diff --git a/core/src/data/tileSource.cpp b/core/src/data/tileSource.cpp index c1a6c4bee7..2210768f8e 100644 --- a/core/src/data/tileSource.cpp +++ b/core/src/data/tileSource.cpp @@ -78,6 +78,16 @@ void TileSource::clearData() { m_generation++; } +TileID TileSource::getFallbackTileID(const TileID& _tileID) { + + if (m_sources) { + return m_sources->getFallbackTileID(_tileID, maxZoom(), zoomBias()); + } + else { + return _tileID; + } +} + void TileSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { if (m_sources) { diff --git a/core/src/tile/tileManager.cpp b/core/src/tile/tileManager.cpp index bd244e5955..c4eaaf02c1 100644 --- a/core/src/tile/tileManager.cpp +++ b/core/src/tile/tileManager.cpp @@ -296,8 +296,16 @@ void TileManager::updateTileSets(const View& _view) { auto zoomBias = tileSet.source->zoomBias(); auto maxZoom = tileSet.source->maxZoom(); - // Insert scaled and maxZoom mapped tileID in the visible set - tileSet.visibleTiles.insert(_tileID.zoomBiasAdjusted(zoomBias).withMaxSourceZoom(maxZoom)); + TileID fallbackTileID = tileSet.source->getFallbackTileID(_tileID); + + if (fallbackTileID == _tileID) { + fallbackTileID = fallbackTileID.zoomBiasAdjusted(zoomBias); + } + else if (fallbackTileID.s > (int8_t)maxZoom) { + fallbackTileID.s = (int8_t)maxZoom; + } + + tileSet.visibleTiles.insert(fallbackTileID.withMaxSourceZoom(maxZoom)); } };