From 58349702aa1b03d2ff2f1685fbd81d0bb9fa7eb5 Mon Sep 17 00:00:00 2001 From: markus-moser Date: Wed, 15 May 2024 14:29:35 +0200 Subject: [PATCH 1/2] Fix document counts in index stats (#155) --- .../search-index-adapter/open-search.yaml | 2 ++ qodana.yaml | 5 +++ .../OpenSearch/IndexStatsService.php | 35 +++++++++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/config/services/search-index-adapter/open-search.yaml b/config/services/search-index-adapter/open-search.yaml index d0dde688..a6a91a77 100644 --- a/config/services/search-index-adapter/open-search.yaml +++ b/config/services/search-index-adapter/open-search.yaml @@ -32,6 +32,8 @@ services: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexStatsServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\IndexStatsService + arguments: + $openSearchClient: '@generic-data-index.opensearch-client' Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\IndexMappingService \ No newline at end of file diff --git a/qodana.yaml b/qodana.yaml index b51a5655..68f707e4 100644 --- a/qodana.yaml +++ b/qodana.yaml @@ -42,6 +42,11 @@ exclude: - name: PhpMethodNamingConventionInspection paths: - src/Migrations + - name: PhpDqlBuilderUnknownModelInspection + paths: + - src/Repository/IndexQueueRepository.php + - src/Service/SearchIndex/IndexService/ElementTypeAdapter/AssetTypeAdapter.php + - src/Service/SearchIndex/IndexService/ElementTypeAdapter/DataObjectTypeAdapter.php include: - name: PhpTaintFunctionInspection - name: PhpVulnerablePathsInspection diff --git a/src/SearchIndexAdapter/OpenSearch/IndexStatsService.php b/src/SearchIndexAdapter/OpenSearch/IndexStatsService.php index c7a44be9..8dc3152f 100644 --- a/src/SearchIndexAdapter/OpenSearch/IndexStatsService.php +++ b/src/SearchIndexAdapter/OpenSearch/IndexStatsService.php @@ -17,6 +17,7 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch; use Exception; +use OpenSearch\Client; use Pimcore\Bundle\GenericDataIndexBundle\Model\Stats\IndexStats; use Pimcore\Bundle\GenericDataIndexBundle\Model\Stats\IndexStatsIndex; use Pimcore\Bundle\GenericDataIndexBundle\Repository\IndexQueueRepository; @@ -30,9 +31,10 @@ final class IndexStatsService implements IndexStatsServiceInterface use LoggerAwareTrait; public function __construct( - protected readonly SearchIndexConfigServiceInterface $searchIndexConfigService, - protected readonly IndexQueueRepository $indexQueueRepository, - protected readonly SearchIndexServiceInterface $openSearchService, + private readonly SearchIndexConfigServiceInterface $searchIndexConfigService, + private readonly IndexQueueRepository $indexQueueRepository, + private readonly SearchIndexServiceInterface $openSearchService, + private readonly Client $openSearchClient, ) { } @@ -43,12 +45,31 @@ public function getStats(): IndexStats $this->searchIndexConfigService->getIndexPrefix() . '*' ); + $aggregationResult = $this->openSearchClient->search([ + 'index' => $this->searchIndexConfigService->getIndexPrefix() . '*', + 'body' => [ + 'size' => 0, + 'aggs' => [ + 'indices' => [ + 'terms' => [ + 'field' => '_index', + 'size' => 10000, + 'order' => [ + '_key' => 'asc', + ], + ], + ], + ], + ], + ]); + $indices = []; - foreach ($allStats['indices'] as $indexName => $index) { + foreach ($aggregationResult['aggregations']['indices']['buckets'] as $bucket) { + $sizeInBytes = (int)($allStats['indices'][$bucket['key']]['total']['store']['size_in_bytes'] ?? 0); $indices[] = new IndexStatsIndex( - indexName: $indexName, - itemsCount: $index['total']['docs']['count'], - sizeInKb: round(((int)$index['total']['store']['size_in_bytes'] / 1024), 2) + indexName: $bucket['key'], + itemsCount: $bucket['doc_count'], + sizeInKb: round(($sizeInBytes / 1024), 2) ); } From 018d10434f7e16f8f9a1d7608bcf0ab41941b493 Mon Sep 17 00:00:00 2001 From: markus-moser Date: Thu, 13 Jun 2024 08:57:02 +0200 Subject: [PATCH 2/2] Fix input quantity value data object type OpenSearch mapping (#167) --- .github/workflows/qodana.yml | 29 - .qodana-profile.xml | 625 ------------------ .../data-object/field-definition-adapters.yml | 4 + qodana.yaml | 57 -- .../InputQuantityValueAdapter.php | 39 ++ 5 files changed, 43 insertions(+), 711 deletions(-) delete mode 100644 .github/workflows/qodana.yml delete mode 100644 .qodana-profile.xml delete mode 100644 qodana.yaml create mode 100644 src/SearchIndexAdapter/OpenSearch/DataObject/FieldDefinitionAdapter/InputQuantityValueAdapter.php diff --git a/.github/workflows/qodana.yml b/.github/workflows/qodana.yml deleted file mode 100644 index 468d2560..00000000 --- a/.github/workflows/qodana.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Qodana -on: - schedule: - - cron: '0 01 * * *' # Run once per day - workflow_dispatch: - push: - paths: - - '**.php' - - '**.yml' - - '**.yaml' - branches: - - '*' - - '**' - pull_request_target: - types: [opened, synchronize, reopened] - paths: - - '**.php' - - '**.yml' - - '**.yaml' - branches: - - '*' - - '**' - -jobs: - qodana-check-workflow: - uses: pimcore/workflows-collection-public/.github/workflows/reusable-qodana-check.yaml@main - secrets: - COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN: ${{ secrets.COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN }} - QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} \ No newline at end of file diff --git a/.qodana-profile.xml b/.qodana-profile.xml deleted file mode 100644 index c1a0bf25..00000000 --- a/.qodana-profile.xml +++ /dev/null @@ -1,625 +0,0 @@ - - \ No newline at end of file diff --git a/config/services/search/data-object/field-definition-adapters.yml b/config/services/search/data-object/field-definition-adapters.yml index 1688cde2..d121a043 100644 --- a/config/services/search/data-object/field-definition-adapters.yml +++ b/config/services/search/data-object/field-definition-adapters.yml @@ -44,6 +44,10 @@ services: shared: false tags: - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "quantityValue" } + + Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\DataObject\FieldDefinitionAdapter\InputQuantityValueAdapter: + shared: false + tags: - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "inputQuantityValue" } Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\DataObject\FieldDefinitionAdapter\BooleanAdapter: diff --git a/qodana.yaml b/qodana.yaml deleted file mode 100644 index 68f707e4..00000000 --- a/qodana.yaml +++ /dev/null @@ -1,57 +0,0 @@ -version: "1.0" -linter: jetbrains/qodana-php:latest -profile: - path: .qodana-profile.xml -php: - version: "8.2" -failThreshold: 0 -exclude: - - name: All - paths: - - vendor - - public - - tests - - var - - translations - - .php_cs.dist - - doc - - .github - - .php-cs-fixer.dist.php - - .qodana-profile.xml - - config - - .editorconfig - - composer.json - - composer.lock - - docker-compose.yml - - README.md - - SECURITY.md - - codeception.dist.yml - - phpstan-bootstrap.php - - phpstan-baseline.neon - - phpstan.neon - - LICENSE.md - - .env - - .gitatributes - - .docker - - .gitignore - - src/DependencyInjection/Configuration.php - - placeholderReplace.sh - - name: PhpUnusedParameterInspection - paths: - - src/SearchIndexAdapter/OpenSearch/Search/Modifier - - name: PhpMethodNamingConventionInspection - paths: - - src/Migrations - - name: PhpDqlBuilderUnknownModelInspection - paths: - - src/Repository/IndexQueueRepository.php - - src/Service/SearchIndex/IndexService/ElementTypeAdapter/AssetTypeAdapter.php - - src/Service/SearchIndex/IndexService/ElementTypeAdapter/DataObjectTypeAdapter.php -include: - - name: PhpTaintFunctionInspection - - name: PhpVulnerablePathsInspection -plugins: - - id: de.espend.idea.php.annotation - - id: com.kalessil.phpStorm.phpInspectionsEA - - id: de.espend.idea.php.toolbox - - id: fr.adrienbrault.idea.symfony2plugin \ No newline at end of file diff --git a/src/SearchIndexAdapter/OpenSearch/DataObject/FieldDefinitionAdapter/InputQuantityValueAdapter.php b/src/SearchIndexAdapter/OpenSearch/DataObject/FieldDefinitionAdapter/InputQuantityValueAdapter.php new file mode 100644 index 00000000..9ddfd686 --- /dev/null +++ b/src/SearchIndexAdapter/OpenSearch/DataObject/FieldDefinitionAdapter/InputQuantityValueAdapter.php @@ -0,0 +1,39 @@ + [ + 'value' => [ + 'type' => AttributeType::KEYWORD->value, + ], + 'unitId' => [ + 'type' => AttributeType::TEXT->value, + ], + ], + ]; + } +}