Add services to search for all elements and their dependencies (#159)

.github/ci/files/config/services_test.yaml

@@ -7,6 +7,7 @@ services:
   generic-data-index.test.service.asset-search-service: '@Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\AssetSearchServiceInterface'
   generic-data-index.test.service.data-object-search-service: '@Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\DataObject\DataObjectSearchServiceInterface'
   generic-data-index.test.service.document-search-service: '@Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Document\DocumentSearchServiceInterface'
+  generic-data-index.test.service.element-search-service: '@Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element\ElementSearchServiceInterface'
 
   test.calculatorservice:
     class: Pimcore\Tests\Support\Helper\DataType\Calculator

README.md

@@ -11,7 +11,7 @@ This bundle can be extended and customized to fit your specific needs, for examp
 ## Features in a Nutshell
 - Based on OpenSearch
 - Centralized data index for multiple bundles (Portal Engine, Studio API/UI, etc.)
-- Indexing of all asset and data objects
+- Indexing of all documents, assets and data objects
 - Provides search services and models to search, filter and aggregate the data saved in the OpenSearch indices 
 
 ## Documentation Overview

config/pimcore/config.yaml

@@ -57,6 +57,8 @@ pimcore_generic_data_index:
       general:
         id:
           type: long
+        elementType:
+          type: keyword
         parentId:
           type: long
         creationDate:
@@ -113,6 +115,15 @@ pimcore_generic_data_index:
           type: boolean
         hasWorkflowWithPermissions:
           type: boolean
+        dependencies:
+          type: object
+          properties:
+            asset:
+              type: long
+            document:
+              type: long
+            object:
+              type: long
       document:
         published:
           type: boolean

config/services/dependency.yaml

@@ -0,0 +1,8 @@
+services:
+  _defaults:
+    autowire: true
+    autoconfigure: true
+    public: false
+
+  Pimcore\Bundle\GenericDataIndexBundle\Service\Dependency\DependencyServiceInterface:
+    class: Pimcore\Bundle\GenericDataIndexBundle\Service\Dependency\DependencyService

config/services/search-index-adapter/open-search.yaml

@@ -12,6 +12,11 @@ services:
         arguments:
             $openSearchClient: '@generic-data-index.opensearch-client'
 
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexAliasServiceInterface:
+        class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\IndexAliasService
+        arguments:
+            $openSearchClient: '@generic-data-index.opensearch-client'
+
     Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\SearchExecutionServiceInterface:
         class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\SearchExecutionService
         arguments:
@@ -48,6 +53,9 @@ services:
     Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Workspace\QueryServiceInterface:
         class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Workspace\QueryService
 
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Workspace\ElementWorkspacesQueryServiceInterface:
+        class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Workspace\ElementWorkspacesQueryService
+
     Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\QueryLanguage\PqlAdapterInterface:
         class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\PqlAdapter
 

config/services/search-index-adapter/open-search/modifiers/search-modifiers.yaml

@@ -11,6 +11,7 @@ services:
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\Filter\BasicFilters: ~
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\Filter\AssetFilters: ~
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\Filter\TreeFilters: ~
+  Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\Filter\DependencyFilters: ~
 
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\FullTextSearch\FullTextSearchHandlers: ~
 

config/services/search/index.yaml

@@ -69,4 +69,7 @@ services:
         arguments:
             - [ ]
 
-    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\IndexHandler\DocumentIndexHandler: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\IndexHandler\DocumentIndexHandler: ~
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\GlobalIndexAliasServiceInterface:
+        class: Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\GlobalIndexAliasService

config/services/search/search-services.yaml

@@ -11,11 +11,16 @@ services:
       class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\DataObject\DataObjectSearchService
 
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Document\DocumentSearchServiceInterface:
-    class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Document\DocumentSearchService
+      class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Document\DocumentSearchService
+
+  Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element\ElementSearchServiceInterface:
+      class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element\ElementSearchService
 
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\DataObject\SearchHelper: ~
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\SearchHelper: ~
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Document\SearchHelper: ~
+  Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element\ElementSearchHelperInterface:
+      class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element\SearchHelper
 
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface:
       class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProvider

doc/01_Installation/02_Upgrade.md

@@ -0,0 +1,8 @@
+# Upgrade Information
+
+Following steps are necessary during updating to newer versions.
+
+## Upgrade to 1.1.0
+- Execute the following command to reindex all elements to be able to use all new features:
+
+  ```bin/console generic-data-index:update:index```

doc/04_Searching_For_Data_In_Index/05_Search_Modifiers/README.md

@@ -21,7 +21,8 @@ $search->addModifier(new ParentIdFilter(1))
 | [PathFilter](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Tree/PathFilter.php)                     | Tree related filters      | Filter by path (depending on use case for all levels or direct children only and with or without the parent item included)                                                                                                                                                                                             |
 | [TagFilter](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Tree/TagFilter.php)                       | Tree related filters      | Filter by tag IDs (it is also possible to include child tags)                                                                                                                                                                                                                                                          |
 | [AssetMetaDataFilter](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Asset/AssetMetaDataFilter.php)  | Asset filters             | Filter by asset meta data attribute. The format of the `$data` which needs to be passed depends on the type of the meta data attribute and is handled by its [field definition adapter](https://github.com/pimcore/generic-data-index-bundle/tree/1.x/src/SearchIndexAdapter/OpenSearch/Asset/FieldDefinitionAdapter). |
-| [WorkspaceFilter](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Workspaces/WorkspaceQuery.php)      | Workspace related filters | Filter based on the user workspaces and permissions (this query is added to the search by default)                                                                                                                                                                                                                     |
+| [WorkspaceQuery](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Workspaces/WorkspaceQuery.php)      | Workspace related filters | Filter based on the user workspaces and permissions for a defined element type (this query is added to the asset/document/data object search by default)                                                                                                                                                               |
+| [ElementWorkspacesQuery](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Workspaces/WorkspaceQuery.php)      | Workspace related filters | Filter based on the user workspaces and permissions respecting all element types (this query is added to the element search by default)                                                                                                                                                                                |
 
 
 
@@ -31,6 +32,14 @@ $search->addModifier(new ParentIdFilter(1))
 |--------------------------------------------------------------------------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [ElementKeySearch](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/FullTextSearch/ElementKeySearch.php) | Full text search  | Search by element key like in the studio UI.<br/><br/>* can be used for wildcard searches - for example "Car*" to find all items starting with "Car". |
 
+### Dependencies
+
+| Modifier                                                                                                                                       | Modifier Category | Description                                               |
+|------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------|
+| [RequiresFilter](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Dependency/RequiresFilter.php) | Dependencies      | Get all elements which the given element requires.        |
+| [RequiredByFilter](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Filter/Dependency/RequiredByFilter.php)    | Dependencies      | Get all elements which are required by the given element. |
+
+
 ### Query Language
 
 | Modifier                                                                                                                         | Modifier Category | Description                                                                               |

doc/04_Searching_For_Data_In_Index/README.md

@@ -14,6 +14,8 @@ The regular way to search for assets, data objects or documents is to use the re
 ```php
 use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\AssetSearchServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\ParentIdFilter;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Sort\Tree\OrderByFullPath;
 
 public function searchAction(SearchProviderInterface $searchProvider, AssetSearchServiceInterface $asserSearchService)
 {
@@ -31,8 +33,10 @@ public function searchAction(SearchProviderInterface $searchProvider, AssetSearc
 
 - Example: This example loads all data objects from the root folder (parent ID 1) with a specific class definition and orders them by their full path.
 ```php
-use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\ParentIdFilter;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Sort\Tree\OrderByFullPath;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\DataObject\DataObjectSearchServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface;
 
 public function searchAction(SearchProviderInterface $searchProvider, DataObjectSearchServiceInterface $dataObjectSearchService)
 {
@@ -52,8 +56,10 @@ public function searchAction(SearchProviderInterface $searchProvider, DataObject
 
 - Example: This example loads all documents from the root folder (parent ID 1) and orders them by their full path.
 ```php
-use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\ParentIdFilter;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Sort\Tree\OrderByFullPath;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Document\DocumentSearchServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface;
 
 public function searchAction(SearchProviderInterface $searchProvider, DocumentSearchServiceInterface $documentSearchService)
 {
@@ -67,6 +73,32 @@ public function searchAction(SearchProviderInterface $searchProvider, DocumentSe
 }
 ```
 
+## Element Search Service
+
+The element search service can be used to search for assets, data objects and documents at the same time.
+
+**Hint:** the element search does not support the calculation of the `hasChildren` attributes. This means that the `hasChildren` attribute will always be `false` for all elements.
+
+- Example: This example loads all elements which are required by the asset ID 123 and orders them by their full path.
+```php
+use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Dependency\RequiredByFilter;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Sort\Tree\OrderByFullPath;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element\ElementSearchServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface;
+
+public function searchAction(SearchProviderInterface $searchProvider, ElementSearchServiceInterface $elementSearchService)
+{
+    $elementSearch = $searchProvider->createElementSearch()
+                ->addModifier(new RequiredByFilter(123, ElementType::ASSET))
+                ->addModifier(new OrderByFullPath())
+                ->setPageSize(50)
+                ->setPage(1);
+
+    $searchResult = $elementSearchService->search($elementSearch);
+}
+```
+
 ## Search Modifiers
 
 To influence the data which gets fetched its possible to use so-called search modifiers.

doc/05_Extending_Data_Index/06_Extend_Search_Index.md

@@ -13,8 +13,10 @@ will find code examples below.
 This event can be used to store additional fields in the search index. Depending on if you would like to index additional
 data for assets or data objects use one of the following two events.
 
-* `Pimcore\Bundle\GenericDataIndexBundle\Event\Asset\UpdateIndexDataEvent`
-* `Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\UpdateIndexDataEvent`
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\Asset\UpdateIndexDataEvent` (assets)
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\UpdateIndexDataEvent` (concrete data object classes)
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\UpdateFolderIndexDataEvent` (data object folders)
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\Document\UpdateIndexDataEvent` (documents)
 
 If you take a look at the source of an indexed document within search index you will find a structure like this:
 
@@ -53,8 +55,10 @@ they are searchable through the full text search (depending on the mapping of th
 With this event it's possible to define the [mapping](https://opensearch.org/docs/latest/field-types/)
 of the additional custom fields. Again there are separate events for assets and data objects.
 
-* `Pimcore\Bundle\GenericDataIndexBundle\Event\Asset\ExtractMappingEvent`
-* `Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\ExtractMappingEvent`
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\Asset\ExtractMappingEvent` (assets)
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\ExtractMappingEvent` (concrete data object classes)
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\ExtractFolderMappingEvent` (data object folders)
+* `Pimcore\Bundle\GenericDataIndexBundle\Event\Document\ExtractMappingEvent` (documents)
 
 
 ### Example 1: Assets

src/Command/Update/IndexUpdateCommand.php

@@ -19,6 +19,7 @@
 use Exception;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\CommandAlreadyRunningException;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\IdNotFoundException;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\GlobalIndexAliasServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue\EnqueueServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexUpdateServiceInterface;
 use Pimcore\Console\AbstractCommand;
@@ -42,10 +43,14 @@ final class IndexUpdateCommand extends AbstractCommand
 
     private const OPTION_RECREATE_INDEX = 'recreate_index';
 
+    private const UPDATE_GLOBAL_ALIASES_ONLY = 'update-global-aliases-only';
+
     private IndexUpdateServiceInterface $indexUpdateService;
 
     private EnqueueServiceInterface $enqueueService;
 
+    private GlobalIndexAliasServiceInterface $globalIndexAliasService;
+
     #[Required]
     public function setIndexUpdateService(IndexUpdateServiceInterface $indexUpdateService): void
     {
@@ -58,6 +63,12 @@ public function setEnqueueService(EnqueueServiceInterface $enqueueService): void
         $this->enqueueService = $enqueueService;
     }
 
+    #[Required]
+    public function setGlobalIndexAliasService(GlobalIndexAliasServiceInterface $globalIndexAliasService): void
+    {
+        $this->globalIndexAliasService = $globalIndexAliasService;
+    }
+
     protected function configure(): void
     {
         $this
@@ -83,6 +94,13 @@ protected function configure(): void
                 'Delete and recreate search indices',
                 null
             )
+            ->addOption(
+                self::UPDATE_GLOBAL_ALIASES_ONLY,
+                null,
+                InputOption::VALUE_NONE,
+                'Updates the global index aliases for data-object and element-search indices only.',
+                null
+            )
             ->setDescription(
                 'Updates index/mapping for all classDefinitions/asset without ' .
                 'deleting them. Adds there elements to index queue.'
@@ -101,6 +119,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             );
         }
 
+        if ($input->getOption(self::UPDATE_GLOBAL_ALIASES_ONLY)) {
+            $this->updateGlobalIndexAliases();
+
+            return self::SUCCESS;
+        }
+
         $this->indexUpdateService->setReCreateIndex($input->getOption(self::OPTION_RECREATE_INDEX));
 
         $updateAll = true;
@@ -173,11 +197,22 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         );
 
         $this->enqueueService->dispatchQueueMessages(true);
+        $this->updateGlobalIndexAliases();
 
         $this->release();
 
         $this->output->writeln('<info>Finished</info>', OutputInterface::VERBOSITY_VERBOSE);
 
         return self::SUCCESS;
     }
+
+    private function updateGlobalIndexAliases(): void
+    {
+        $this->output->writeln(
+            '<info>Update global aliases</info>',
+            OutputInterface::VERBOSITY_VERBOSE
+        );
+        $this->globalIndexAliasService->updateDataObjectAlias();
+        $this->globalIndexAliasService->updateElementSearchAlias();
+    }
 }