Skip to content

Commit

Permalink
merge 1.x
Browse files Browse the repository at this point in the history
  • Loading branch information
lukmzig committed Nov 19, 2024
2 parents 975ba43 + bfb4a6b commit 6370f4e
Show file tree
Hide file tree
Showing 253 changed files with 3,336 additions and 996 deletions.
9 changes: 9 additions & 0 deletions .github/ci/files/config/packages/test/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@ pimcore_open_search_client:
default:
hosts: ['%env(PIMCORE_OPEN_SEARCH_HOST)%']

pimcore_elasticsearch_client:
es_clients:
default:
hosts: ['%env(PIMCORE_ELASTIC_SEARCH_HOST)%']
username: 'elastic'
password: 'somethingsecret'
logger_channel: 'pimcore.elasicsearch'
ssl_verification: false

pimcore:
assets:
metadata:
Expand Down
2 changes: 1 addition & 1 deletion .github/ci/files/config/services_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ services:
class: Pimcore\Tests\Support\Helper\DataType\Calculator

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Search\LocateInTreeServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\LocateInTreeService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\LocateInTreeService
public: true

Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchResultIdListServiceInterface:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,11 @@

.github/ci/scripts/setup-pimcore-environment.sh

# Add Elasticsearch config if specified
if [ "$1" == "elasticsearch" ]; then
CONFIG_FILE_PATH="config/packages/test/config.yaml"
echo -e "\n# Added by functional test script\npimcore_generic_data_index:\n index_service:\n client_params:\n client_type: 'elasticsearch'" >> "$CONFIG_FILE_PATH"
fi

cp .github/ci/files/composer.json .
cp bundles/pimcore/generic-data-index-bundle/codeception.dist.yml .
119 changes: 119 additions & 0 deletions .github/workflows/elastic-search-codeception.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
name: "Codeception Tests with Elasticsearch"

on:
# Enable Later.
#schedule:
# - cron: '0 3 * * 1,3,5'
pull_request:
branches:
- "[0-9]+.[0-9]+"
- "[0-9]+.x"
- "feature-*"
push:
branches:
- "[0-9]+.[0-9]+"
- "[0-9]+.x"

env:
PIMCORE_PROJECT_ROOT: ${{ github.workspace }}
APP_ENV: test
PIMCORE_TEST: 1
PIMCORE_TEST_DB_DSN: "mysql://[email protected]:33006/pimcore_test"
PIMCORE_ELASTIC_SEARCH_HOST: "localhost:5300"
CODECEPTION_BUNDLE_PATH: "bundles/pimcore/generic-data-index-bundle/"

jobs:
codeception-tests:
name: "Codeception tests"
runs-on: "ubuntu-20.04"
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
include:
- { php-version: "8.3", dependencies: "highest", pimcore_version: "11.x-dev as 11.99.9", experimental: true, search_engine: "elasticsearch"}

services:
mariadb:
image: "mariadb:10.11"
ports:
- 33006:3306
env:
MYSQL_ALLOW_EMPTY_PASSWORD: yes

elastic:
image: elasticsearch:8.5.3
ports:
- 5300:9200
env:
discovery.type: "single-node"
ES_JAVA_OPTS: "-Xms1g -Xmx1g"
xpack.security.enabled: "true"
xpack.security.authc.anonymous.roles: "superuser,kibana_admin,kibana_system,kibana_user"
ELASTIC_USERNAME: "elastic"
ELASTIC_PASSWORD: "somethingsecret"

steps:
- name: "Checkout code"
uses: "actions/checkout@v2"
with:
path: "bundles/pimcore/generic-data-index-bundle"

- name: "Copy .github directory"
env:
REQUIRE_ADMIN_BUNDLE: "${{ matrix.require_admin_bundle }}"
run: |
cp -R bundles/pimcore/generic-data-index-bundle/.github .github
- name: "Install PHP"
uses: "shivammathur/setup-php@v2"
with:
coverage: xdebug
ini-values: display_errors=On, display_startup_errors=On, error_reporting=32767
php-version: "${{ matrix.php-version }}"

- name: Verify MariaDB connection
run: |
cp .github/ci/files/.my.cnf ~/.my.cnf
while ! mysqladmin ping --silent; do
sleep 1
done
- name: "Setup Pimcore environment"
env:
REQUIRE_ADMIN_BUNDLE: "${{ matrix.require_admin_bundle }}"
run: |
mysql -e "CREATE DATABASE pimcore_test CHARSET=utf8mb4;"
cp -R bundles/pimcore/generic-data-index-bundle/tests tests
mkdir src
chmod 755 .github/ci/scripts/setup-pimcore-environment-functional-tests.sh
.github/ci/scripts/setup-pimcore-environment-functional-tests.sh "${{ matrix.search_engine }}"
- name: "Wait for Elasticsearch to be up"
run: |
if [ "${{ matrix.search_engine }}" == "elasticsearch" ]; then
echo "Waiting for Elasticsearch to be ready..."
until curl -s -u elastic:somethingsecret http://localhost:5300/_cluster/health | grep -q '"status":"green"'; do
echo "Waiting... Elasticsearch is not yet available."
sleep 5
done
echo "Elasticsearch is up and running!"
else
echo "Using OpenSearch for testing, skipping Elasticsearch wait."
fi
- name: "Update Pimcore version"
env:
PIMCORE_VERSION: "${{ matrix.pimcore_version }}"
run: |
if [ ! -z "$PIMCORE_VERSION" ]; then
composer require --no-update pimcore/pimcore:"${PIMCORE_VERSION}"
fi
- name: "Install dependencies with Composer"
uses: "ramsey/composer-install@v2"
with:
dependency-versions: "${{ matrix.dependencies }}"

- name: "Run Codeception"
run: |
vendor/bin/codecept run -c . -vvv --xml --coverage-xml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "Codeception Tests"
name: "Codeception Tests with OpenSearch"

on:
# Enable Later.
Expand Down Expand Up @@ -30,9 +30,9 @@ jobs:
strategy:
matrix:
include:
- { php-version: "8.2", dependencies: "lowest", pimcore_version: "", experimental: false }
- { php-version: "8.3", dependencies: "highest", pimcore_version: "", experimental: false }
- { php-version: "8.3", dependencies: "highest", pimcore_version: "11.x-dev as 11.99.9", experimental: true }
- { php-version: "8.2", dependencies: "lowest", pimcore_version: "", experimental: false, search_engine: "openSearch" }
- { php-version: "8.3", dependencies: "highest", pimcore_version: "", experimental: false, search_engine: "openSearch"}
- { php-version: "8.3", dependencies: "highest", pimcore_version: "11.x-dev as 11.99.9", experimental: true, search_engine: "openSearch"}

services:
mariadb:
Expand Down Expand Up @@ -89,7 +89,7 @@ jobs:
cp -R bundles/pimcore/generic-data-index-bundle/tests tests
mkdir src
chmod 755 .github/ci/scripts/setup-pimcore-environment-functional-tests.sh
.github/ci/scripts/setup-pimcore-environment-functional-tests.sh
.github/ci/scripts/setup-pimcore-environment-functional-tests.sh "${{ matrix.search_engine }}"
- name: "Update Pimcore version"
env:
Expand Down
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ title: Generic Data Index

# Pimcore Generic Data Index

The Pimcore Generic Data Index Bundle provides a centralized way to index and search assets and data objects in Pimcore via OpenSearch indices.
It is shipped with the OpenSearch client and provides a central configuration for it in order to be used in other bundles.
The Pimcore Generic Data Index Bundle provides a centralized way to index and search elements (assets, data objects and documents) in Pimcore via indices (e.g OpenSearch, Elasticsearch).
It is shipped with the OpenSearch and Elasticsearch clients and provides a central configuration for them in order to be used in other bundles.
This bundle can be extended and customized to fit your specific needs, for example if you would like to extend the search indices with custom attributes.

## Features in a Nutshell
- Based on OpenSearch
- Based on OpenSearch/Elasticsearch
- Centralized data index for multiple bundles (Portal Engine, Studio API/UI, etc.)
- Indexing of all documents, assets and data objects
- Provides search services and models to search, filter and aggregate the data saved in the OpenSearch indices
- Provides search services and models to search, filter and aggregate the data saved in the search indices

## Documentation Overview
- [Installation](./doc/01_Installation/README.md)
Expand Down
10 changes: 7 additions & 3 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@
"minimum-stability": "dev",
"require": {
"php": "~8.2.0 || ~8.3.0 ",
"pimcore/pimcore": "^11.3.0",
"pimcore/pimcore": "^11.x-dev",
"pimcore/static-resolver-bundle": "^1.4.0",
"pimcore/opensearch-client": "^1.0.0",
"pimcore/opensearch-client": "^1.x-dev",
"pimcore/elasticsearch-client": "^1.x-dev",
"doctrine/orm": "^2.17.2",
"symfony/scheduler": "^6.4",
"symfony/messenger": "^6.4"
Expand All @@ -42,7 +43,10 @@
"autoload": {
"psr-4": {
"Pimcore\\Bundle\\GenericDataIndexBundle\\": "src/"
}
},
"files": [
"src/autoload.php"
]
},
"autoload-dev": {
"psr-4": {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,79 +1,79 @@
imports:
- { resource: 'services/search-index-adapter/open-search/modifiers/*' }
- { resource: 'services/search-index-adapter/open-search/pql-field-name-transformers.yaml' }
- { resource: 'services/search-index-adapter/default-search/modifiers/*' }
- { resource: 'services/search-index-adapter/default-search/pql-field-name-transformers.yaml' }

services:
_defaults:
autowire: true
autoconfigure: true
public: false

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\SearchIndexServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\OpenSearchService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DefaultSearchService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexAliasServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\IndexAliasService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\IndexAliasService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\SearchExecutionServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\SearchExecutionService
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\SearchExecutionServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\SearchExecutionService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\BulkOperationServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\BulkOperationService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\BulkOperationService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\PathServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\PathService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\PathService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Search\Pagination\PaginationInfoServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Pagination\PaginationInfoService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Pagination\PaginationInfoService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Search\LocateInTreeServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\LocateInTreeService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\LocateInTreeService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Search\FetchIdsServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\FetchIdsService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\FetchIdsService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\FetchIdsBySearchServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\FetchIdsBySearchService
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\FetchIdsBySearchServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\FetchIdsBySearchService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerService
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\MappingAnalyzerServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\MappingAnalyzerService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexStatsServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\IndexStatsService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\IndexStatsService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\IndexMappingService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\IndexMappingService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Workspace\QueryServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Workspace\QueryService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Workspace\QueryService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Workspace\ElementWorkspacesQueryServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Workspace\ElementWorkspacesQueryService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Workspace\ElementWorkspacesQueryService

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\QueryLanguage\PqlAdapterInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\PqlAdapter
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\PqlAdapter

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\SubQueriesProcessorInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\SubQueriesProcessor
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\SubQueriesProcessorInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\SubQueriesProcessor


Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameValidator\:
resource: '../../../src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameValidator'
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\FieldNameValidator\:
resource: '../../../src/SearchIndexAdapter/DefaultSearch/QueryLanguage/FieldNameValidator'
tags: ['pimcore.generic_data_index.pql_field_name_validator']

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DataObject\IndexIconUpdateServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\DataObject\IndexIconUpdateService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\IndexIconUpdateService
arguments:
$openSearchClient: '@generic-data-index.opensearch-client'
$client: '@generic-data-index.search-client'
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ services:
public: false

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Search\Modifier\SearchModifierServiceInterface:
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\SearchModifierService
class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\SearchModifierService
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
services:
_defaults:
autowire: true
autoconfigure: true
public: false


Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Aggregation\AssetAggregations: ~
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Aggregation\TreeAggregations: ~

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Filter\BasicFilters: ~
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Filter\AssetFilters: ~
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Filter\TreeFilters: ~
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Filter\DependencyFilters: ~
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Filter\FieldTypeFilters: ~

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\FullTextSearch\FullTextSearchHandlers: ~

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Sort\TreeSortHandlers: ~
Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Sort\OrderByFieldsHandlers: ~

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\QueryLanguage\QueryLanguageHandlers: ~

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Filter\Workspace\WorkspaceQueryHandler: ~
Loading

0 comments on commit 6370f4e

Please sign in to comment.