IBX-11441: Provided proper index limit for the StringField for search engines

phpstan-baseline-7.4.neon

@@ -150,12 +150,6 @@ parameters:
 			count: 1
 			path: src/lib/FieldType/Selection/SearchField.php
 
-		-
-			message: '#^Parameter \#1 \$str of function mb_substr expects string, string\|false given\.$#'
-			identifier: argument.type
-			count: 1
-			path: src/lib/FieldType/TextBlock/SearchField.php
-
 		-
 			message: '#^Parameter \#1 \$str of function mb_substr expects string, string\|false given\.$#'
 			identifier: argument.type

phpstan-baseline-gte-8.0.neon

@@ -162,12 +162,6 @@ parameters:
 			count: 1
 			path: src/lib/FieldType/Selection/SearchField.php
 
-		-
-			message: '#^Parameter \#1 \$string of function mb_substr expects string, string\|false given\.$#'
-			identifier: argument.type
-			count: 1
-			path: src/lib/FieldType/TextBlock/SearchField.php
-
 		-
 			message: '#^Parameter \#1 \$string of function mb_substr expects string, string\|false given\.$#'
 			identifier: argument.type

phpstan-baseline.neon

@@ -11946,12 +11946,6 @@ parameters:
 			count: 1
 			path: src/lib/FieldType/StorageGateway.php
 
-		-
-			message: '#^Parameter \#1 \$string of method Ibexa\\Core\\FieldType\\TextBlock\\SearchField\:\:extractShortText\(\) expects string, array\|bool\|float\|int\|string\|null given\.$#'
-			identifier: argument.type
-			count: 1
-			path: src/lib/FieldType/TextBlock/SearchField.php
-
 		-
 			message: '#^Access to an undefined property Ibexa\\Contracts\\Core\\FieldType\\Value\:\:\$text\.$#'
 			identifier: property.notFound
@@ -15828,12 +15822,6 @@ parameters:
 			count: 1
 			path: src/lib/MVC/Symfony/Translation/ExceptionMessageTemplateFileVisitor.php
 
-		-
-			message: '#^Access to an undefined property PhpParser\\Node\\Arg\|PhpParser\\Node\\VariadicPlaceholder\:\:\$value\.$#'
-			identifier: property.notFound
-			count: 1
-			path: src/lib/MVC/Symfony/Translation/ExceptionMessageTemplateFileVisitor.php
-
 		-
 			message: '#^Cannot call method error\(\) on Psr\\Log\\LoggerInterface\|null\.$#'
 			identifier: method.nonObject
@@ -18084,7 +18072,6 @@ parameters:
 			count: 1
 			path: src/lib/Persistence/Legacy/Content/FieldValue/Converter/DateAndTimeConverter.php
 
-
 		-
 			message: '#^Method Ibexa\\Core\\Persistence\\Legacy\\Content\\FieldValue\\Converter\\DateAndTimeConverter\:\:toFieldDefinition\(\) has no return type specified\.$#'
 			identifier: missingType.return
@@ -26173,12 +26160,6 @@ parameters:
 			count: 1
 			path: src/lib/Search/Common/FieldValueMapper/MultipleStringMapper.php
 
-		-
-			message: '#^Method Ibexa\\Core\\Search\\Common\\FieldValueMapper\\StringMapper\:\:convert\(\) should return string but returns string\|null\.$#'
-			identifier: return.type
-			count: 1
-			path: src/lib/Search/Common/FieldValueMapper/StringMapper.php
-
 		-
 			message: '#^Method Ibexa\\Core\\Search\\Common\\IncrementalIndexer\:\:createSearchIndex\(\) has no return type specified\.$#'
 			identifier: missingType.return
@@ -32305,7 +32286,6 @@ parameters:
 			count: 1
 			path: tests/integration/Core/Repository/Common/SlugConverter.php
 
-
 		-
 			message: '#^Cannot call method getValue\(\) on Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Field\|null\.$#'
 			identifier: method.nonObject
@@ -41930,7 +41910,6 @@ parameters:
 			count: 1
 			path: tests/integration/Core/Repository/ObjectStateServiceTest.php
 
-
 		-
 			message: '#^Method Ibexa\\Tests\\Integration\\Core\\Repository\\ObjectStateServiceTest\:\:assertObjectsLoadedByIdentifiers\(\) has no return type specified\.$#'
 			identifier: missingType.return
@@ -41961,7 +41940,6 @@ parameters:
 			count: 1
 			path: tests/integration/Core/Repository/ObjectStateServiceTest.php
 
-
 		-
 			message: '#^Method Ibexa\\Tests\\Integration\\Core\\Repository\\ObjectStateServiceTest\:\:deleteExistingObjectStateGroups\(\) has no return type specified\.$#'
 			identifier: missingType.return

src/lib/FieldType/TextBlock/SearchField.php

@@ -21,7 +21,7 @@ public function getIndexData(Field $field, FieldDefinition $fieldDefinition)
         return [
             new Search\Field(
                 'value',
-                $this->extractShortText($field->value->data),
+                $this->extractText($field->value->data),
                 new Search\FieldType\StringField()
             ),
             new Search\Field(
@@ -33,15 +33,17 @@ public function getIndexData(Field $field, FieldDefinition $fieldDefinition)
     }
 
     /**
-     * Extracts short snippet of the given $string.
-     *
-     * @param string $string
-     *
-     * @return string
+     * @param mixed $string
      */
-    private function extractShortText($string)
+    private function extractText($string): string
     {
-        return mb_substr(strtok(trim((string)$string), "\r\n"), 0, 255);
+        if (!is_string($string)) {
+            return '';
+        }
+
+        $lines = explode("\n", str_replace(["\r\n", "\r"], "\n", $string));
+
+        return implode(' ', array_map('trim', $lines));
     }
 
     public function getIndexDefinition()

src/lib/MVC/Symfony/Translation/ExceptionMessageTemplateFileVisitor.php

@@ -48,7 +48,7 @@
         $this->traverser->addVisitor($this);
     }
 
     public function enterNode(Node $node): void
     {
         $methodCallNodeName = null;
         if ($node instanceof Node\Expr\MethodCall) {
@@ -65,12 +65,21 @@
 
         $ignore = $this->isIgnore($node);
 
+        if (!$node->args[0] instanceof Node\Arg) {
+            return;
+        }
+
         if (!$node->args[0]->value instanceof String_) {
             if ($ignore) {
                 return;
             }
 
-            $message = sprintf('Can only extract the translation id from a scalar string, but got "%s". Please refactor your code to make it extractable, or add the doc comment /** @Ignore */ to this code element (in %s on line %d).', get_class($node->args[0]->value), $this->file, $node->args[0]->value->getLine());
+            $message = sprintf(
+                'Can only extract the translation id from a scalar string, but got "%s". Please refactor your code to make it extractable, or add the doc comment /** @Ignore */ to this code element (in %s on line %d).',
+                get_class($node->args[0]->value),
+                $this->file,
+                $node->args[0]->value->getLine()
+            );
 
             $this->logger->error($message);
 

src/lib/Resources/settings/search_engines/field_value_mappers.yml

@@ -62,6 +62,7 @@ services:
             - { name: ibexa.search.common.field_value.mapper, maps: Ibexa\Contracts\Core\Search\FieldType\PriceField }
 
     Ibexa\Core\Search\Common\FieldValueMapper\StringMapper:
+        autoconfigure: true
         class: Ibexa\Core\Search\Common\FieldValueMapper\StringMapper
         tags:
             - { name: ibexa.search.common.field_value.mapper, maps: Ibexa\Contracts\Core\Search\FieldType\StringField }

src/lib/Search/Common/FieldValueMapper/StringMapper.php

@@ -9,14 +9,27 @@
 use Ibexa\Contracts\Core\Search\Field;
 use Ibexa\Contracts\Core\Search\FieldType;
 use Ibexa\Core\Search\Common\FieldValueMapper;
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
 
 /**
  * Common string field value mapper implementation.
  */
-class StringMapper extends FieldValueMapper
+class StringMapper extends FieldValueMapper implements LoggerAwareInterface
 {
+    use LoggerAwareTrait;
+
+    public function __construct(
+        ?LoggerInterface $logger = null
+    ) {
+        $this->logger = $logger ?? new NullLogger();
+    }
+
     public const REPLACE_WITH_SPACE_PATTERN = '([\x09\x0B\x0C]+)';
     public const REMOVE_PATTERN = '([\x00-\x08\x0E-\x1F]+)';
+    public const MAX_TERM_LENGTH = 32766;
 
     public function canMap(Field $field): bool
     {
@@ -43,11 +56,28 @@ protected function convert($value): string
         );
 
         // Remove non-printable characters.
-        return preg_replace(
+        $value = preg_replace(
             self::REMOVE_PATTERN,
             '',
             (string)$value
         );
+
+        // Enforce Lucene's bytes MAX_TERM_LENGTH to avoid silent indexing failures
+        $value = (string)$value;
+        $truncated = mb_strcut($value, 0, self::MAX_TERM_LENGTH);
+
+        if (strlen($truncated) < strlen($value)) {
+            $this->logger->warning(
+                sprintf(
+                    'String field value was truncated from %d to %d bytes (max term length: %d).',
+                    strlen($value),
+                    strlen($truncated),
+                    self::MAX_TERM_LENGTH
+                )
+            );
+        }
+
+        return $truncated;
     }
 }
 

tests/integration/Core/Repository/FieldType/TextBlockIntegrationTest.php

@@ -287,24 +287,24 @@ public function providerForTestIsNotEmptyValue()
 
     protected function getValidSearchValueOne()
     {
-        return 'caution is the " path to mediocrity' . PHP_EOL . 'something completely different';
+        return 'caution is the " path to mediocrity something completely different';
     }
 
     protected function getSearchTargetValueOne()
     {
         // ensure case-insensitivity
-        return strtoupper('caution is the " path to mediocrity');
+        return strtoupper('caution is the " path to mediocrity something completely different');
     }
 
     protected function getValidSearchValueTwo()
     {
-        return "truth suffers from ' too much analysis\n hello and goodbye";
+        return "truth suffers from ' too much analysis hello and goodbye";
     }
 
     protected function getSearchTargetValueTwo()
     {
         // ensure case-insensitivity
-        return strtoupper("truth suffers from ' too much analysis");
+        return strtoupper("truth suffers from ' too much analysis hello and goodbye");
     }
 
     protected function getFullTextIndexedFieldData()

tests/lib/Search/Common/FieldValueMapper/RemoteIdentifierMapperTest.php

@@ -15,6 +15,7 @@
 use Ibexa\Contracts\Core\Search\FieldType\StringField;
 use Ibexa\Core\Search\Common\FieldValueMapper\RemoteIdentifierMapper;
 use Ibexa\Tests\Core\Search\TestCase;
+use Psr\Log\LoggerInterface;
 
 /**
  * @covers \Ibexa\Core\Search\Common\FieldValueMapper\RemoteIdentifierMapper
@@ -26,7 +27,9 @@ final class RemoteIdentifierMapperTest extends TestCase
 
     protected function setUp(): void
     {
-        $this->mapper = new RemoteIdentifierMapper();
+        $this->mapper = new RemoteIdentifierMapper(
+            $this->createMock(LoggerInterface::class)
+        );
     }
 
     /**

tests/lib/Search/Common/FieldValueMapper/StringMapperTest.php

@@ -0,0 +1,105 @@
+<?php
+
+/**
+ * @copyright Copyright (C) Ibexa AS. All rights reserved.
+ * @license For full copyright and license information view LICENSE file distributed with this source code.
+ */
+declare(strict_types=1);
+
+namespace Ibexa\Tests\Core\Search\Common\FieldValueMapper;
+
+use Ibexa\Contracts\Core\Search\Field;
+use Ibexa\Contracts\Core\Search\FieldType\StringField;
+use Ibexa\Core\Search\Common\FieldValueMapper\StringMapper;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Ibexa\Core\Search\Common\FieldValueMapper\StringMapper
+ */
+final class StringMapperTest extends TestCase
+{
+    private StringMapper $mapper;
+
+    protected function setUp(): void
+    {
+        $this->mapper = new StringMapper();
+    }
+
+    public function testCanMap(): void
+    {
+        $field = $this->createFieldWithValue('hello', new StringField());
+
+        self::assertTrue($this->mapper->canMap($field));
+    }
+
+    public function testMapsPlainString(): void
+    {
+        $field = $this->createFieldWithValue('hello world', new StringField());
+
+        self::assertSame('hello world', $this->mapper->map($field));
+    }
+
+    public function testStripsNonPrintableCharacters(): void
+    {
+        $field = $this->createFieldWithValue("hello\x01\x02world", new StringField());
+
+        self::assertSame('helloworld', $this->mapper->map($field));
+    }
+
+    public function testReplacesTabAndVerticalWhitespaceWithSpace(): void
+    {
+        $field = $this->createFieldWithValue("hello\x09world\x0Bfoo", new StringField());
+
+        self::assertSame('hello world foo', $this->mapper->map($field));
+    }
+
+    public function testTruncatesToMaxTermLength(): void
+    {
+        $longValue = str_repeat('a', StringMapper::MAX_TERM_LENGTH + 100);
+        $field = $this->createFieldWithValue($longValue, new StringField());
+        $result = $this->mapper->map($field);
+
+        self::assertSame(StringMapper::MAX_TERM_LENGTH, strlen($result));
+    }
+
+    public function testTruncatesMultibyteStringAtCharacterBoundary(): void
+    {
+        // Each UTF-8 character here is 3 bytes (€ = E2 82 AC).
+        // Fill up to just past the limit so truncation must happen on a char boundary.
+        $char = '€';
+        $charBytes = strlen($char);
+
+        self::assertSame(3, $charBytes);
+
+        $count = (int) ceil((StringMapper::MAX_TERM_LENGTH + $charBytes) / $charBytes);
+        $longValue = str_repeat($char, $count);
+
+        $field = $this->createFieldWithValue($longValue, new StringField());
+        $result = $this->mapper->map($field);
+
+        self::assertSame(StringMapper::MAX_TERM_LENGTH, strlen($result));
+        // Result must be valid UTF-8 (no split mid-character).
+        self::assertSame(1, preg_match('//u', $result));
+    }
+
+    public function testValueWithinLimitIsNotTruncated(): void
+    {
+        $value = str_repeat('a', StringMapper::MAX_TERM_LENGTH);
+        $field = $this->createFieldWithValue($value, new StringField());
+
+        self::assertSame($value, $this->mapper->map($field));
+    }
+
+    private function createFieldWithValue(string $value, StringField $type): Field
+    {
+        $field = $this->createMock(Field::class);
+        $field
+            ->method('getValue')
+            ->willReturn($value);
+        $field
+            ->method('getType')
+            ->willReturn($type);
+
+        return $field;
+    }
+}