40 lines
1.3 KiB
PHP
40 lines
1.3 KiB
PHP
<?php
|
|
|
|
namespace Modules\AI\Services;
|
|
|
|
use Modules\AI\Models\AIKnowledgeSource;
|
|
|
|
final class KnowledgeIndexer
|
|
{
|
|
public function index(AIKnowledgeSource $source, int $chunkSize = 1800): AIKnowledgeSource
|
|
{
|
|
$content = trim((string) $source->content);
|
|
$source->chunks()->delete();
|
|
$chunks = $content === '' ? [] : $this->split($content, max(500, $chunkSize));
|
|
foreach ($chunks as $position => $chunk) {
|
|
$source->chunks()->create(['position' => $position + 1, 'content' => $chunk]);
|
|
}
|
|
$source->update(['checksum' => hash('sha256', $content), 'status' => 'indexed', 'chunk_count' => count($chunks), 'indexed_at' => now()]);
|
|
|
|
return $source->refresh();
|
|
}
|
|
|
|
private function split(string $content, int $size): array
|
|
{
|
|
$paragraphs = preg_split('/\R{2,}/', $content) ?: [$content];
|
|
$chunks = [];
|
|
$buffer = '';
|
|
foreach ($paragraphs as $paragraph) {
|
|
if (strlen($buffer) + strlen($paragraph) + 2 > $size && $buffer !== '') {
|
|
$chunks[] = trim($buffer);
|
|
$buffer = '';
|
|
} $buffer .= $paragraph."\n\n";
|
|
}
|
|
if (trim($buffer) !== '') {
|
|
$chunks[] = trim($buffer);
|
|
}
|
|
|
|
return $chunks;
|
|
}
|
|
}
|