Mark Gregory
5dbb04e3ad
* OpenAI-compatible provider adapter * OpenAI support * LM Studio support * Ollama OpenAI-compatible support * Encrypted API-key storage * Default provider and model selection * Provider connection testing * Reusable prompt library * Prompt variables and templates * Prompt version field * AI execution history * Token-usage storage * Duration and failure tracking * Manual knowledge sources * Knowledge-source chunking * Knowledge indexing status * Authenticated AI REST endpoint * AI administration dashboard * AI permissions * Module registration and boot test
21 lines
1.0 KiB
PHP
21 lines
1.0 KiB
PHP
<?php
|
|
namespace Modules\AI\Services;
|
|
use Modules\AI\Models\AIKnowledgeSource;
|
|
final class KnowledgeIndexer
|
|
{
|
|
public function index(AIKnowledgeSource $source,int $chunkSize=1800): AIKnowledgeSource
|
|
{
|
|
$content=trim((string)$source->content); $source->chunks()->delete();
|
|
$chunks=$content===''?[]:$this->split($content,max(500,$chunkSize));
|
|
foreach($chunks as $position=>$chunk) $source->chunks()->create(['position'=>$position+1,'content'=>$chunk]);
|
|
$source->update(['checksum'=>hash('sha256',$content),'status'=>'indexed','chunk_count'=>count($chunks),'indexed_at'=>now()]);
|
|
return $source->refresh();
|
|
}
|
|
private function split(string $content,int $size): array
|
|
{
|
|
$paragraphs=preg_split('/\R{2,}/',$content) ?: [$content]; $chunks=[]; $buffer='';
|
|
foreach($paragraphs as $paragraph){ if(strlen($buffer)+strlen($paragraph)+2>$size && $buffer!==''){ $chunks[]=trim($buffer); $buffer=''; } $buffer.=$paragraph."\n\n"; }
|
|
if(trim($buffer)!=='') $chunks[]=trim($buffer); return $chunks;
|
|
}
|
|
}
|