Skip to content

Commit 6de5d57

Browse files
committed
feat(vector): add save/load persistence to VectorRetriever
Add VectorRetriever.save() and VectorRetriever.load() for FAISS-backed file persistence. save() writes the vector index and a .registry JSON sidecar; load() restores both. InMemoryVectorStore raises NotImplementedError. Add 3 tests (save/load roundtrip, registry integrity, InMemory raises).
1 parent c91e075 commit 6de5d57

2 files changed

Lines changed: 143 additions & 0 deletions

File tree

src/lang2sql/components/retrieval/vector.py

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,78 @@ def add(self, chunks: list[IndexedChunk]) -> None:
168168
self._vectorstore.upsert(ids, vectors)
169169
self._registry.update({c["chunk_id"]: c for c in chunks})
170170

171+
# ── Persistence ──────────────────────────────────────────────────
172+
173+
def save(self, path: str) -> None:
174+
"""벡터 인덱스와 registry를 path에 저장.
175+
176+
FAISSVectorStore처럼 save()를 지원하는 store에서만 동작한다.
177+
InMemoryVectorStore 등 save()가 없는 store는 NotImplementedError.
178+
179+
저장 파일:
180+
{path} — FAISSVectorStore 벡터 인덱스
181+
{path}.meta — chunk_id 순서 목록 (FAISSVectorStore 내부)
182+
{path}.registry — registry JSON
183+
"""
184+
import json
185+
import pathlib
186+
187+
save_fn = getattr(self._vectorstore, "save", None)
188+
if save_fn is None:
189+
raise NotImplementedError(
190+
f"{type(self._vectorstore).__name__} does not support save(). "
191+
"Use FAISSVectorStore for file-based persistence."
192+
)
193+
save_fn(path)
194+
pathlib.Path(path + ".registry").write_text(
195+
json.dumps(self._registry), encoding="utf-8"
196+
)
197+
198+
@classmethod
199+
def load(
200+
cls,
201+
path: str,
202+
*,
203+
embedding: EmbeddingPort,
204+
top_n: int = 5,
205+
score_threshold: float = 0.0,
206+
name: Optional[str] = None,
207+
hook: Optional[TraceHook] = None,
208+
) -> "VectorRetriever":
209+
"""저장된 인덱스와 registry를 복원해 VectorRetriever를 반환.
210+
211+
save()로 저장한 path를 그대로 전달한다.
212+
embedding은 쿼리 시 embed_query()에 사용되므로 반드시 전달해야 한다.
213+
214+
Args:
215+
path: save() 시 사용한 경로.
216+
embedding: EmbeddingPort 구현체.
217+
top_n: 최대 반환 스키마/컨텍스트 수. 기본 5.
218+
score_threshold: 이 점수 이하는 결과에서 제외. 기본 0.0.
219+
"""
220+
import json
221+
import pathlib
222+
223+
from ...integrations.vectorstore.faiss_ import FAISSVectorStore
224+
225+
registry_path = pathlib.Path(path + ".registry")
226+
if not registry_path.exists():
227+
raise FileNotFoundError(f"Registry file not found: {registry_path}")
228+
229+
store = FAISSVectorStore.load(path)
230+
registry = json.loads(registry_path.read_text(encoding="utf-8"))
231+
return cls(
232+
vectorstore=store,
233+
embedding=embedding,
234+
registry=registry,
235+
top_n=top_n,
236+
score_threshold=score_threshold,
237+
name=name,
238+
hook=hook,
239+
)
240+
241+
# ── Core retrieval ────────────────────────────────────────────────
242+
171243
def _run(self, query: str) -> RetrievalResult:
172244
"""
173245
Args:

tests/test_components_vector_retriever.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,3 +502,74 @@ def test_catalog_chunker_split_batch():
502502
by_chunk = [c for entry in CATALOG for c in chunker.chunk(entry)]
503503

504504
assert [c["chunk_id"] for c in by_split] == [c["chunk_id"] for c in by_chunk]
505+
506+
507+
# ---------------------------------------------------------------------------
508+
# 20-22. VectorRetriever save / load (FAISS 필요)
509+
# ---------------------------------------------------------------------------
510+
511+
faiss = pytest.importorskip("faiss", reason="faiss-cpu not installed")
512+
513+
514+
class FakeEmbeddingFAISS:
515+
"""FAISS L2 정규화에서 zero-vector 오류가 안 나도록 비영벡터를 반환."""
516+
517+
def _vec(self, text: str) -> list[float]:
518+
# 텍스트별로 구별 가능한 비영벡터
519+
h = abs(hash(text)) % 900 + 100
520+
return [h * 0.001, 1.0, 1.0, 1.0]
521+
522+
def embed_query(self, text: str) -> list[float]:
523+
return self._vec(text)
524+
525+
def embed_texts(self, texts: list[str]) -> list[list[float]]:
526+
return [self._vec(t) for t in texts]
527+
528+
529+
def test_save_and_load_returns_same_results(tmp_path):
530+
"""save → load 후 동일 쿼리에 동일 스키마가 반환된다."""
531+
path = str(tmp_path / "catalog")
532+
embedding = FakeEmbeddingFAISS()
533+
534+
from lang2sql.integrations.vectorstore.faiss_ import FAISSVectorStore
535+
536+
store = FAISSVectorStore(index_path=path + ".faiss")
537+
chunks = CatalogChunker().split(CATALOG)
538+
original = VectorRetriever.from_chunks(chunks, embedding=embedding, vectorstore=store)
539+
original.save(path)
540+
541+
loaded = VectorRetriever.load(path, embedding=embedding)
542+
result = loaded.run("주문 정보")
543+
544+
assert len(result.schemas) > 0
545+
assert result.schemas[0]["name"] == original.run("주문 정보").schemas[0]["name"]
546+
547+
548+
def test_load_registry_intact(tmp_path):
549+
"""load 후 registry의 키·값이 원본과 동일하다."""
550+
path = str(tmp_path / "catalog")
551+
embedding = FakeEmbeddingFAISS()
552+
553+
from lang2sql.integrations.vectorstore.faiss_ import FAISSVectorStore
554+
555+
store = FAISSVectorStore(index_path=path + ".faiss")
556+
chunks = CatalogChunker().split(CATALOG)
557+
original = VectorRetriever.from_chunks(chunks, embedding=embedding, vectorstore=store)
558+
original.save(path)
559+
560+
loaded = VectorRetriever.load(path, embedding=embedding)
561+
562+
assert set(loaded._registry.keys()) == set(original._registry.keys())
563+
for chunk_id, chunk in original._registry.items():
564+
assert loaded._registry[chunk_id]["text"] == chunk["text"]
565+
assert loaded._registry[chunk_id]["source_id"] == chunk["source_id"]
566+
567+
568+
def test_save_raises_for_inmemory():
569+
"""InMemoryVectorStore는 save()를 지원하지 않아 NotImplementedError가 발생한다."""
570+
embedding = FakeEmbeddingFAISS()
571+
chunks = CatalogChunker().split(CATALOG)
572+
retriever = VectorRetriever.from_chunks(chunks, embedding=embedding) # InMemory 기본값
573+
574+
with pytest.raises(NotImplementedError, match="does not support save"):
575+
retriever.save("/tmp/should_not_exist")

0 commit comments

Comments
 (0)