Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions starter/studio-platform-starter-ai-web/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,10 @@ studio:
projection job 상태와 미리 계산된 좌표만 저장한다. 화면 요청 시마다 고차원 벡터를 다시 projection하지 않는다.

기본 알고리즘은 `PCA`다. v1 구현은 Java 내장 연산으로 PCA 좌표를 계산하고, 후속 UMAP/t-SNE는
`VectorProjectionGenerator` 구현을 추가해 확장한다. `targetTypes`가 비어 있으면 전체 vector item을 대상으로 한다.
`VectorProjectionGenerator` 구현을 추가해 확장한다. `targetTypes`는 UI 문서 분류가 아니라
`tb_ai_document_chunk.object_type`에 저장된 RAG index objectType 기준이다. 예를 들어 `attachment`,
`forums-post-attachment`, 정책 object type 값처럼 색인 job이 사용한 objectType을 지정한다.
`targetTypes`가 비어 있으면 전체 vector item을 대상으로 한다.
`filters`는 v1에서 metadata equality 조건만 사용하며 null 값은 무시한다. 한 projection job은 최대
1,000개 vector item, 2,048 embedding dimension까지 처리한다. 더 큰 범위는 `targetTypes`나 metadata filter로 나눠 생성한다.
projection 생성과 point/item/search visualization 조회는 object별 ACL을 행마다 평가하지 않는 corpus-level 관리 API이므로
Expand All @@ -139,7 +142,9 @@ Content-Type: application/json

응답은 즉시 `REQUESTED`를 반환한다. 서버는 비동기 job에서 `PROCESSING`으로 전환한 뒤 기존
`tb_ai_document_chunk`의 embedding을 읽어 좌표를 만들고, 기존 point를 삭제 후 재생성한다.
완료 시 `COMPLETED`, 실패 시 `FAILED`와 `errorMessage`를 저장한다.
완료 시 `COMPLETED`, 실패 시 `FAILED`와 `errorMessage`를 저장한다. 완료된 projection의 목록/상세 응답
`targetTypes`는 실제 좌표에 포함된 `tb_ai_document_chunk.object_type` 목록을 반환하므로 클라이언트는 이 값을
필터와 범례 구성에 사용할 수 있다.

```json
{
Expand Down Expand Up @@ -184,7 +189,7 @@ Content-Type: application/json
projection point를 매칭하고, query 위치는 매칭된 Top-K point 좌표의 평균으로 계산한다.
매칭 point가 없으면 `query.x`, `query.y`는 `null`, `results`는 빈 배열로 200 응답한다.
검색은 선택된 projection의 `targetTypes`와 `filters` 범위를 기준으로 제한하고, 요청 `targetTypes`가 있으면
projection 범위와 교집합인 type만 대상으로 한다. `query`는 provider 비용과 지연을 제한하기 위해 최대
projection 범위와 교집합인 RAG index objectType만 대상으로 한다. `query`는 provider 비용과 지연을 제한하기 위해 최대
2,000자까지 허용한다.

### RAG Index Job Management
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ private ProjectionSummaryResponse summary(VectorProjection projection) {
projection.name(),
projection.algorithm().name(),
projection.status().name(),
projection.targetTypes(),
projection.itemCount(),
projection.createdAt(),
projection.completedAt());
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package studio.one.platform.ai.web.dto.visualization;

import java.time.Instant;
import java.util.List;

public record ProjectionSummaryResponse(
String projectionId,
String name,
String algorithm,
String status,
List<String> targetTypes,
int itemCount,
Instant createdAt,
Instant completedAt) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,20 @@ void pointsReturnsClientOrientedShape() {
});
}

@Test
void listProjectionIncludesActualTargetTypes() {
VectorProjectionService projectionService = mock(VectorProjectionService.class);
when(projectionService.list(50, 0)).thenReturn(List.of(projection(ProjectionStatus.COMPLETED)));
VectorVisualizationMgmtController controller = new VectorVisualizationMgmtController(
projectionService,
mock(VectorSearchVisualizationService.class));

var response = controller.listProjections(50, 0);

assertThat(response.getBody().getData().items()).singleElement()
.satisfies(item -> assertThat(item.targetTypes()).containsExactly("COURSE_CHUNK"));
}

@Test
void itemDetailDoesNotReturnEmbeddingMetadata() {
VectorProjectionService projectionService = mock(VectorProjectionService.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public void run(String projectionId) {
}
pointRepository.deleteByProjectionId(projectionId);
pointRepository.saveAll(points);
projectionRepository.markCompleted(projectionId, points.size(), Instant.now());
projectionRepository.markCompleted(projectionId, points.size(), actualTargetTypes(items), Instant.now());
} catch (Exception ex) {
log.warn("Vector projection job failed. projectionId={}", projectionId, ex);
projectionRepository.updateStatus(
Expand All @@ -72,4 +72,12 @@ private VectorProjectionGenerator generatorFor(VectorProjection projection) {
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("UNSUPPORTED_PROJECTION_ALGORITHM"));
}

private List<String> actualTargetTypes(List<VectorItem> items) {
return items.stream()
.map(VectorItem::targetType)
.filter(value -> value != null && !value.isBlank())
.distinct()
.toList();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,19 @@ public void updateStatus(String projectionId, ProjectionStatus status, String er
}

@Override
public void markCompleted(String projectionId, int itemCount, Instant completedAt) {
public void markCompleted(String projectionId, int itemCount, List<String> targetTypes, Instant completedAt) {
jdbcTemplate.update("""
UPDATE tb_ai_vector_projection
SET status = 'COMPLETED',
target_types = :targetTypes,
item_count = :itemCount,
error_message = NULL,
completed_at = :completedAt
WHERE projection_id = :projectionId
""", new MapSqlParameterSource()
.addValue("projectionId", projectionId)
.addValue("itemCount", itemCount)
.addValue("targetTypes", String.join(",", targetTypes == null ? List.of() : targetTypes))
.addValue("completedAt", timestamp(completedAt)));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ static boolean isPostgres(NamedParameterJdbcTemplate template) {
static String jsonText(String alias, String keyExpression, boolean postgres) {
String column = alias == null || alias.isBlank() ? "metadata" : alias + ".metadata";
if (postgres) {
return column + " ->> " + keyExpression;
if (keyExpression.startsWith(":")) {
return column + " ->> " + keyExpression;
}
return column + " ->> '" + keyExpression.replace("'", "''") + "'";
}
String path = keyExpression.startsWith(":")
? "CONCAT('$.', " + keyExpression + ")"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public List<VectorProjectionPoint> generate(String projectionId, List<VectorItem
VectorProjection saved = projections.findById(projection.projectionId()).orElseThrow();
assertThat(saved.status()).isEqualTo(ProjectionStatus.COMPLETED);
assertThat(saved.itemCount()).isEqualTo(1);
assertThat(saved.targetTypes()).containsExactly("COURSE_CHUNK");
assertThat(points.points).hasSize(1);
}

Expand Down Expand Up @@ -183,14 +184,14 @@ public void updateStatus(String projectionId, ProjectionStatus status, String er
}

@Override
public void markCompleted(String projectionId, int itemCount, Instant completedAt) {
public void markCompleted(String projectionId, int itemCount, List<String> targetTypes, Instant completedAt) {
VectorProjection current = projections.get(projectionId);
projections.put(projectionId, new VectorProjection(
current.projectionId(),
current.name(),
current.algorithm(),
ProjectionStatus.COMPLETED,
current.targetTypes(),
targetTypes,
current.filters(),
itemCount,
null,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package studio.one.platform.ai.service.visualization;

import static org.assertj.core.api.Assertions.assertThat;

import org.junit.jupiter.api.Test;

class JdbcVectorProjectionSqlTest {

@Test
void postgresJsonTextQuotesLiteralKey() {
assertThat(JdbcVectorProjectionSql.jsonText("c", "chunkId", true))
.isEqualTo("c.metadata ->> 'chunkId'");
}

@Test
void postgresJsonTextKeepsNamedParameterKey() {
assertThat(JdbcVectorProjectionSql.jsonText(null, ":filterKey0", true))
.isEqualTo("metadata ->> :filterKey0");
}

@Test
void mysqlJsonTextUsesJsonExtractPath() {
assertThat(JdbcVectorProjectionSql.jsonText("c", "chunkId", false))
.isEqualTo("JSON_UNQUOTE(JSON_EXTRACT(c.metadata, '$.chunkId'))");
}

@Test
void mysqlJsonTextUsesParameterizedPath() {
assertThat(JdbcVectorProjectionSql.jsonText(null, ":filterKey0", false))
.isEqualTo("JSON_UNQUOTE(JSON_EXTRACT(metadata, CONCAT('$.', :filterKey0)))");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ public interface VectorProjectionRepository {

void updateStatus(String projectionId, ProjectionStatus status, String errorMessage, Instant completedAt);

void markCompleted(String projectionId, int itemCount, Instant completedAt);
void markCompleted(String projectionId, int itemCount, List<String> targetTypes, Instant completedAt);
}
Loading