index.go
package search
import (
"encoding/json"
"log"
"strings"
"github.com/readysite/readysite/website/internal/helpers"
"github.com/readysite/readysite/website/models"
)
const settingIndexBuilt = "search_index_built"
// isIndexBuilt checks whether the initial index has been populated.
func isIndexBuilt() bool {
return helpers.GetSetting(settingIndexBuilt) == "true"
}
// markIndexBuilt records that the initial index is complete.
func markIndexBuilt() {
helpers.SetSetting(settingIndexBuilt, "true")
}
// IndexAll rebuilds the entire search index from scratch.
func IndexAll() {
// Clear existing index
if _, err := models.DB.Exec("DELETE FROM SearchIndex"); err != nil {
log.Printf("[search] Failed to clear index: %v", err)
return
}
indexPages()
indexCollections()
indexDocuments()
indexFiles()
indexPartials()
indexNotes()
indexUsers()
}
// IndexEntity adds or updates a single entity in the search index.
func IndexEntity(entityType, entityID string) {
RemoveEntity(entityType, entityID)
switch entityType {
case "page":
page, err := models.Pages.Get(entityID)
if err != nil {
return
}
insertEntry(page.Title(), StripHTML(page.HTML()), "page", page.ID, "page", "")
case "collection":
col, err := models.Collections.Get(entityID)
if err != nil {
return
}
body := col.Description
if fields, err := schemaFieldNames(col.Schema); err == nil && len(fields) > 0 {
body += " " + fields
}
insertEntry(col.Name, body, "collection", col.ID, "collection", "")
case "document":
doc, err := models.Documents.Get(entityID)
if err != nil {
return
}
title, body := extractDocumentContent(doc)
tags := "document " + doc.CollectionID
insertEntry(title, body, tags, doc.ID, "document", doc.CollectionID)
case "partial":
p, err := models.Partials.Get(entityID)
if err != nil {
return
}
insertEntry(p.Name, StripHTML(p.HTML), "partial", p.ID, "partial", "")
case "note":
n, err := models.Notes.Get(entityID)
if err != nil {
return
}
insertEntry(n.Title, n.Content, "note "+n.Type, n.ID, "note", "")
case "file":
f, err := models.Files.Get(entityID)
if err != nil {
return
}
body := cachedFileContent(f.ID)
insertEntry(f.Name, body, "file "+f.MimeType, f.ID, "file", "")
case "user":
u, err := models.Users.Get(entityID)
if err != nil {
return
}
insertEntry(u.Name, u.Email, "user "+u.Role, u.ID, "user", "")
}
}
// RemoveEntity removes an entity from the search index.
func RemoveEntity(entityType, entityID string) {
_, err := models.DB.Exec(
"DELETE FROM SearchIndex WHERE entity_id = ? AND entity_type = ?",
entityID, entityType,
)
if err != nil {
log.Printf("[search] Failed to remove %s/%s: %v", entityType, entityID, err)
}
}
// --- Internal indexing functions ---
func insertEntry(title, body, tags, entityID, entityType, collectionID string) {
_, err := models.DB.Exec(
"INSERT INTO SearchIndex (title, body, tags, entity_id, entity_type, collection_id) VALUES (?, ?, ?, ?, ?, ?)",
title, body, tags, entityID, entityType, collectionID,
)
if err != nil {
log.Printf("[search] Failed to index %s/%s: %v", entityType, entityID, err)
}
}
func indexPages() {
pages, err := models.Pages.All()
if err != nil {
log.Printf("[search] Failed to load pages: %v", err)
return
}
for _, page := range pages {
insertEntry(page.Title(), StripHTML(page.HTML()), "page", page.ID, "page", "")
}
log.Printf("[search] Indexed %d pages", len(pages))
}
func indexCollections() {
cols, err := models.Collections.All()
if err != nil {
log.Printf("[search] Failed to load collections: %v", err)
return
}
for _, col := range cols {
body := col.Description
if fields, err := schemaFieldNames(col.Schema); err == nil && len(fields) > 0 {
body += " " + fields
}
insertEntry(col.Name, body, "collection", col.ID, "collection", "")
}
log.Printf("[search] Indexed %d collections", len(cols))
}
func indexDocuments() {
docs, err := models.Documents.All()
if err != nil {
log.Printf("[search] Failed to load documents: %v", err)
return
}
for _, doc := range docs {
title, body := extractDocumentContent(doc)
tags := "document " + doc.CollectionID
insertEntry(title, body, tags, doc.ID, "document", doc.CollectionID)
}
log.Printf("[search] Indexed %d documents", len(docs))
}
func indexFiles() {
files, err := models.Files.All()
if err != nil {
log.Printf("[search] Failed to load files: %v", err)
return
}
for _, f := range files {
body := cachedFileContent(f.ID)
insertEntry(f.Name, body, "file "+f.MimeType, f.ID, "file", "")
}
log.Printf("[search] Indexed %d files", len(files))
}
func indexPartials() {
partials, err := models.Partials.All()
if err != nil {
log.Printf("[search] Failed to load partials: %v", err)
return
}
for _, p := range partials {
insertEntry(p.Name, StripHTML(p.HTML), "partial", p.ID, "partial", "")
}
log.Printf("[search] Indexed %d partials", len(partials))
}
func indexNotes() {
notes, err := models.Notes.All()
if err != nil {
log.Printf("[search] Failed to load notes: %v", err)
return
}
for _, n := range notes {
insertEntry(n.Title, n.Content, "note "+n.Type, n.ID, "note", "")
}
log.Printf("[search] Indexed %d notes", len(notes))
}
func indexUsers() {
users, err := models.Users.All()
if err != nil {
log.Printf("[search] Failed to load users: %v", err)
return
}
for _, u := range users {
insertEntry(u.Name, u.Email, "user "+u.Role, u.ID, "user", "")
}
log.Printf("[search] Indexed %d users", len(users))
}
// --- Helpers ---
// extractDocumentContent pulls a title and body from a document's JSON data.
func extractDocumentContent(doc *models.Document) (title, body string) {
if doc.Data == "" {
return "", ""
}
var data map[string]any
if err := json.Unmarshal([]byte(doc.Data), &data); err != nil {
return "", ""
}
// Try common title fields
for _, key := range []string{"title", "name", "Title", "Name"} {
if v, ok := data[key]; ok {
if s, ok := v.(string); ok && s != "" {
title = s
break
}
}
}
// If no title found, use the first non-empty string field
if title == "" {
for _, v := range data {
if s, ok := v.(string); ok && s != "" {
title = s
break
}
}
}
body = FlattenJSON(doc.Data)
return title, body
}
// schemaFieldNames extracts field names from a collection's JSON schema.
func schemaFieldNames(schemaJSON string) (string, error) {
if schemaJSON == "" {
return "", nil
}
var fields []struct {
Name string `json:"name"`
}
if err := json.Unmarshal([]byte(schemaJSON), &fields); err != nil {
return "", err
}
names := make([]string, len(fields))
for i, f := range fields {
names[i] = f.Name
}
return joinStrings(names), nil
}
func joinStrings(ss []string) string {
var buf strings.Builder
for i, s := range ss {
if i > 0 {
buf.WriteByte(' ')
}
buf.WriteString(s)
}
return buf.String()
}
// cachedFileContent returns cached text content for a file, if available.
func cachedFileContent(fileID string) string {
cache, err := models.FileContentCaches.First("WHERE FileID = ?", fileID)
if err != nil || cache == nil {
return ""
}
return cache.TextContent
}