37.7 KB
collections.go
package controllers
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/readysite/readysite/pkg/application"
"github.com/readysite/readysite/website/internal/access"
"github.com/readysite/readysite/website/internal/content"
"github.com/readysite/readysite/website/internal/helpers"
"github.com/readysite/readysite/website/models"
)
// Collections returns the collections controller.
func Collections() (string, *CollectionsController) {
return "collections", &CollectionsController{}
}
// CollectionsController handles collection and document CRUD operations.
type CollectionsController struct {
application.BaseController
}
// Setup registers routes.
func (c *CollectionsController) Setup(app *application.App) {
c.BaseController.Setup(app)
// Export, backup, and import routes (used by React admin for file downloads)
http.Handle("GET /admin/collections/{id}/backup", app.Method(c, "Backup", RequireAuth))
http.Handle("GET /admin/collections/{id}/export", app.Method(c, "Export", RequireAuth))
http.Handle("POST /admin/collections/{id}/import", app.Method(c, "Import", RequireAuth))
}
// Handle implements Controller interface with value receiver for request isolation.
func (c CollectionsController) Handle(r *http.Request) application.Controller {
c.Request = r
return &c
}
// Collection returns the current collection from path parameter.
func (c *CollectionsController) Collection() *models.Collection {
if c.Request == nil {
return nil
}
id := c.PathValue("id")
if id == "" {
return nil
}
collection, err := models.Collections.Get(id)
if err != nil {
return nil
}
return collection
}
// Collections returns all collections.
func (c *CollectionsController) Collections() []*models.Collection {
collections, _ := models.Collections.Search("ORDER BY Name")
return collections
}
// Document returns the current document from path parameter.
func (c *CollectionsController) Document() *models.Document {
if c.Request == nil {
return nil
}
docID := c.PathValue("doc")
if docID == "" {
return nil
}
doc, err := models.Documents.Get(docID)
if err != nil {
return nil
}
return doc
}
// DocumentPagination returns pagination state for documents list.
func (c *CollectionsController) DocumentPagination() *helpers.Pagination {
col := c.Collection()
if col == nil {
return helpers.NewPagination(c.Request, 0)
}
total := col.DocumentCount()
return helpers.NewPagination(c.Request, total)
}
// Documents returns paginated documents for the current collection.
func (c *CollectionsController) Documents() []*models.Document {
col := c.Collection()
if col == nil {
return nil
}
p := c.DocumentPagination()
docs, _ := models.Documents.Search("WHERE CollectionID = ? ORDER BY CreatedAt DESC LIMIT ? OFFSET ?",
col.ID, p.PageSize, p.Offset)
return docs
}
// SchemaFields returns the parsed schema fields for the current collection.
func (c *CollectionsController) SchemaFields() []content.Field {
col := c.Collection()
if col == nil {
return nil
}
fields, _ := content.GetFields(col)
return fields
}
// IsNewCollection returns true if creating a new collection.
func (c *CollectionsController) IsNewCollection() bool {
if c.Request == nil {
return true
}
id := c.PathValue("id")
return id == ""
}
// IsNewDocument returns true if creating a new document.
func (c *CollectionsController) IsNewDocument() bool {
if c.Request == nil {
return true
}
docID := c.PathValue("doc")
return docID == ""
}
// FieldTypes returns available field types for schema builder.
func (c *CollectionsController) FieldTypes() []string {
return []string{
content.Text,
content.Number,
content.Bool,
content.Date,
content.Email,
content.URL,
content.Select,
content.Relation,
content.File,
content.JSON,
}
}
// Create creates a new collection.
func (c *CollectionsController) Create(w http.ResponseWriter, r *http.Request) {
slug := r.FormValue("slug")
name := r.FormValue("name")
description := r.FormValue("description")
schema := r.FormValue("schema")
// Validate slug if provided
if err := content.ValidateSlug(slug); err != nil {
c.RenderError(w, r, err)
return
}
if err := content.CheckSlugAvailable(slug, "collection"); err != nil {
c.RenderError(w, r, err)
return
}
// Validate collection input fields
if err := content.ValidateCollectionInput(name, description, schema); err != nil {
c.RenderError(w, r, err)
return
}
collection := &models.Collection{
Name: name,
Description: description,
Schema: schema,
}
// Use slug as ID if provided
if slug != "" {
collection.ID = slug
}
id, err := models.Collections.Insert(collection)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to create collection: %w", err))
return
}
// Audit log
userID := ""
if user := access.GetUserFromJWT(r); user != nil {
userID = user.ID
}
helpers.AuditCreate(r, userID, helpers.ResourceCollection, id, collection.Name)
w.Header().Set("HX-Trigger", `{"showToast":"Collection created"}`)
c.Refresh(w, r)
}
// Update updates an existing collection.
func (c *CollectionsController) Update(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
name := r.FormValue("name")
description := r.FormValue("description")
schema := r.FormValue("schema")
// Validate collection input fields
if err := content.ValidateCollectionInput(name, description, schema); err != nil {
c.RenderError(w, r, err)
return
}
collection, err := models.Collections.Get(id)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Check for concurrent edit conflict using UpdatedAt
expectedVersion := r.FormValue("version")
if expectedVersion != "" && collection.UpdatedAt.Format("2006-01-02T15:04:05") != expectedVersion {
c.RenderError(w, r, fmt.Errorf("This collection was modified by another user. Please reload and try again."))
return
}
collection.Name = name
collection.Description = description
collection.Schema = schema
if err := models.Collections.Update(collection); err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to update collection: %w", err))
return
}
// Audit log
userID := ""
if user := access.GetUserFromJWT(r); user != nil {
userID = user.ID
}
helpers.AuditUpdate(r, userID, helpers.ResourceCollection, id, collection.Name, nil)
w.Header().Set("HX-Trigger", `{"showToast":"Collection saved"}`)
c.Refresh(w, r)
}
// Delete deletes a collection and all its documents.
func (c *CollectionsController) Delete(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
collection, err := models.Collections.Get(id)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Delete all documents first
docs, _ := collection.Documents()
for _, doc := range docs {
models.Documents.Delete(doc)
}
if err := models.Collections.Delete(collection); err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to delete collection: %w", err))
return
}
// Audit log
userID := ""
if user := access.GetUserFromJWT(r); user != nil {
userID = user.ID
}
helpers.AuditDelete(r, userID, helpers.ResourceCollection, id, collection.Name)
c.Redirect(w, r, "/admin/collections")
}
// CreateDocument creates a new document.
func (c *CollectionsController) CreateDocument(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
collection, err := models.Collections.Get(collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Check authorization - user must have write permission on collection
user := access.GetUserFromJWT(r)
if !access.CheckAccess(user, access.ResourceCollection, collectionID, access.PermWrite) {
c.RenderError(w, r, fmt.Errorf("You don't have permission to add documents to this collection"))
return
}
// Parse form data into JSON
data, err := content.ParseFormData(collection, r)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to parse form data: %w", err))
return
}
// Validate document against schema
if err := content.ValidateDocument(collection, data); err != nil {
c.RenderError(w, r, fmt.Errorf("Validation failed: %w", err))
return
}
jsonData, err := json.Marshal(data)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to encode document: %w", err))
return
}
doc := &models.Document{
CollectionID: collectionID,
Data: string(jsonData),
}
docID, err := models.Documents.Insert(doc)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to create document: %w", err))
return
}
// Audit log
userID := ""
if user := access.GetUserFromJWT(r); user != nil {
userID = user.ID
}
helpers.AuditCreate(r, userID, helpers.ResourceDocument, docID, collection.Name+" document")
// Check for redirect URL template (for public pages like blog-new)
// Replace {{id}} with the new document ID
if redirectURL := r.Header.Get("X-Redirect-URL"); redirectURL != "" {
redirectURL = strings.ReplaceAll(redirectURL, "{{id}}", docID)
w.Header().Set("HX-Redirect", redirectURL)
return
}
w.Header().Set("HX-Trigger", `{"showToast":"Document created"}`)
c.Refresh(w, r)
}
// UpdateDocument updates an existing document.
func (c *CollectionsController) UpdateDocument(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
docID := r.PathValue("doc")
collection, err := models.Collections.Get(collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Check authorization - user must have write permission on collection
user := access.GetUserFromJWT(r)
if !access.CheckAccess(user, access.ResourceCollection, collectionID, access.PermWrite) {
c.RenderError(w, r, fmt.Errorf("You don't have permission to edit documents in this collection"))
return
}
doc, err := models.Documents.Get(docID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Document not found"))
return
}
// Check for concurrent edit conflict using UpdatedAt
expectedVersion := r.FormValue("version")
if expectedVersion != "" && doc.UpdatedAt.Format("2006-01-02T15:04:05") != expectedVersion {
c.RenderError(w, r, fmt.Errorf("This document was modified by another user. Please reload and try again."))
return
}
// Parse form data into JSON
data, err := content.ParseFormData(collection, r)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to parse form data: %w", err))
return
}
// Validate document against schema
if err := content.ValidateDocument(collection, data); err != nil {
c.RenderError(w, r, fmt.Errorf("Validation failed: %w", err))
return
}
jsonData, err := json.Marshal(data)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to encode document: %w", err))
return
}
doc.Data = string(jsonData)
if err := models.Documents.Update(doc); err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to update document: %w", err))
return
}
// Audit log
userID := ""
if user := access.GetUserFromJWT(r); user != nil {
userID = user.ID
}
helpers.AuditUpdate(r, userID, helpers.ResourceDocument, docID, collection.Name+" document", nil)
// Check for redirect URL (for public pages like blog-edit)
if redirectURL := r.Header.Get("X-Redirect-URL"); redirectURL != "" {
redirectURL = strings.ReplaceAll(redirectURL, "{{id}}", docID)
w.Header().Set("HX-Redirect", redirectURL)
return
}
w.Header().Set("HX-Trigger", `{"showToast":"Document saved"}`)
c.Refresh(w, r)
}
// DeleteDocument deletes a document.
func (c *CollectionsController) DeleteDocument(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
docID := r.PathValue("doc")
// Check authorization - user must have delete permission on collection
user := access.GetUserFromJWT(r)
if !access.CheckAccess(user, access.ResourceCollection, collectionID, access.PermDelete) {
c.RenderError(w, r, fmt.Errorf("You don't have permission to delete documents from this collection"))
return
}
doc, err := models.Documents.Get(docID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Document not found"))
return
}
if err := models.Documents.Delete(doc); err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to delete document: %w", err))
return
}
// Audit log
userID := ""
if user := access.GetUserFromJWT(r); user != nil {
userID = user.ID
}
collection, _ := models.Collections.Get(collectionID)
collectionName := collectionID
if collection != nil {
collectionName = collection.Name
}
helpers.AuditDelete(r, userID, helpers.ResourceDocument, docID, collectionName+" document")
// Check for redirect URL (for public pages like blog-edit)
if redirectURL := r.Header.Get("X-Redirect-URL"); redirectURL != "" {
w.Header().Set("HX-Redirect", redirectURL)
return
}
c.Redirect(w, r, "/admin/collections/"+collectionID+"/documents")
}
// --- Backup and Export ---
// CollectionBackup represents a backup of a collection.
type CollectionBackup struct {
Version string `json:"version"`
ExportedAt string `json:"exportedAt"`
Collection CollectionMeta `json:"collection"`
Documents []DocumentData `json:"documents"`
}
// CollectionMeta holds collection metadata for backup.
type CollectionMeta struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Type string `json:"type"`
Schema string `json:"schema"`
Query string `json:"query,omitempty"`
ListRule string `json:"listRule,omitempty"`
ViewRule string `json:"viewRule,omitempty"`
CreateRule string `json:"createRule,omitempty"`
UpdateRule string `json:"updateRule,omitempty"`
DeleteRule string `json:"deleteRule,omitempty"`
}
// DocumentData holds document data for backup/export.
type DocumentData struct {
ID string `json:"id"`
Data map[string]any `json:"data"`
CreatedAt string `json:"created"`
UpdatedAt string `json:"updated"`
}
// Backup downloads a collection backup as JSON.
// GET /admin/collections/{id}/backup
func (c *CollectionsController) Backup(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
// Check authorization
user := access.GetUserFromJWT(r)
if !access.IsAdmin(user) {
c.RenderError(w, r, fmt.Errorf("Admin access required for backup"))
return
}
collection, err := models.Collections.Get(collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Get all documents
docs, err := models.Documents.Search("WHERE CollectionID = ?", collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to fetch documents: %w", err))
return
}
// Build backup structure
backup := CollectionBackup{
Version: "1.0",
ExportedAt: time.Now().Format("2006-01-02T15:04:05Z"),
Collection: CollectionMeta{
ID: collection.ID,
Name: collection.Name,
Description: collection.Description,
Type: collection.Type,
Schema: collection.Schema,
Query: collection.Query,
ListRule: collection.ListRule,
ViewRule: collection.ViewRule,
CreateRule: collection.CreateRule,
UpdateRule: collection.UpdateRule,
DeleteRule: collection.DeleteRule,
},
Documents: make([]DocumentData, 0, len(docs)),
}
for _, doc := range docs {
var data map[string]any
if doc.Data != "" {
json.Unmarshal([]byte(doc.Data), &data)
}
backup.Documents = append(backup.Documents, DocumentData{
ID: doc.ID,
Data: data,
CreatedAt: doc.CreatedAt.Format("2006-01-02T15:04:05Z"),
UpdatedAt: doc.UpdatedAt.Format("2006-01-02T15:04:05Z"),
})
}
// Set response headers for download
filename := fmt.Sprintf("%s-backup-%s.json", collection.ID, time.Now().Format("2006-01-02"))
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
// Audit log
helpers.AuditRead(r, user.ID, helpers.ResourceCollection, collectionID, collection.Name+" backup")
// Encode and send
encoder := json.NewEncoder(w)
encoder.SetIndent("", " ")
encoder.Encode(backup)
}
// Restore restores a collection from a backup.
// POST /admin/collections/{id}/restore
// Query param: mode=replace|merge|append (default: merge)
func (c *CollectionsController) Restore(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
mode := r.URL.Query().Get("mode")
if mode == "" {
mode = "merge"
}
// Check authorization
user := access.GetUserFromJWT(r)
if !access.IsAdmin(user) {
c.RenderError(w, r, fmt.Errorf("Admin access required for restore"))
return
}
collection, err := models.Collections.Get(collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Parse uploaded file
r.ParseMultipartForm(50 << 20) // 50MB max
file, _, err := r.FormFile("file")
if err != nil {
c.RenderError(w, r, fmt.Errorf("No backup file provided"))
return
}
defer file.Close()
// Decode backup
var backup CollectionBackup
if err := json.NewDecoder(file).Decode(&backup); err != nil {
c.RenderError(w, r, fmt.Errorf("Invalid backup file: %w", err))
return
}
// Apply based on mode
var inserted, updated, skipped int
switch mode {
case "replace":
// Delete all existing documents first
existing, _ := models.Documents.Search("WHERE CollectionID = ?", collectionID)
for _, doc := range existing {
models.Documents.Delete(doc)
}
// Insert all from backup
for _, docData := range backup.Documents {
doc := &models.Document{
CollectionID: collectionID,
}
dataBytes, _ := json.Marshal(docData.Data)
doc.Data = string(dataBytes)
models.Documents.Insert(doc)
inserted++
}
case "merge":
// Update existing by ID, insert new
for _, docData := range backup.Documents {
existing, _ := models.Documents.Get(docData.ID)
if existing != nil && existing.CollectionID == collectionID {
// Update existing
dataBytes, _ := json.Marshal(docData.Data)
existing.Data = string(dataBytes)
models.Documents.Update(existing)
updated++
} else {
// Insert new
doc := &models.Document{
CollectionID: collectionID,
}
dataBytes, _ := json.Marshal(docData.Data)
doc.Data = string(dataBytes)
models.Documents.Insert(doc)
inserted++
}
}
case "append":
// Insert all with new IDs
for _, docData := range backup.Documents {
doc := &models.Document{
CollectionID: collectionID,
}
dataBytes, _ := json.Marshal(docData.Data)
doc.Data = string(dataBytes)
models.Documents.Insert(doc)
inserted++
}
default:
c.RenderError(w, r, fmt.Errorf("Invalid restore mode: %s", mode))
return
}
// Audit log
helpers.AuditUpdate(r, user.ID, helpers.ResourceCollection, collectionID,
fmt.Sprintf("%s restore (%s mode): %d inserted, %d updated, %d skipped",
collection.Name, mode, inserted, updated, skipped), nil)
w.Header().Set("HX-Trigger", fmt.Sprintf(`{"showToast":"Restore complete: %d inserted, %d updated"}`, inserted, updated))
c.Refresh(w, r)
}
// Export exports collection documents as JSON or CSV.
// GET /admin/collections/{id}/export?format=json|csv
func (c *CollectionsController) Export(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
format := r.URL.Query().Get("format")
if format == "" {
format = "json"
}
// Check authorization
user := access.GetUserFromJWT(r)
if !access.CheckAccess(user, access.ResourceCollection, collectionID, access.PermRead) {
c.RenderError(w, r, fmt.Errorf("Access denied"))
return
}
collection, err := models.Collections.Get(collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Get documents
docs, err := models.Documents.Search("WHERE CollectionID = ?", collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Failed to fetch documents: %w", err))
return
}
filename := fmt.Sprintf("%s-export-%s.%s", collection.ID, time.Now().Format("2006-01-02"), format)
// Audit log
userID := ""
if user != nil {
userID = user.ID
}
helpers.AuditRead(r, userID, helpers.ResourceCollection, collectionID, collection.Name+" export ("+format+")")
switch format {
case "json":
c.exportJSON(w, collection, docs, filename)
case "csv":
c.exportCSV(w, collection, docs, filename)
default:
c.RenderError(w, r, fmt.Errorf("Unsupported format: %s", format))
}
}
func (c *CollectionsController) exportJSON(w http.ResponseWriter, collection *models.Collection, docs []*models.Document, filename string) {
items := make([]map[string]any, 0, len(docs))
for _, doc := range docs {
item := map[string]any{
"id": doc.ID,
"created": doc.CreatedAt.Format("2006-01-02T15:04:05Z"),
"updated": doc.UpdatedAt.Format("2006-01-02T15:04:05Z"),
}
var data map[string]any
if doc.Data != "" {
json.Unmarshal([]byte(doc.Data), &data)
}
for k, v := range data {
item[k] = v
}
items = append(items, item)
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
encoder := json.NewEncoder(w)
encoder.SetIndent("", " ")
encoder.Encode(items)
}
func (c *CollectionsController) exportCSV(w http.ResponseWriter, collection *models.Collection, docs []*models.Document, filename string) {
// Get field names from schema
fields, _ := content.GetFields(collection)
// Build header
headers := []string{"id", "created", "updated"}
for _, f := range fields {
headers = append(headers, f.Name)
}
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
// Write BOM for Excel
w.Write([]byte{0xEF, 0xBB, 0xBF})
// Write header
w.Write([]byte(strings.Join(headers, ",") + "\n"))
// Write rows
for _, doc := range docs {
var data map[string]any
if doc.Data != "" {
json.Unmarshal([]byte(doc.Data), &data)
}
row := []string{
csvEscape(doc.ID),
csvEscape(doc.CreatedAt.Format("2006-01-02T15:04:05Z")),
csvEscape(doc.UpdatedAt.Format("2006-01-02T15:04:05Z")),
}
for _, f := range fields {
val := data[f.Name]
if val == nil {
row = append(row, "")
} else {
switch v := val.(type) {
case string:
row = append(row, csvEscape(v))
case []any, map[string]any:
// JSON encode complex values
bytes, _ := json.Marshal(v)
row = append(row, csvEscape(string(bytes)))
default:
row = append(row, csvEscape(fmt.Sprintf("%v", v)))
}
}
}
w.Write([]byte(strings.Join(row, ",") + "\n"))
}
}
func csvEscape(s string) string {
needsQuote := strings.ContainsAny(s, ",\"\n\r")
if needsQuote {
s = strings.ReplaceAll(s, "\"", "\"\"")
return "\"" + s + "\""
}
return s
}
// Import imports documents from JSON or CSV file.
// POST /admin/collections/{id}/import
// Form fields:
// - file: the file to import
// - format: json|csv (auto-detected from filename if not specified)
// - mode: insert|upsert|replace (default: insert)
// - skipInvalid: skip invalid rows instead of failing (default: false)
// - stopOnError: stop at first error instead of collecting all (default: true)
func (c *CollectionsController) Import(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
mode := r.FormValue("mode")
if mode == "" {
mode = "insert"
}
skipInvalid := r.FormValue("skipInvalid") == "true"
stopOnError := r.FormValue("stopOnError") != "false" // Default true
// Check authorization
user := access.GetUserFromJWT(r)
if !access.CheckAccess(user, access.ResourceCollection, collectionID, access.PermWrite) {
c.RenderError(w, r, fmt.Errorf("Access denied"))
return
}
collection, err := models.Collections.Get(collectionID)
if err != nil {
c.RenderError(w, r, fmt.Errorf("Collection not found"))
return
}
// Get uploaded file
r.ParseMultipartForm(MaxImportFileSize)
file, header, err := r.FormFile("file")
if err != nil {
c.RenderError(w, r, fmt.Errorf("No import file provided"))
return
}
defer file.Close()
// Check file size
if header.Size > MaxImportFileSize {
c.RenderError(w, r, fmt.Errorf("File too large (max %dMB)", MaxImportFileSize>>20))
return
}
// Detect format
format := r.FormValue("format")
if format == "" {
if strings.HasSuffix(strings.ToLower(header.Filename), ".csv") {
format = "csv"
} else {
format = "json"
}
}
// Import based on format
var inserted, updated, skipped int
var errors []string
switch format {
case "json":
inserted, updated, skipped, errors = c.importJSON(collection, file, mode, skipInvalid, stopOnError)
case "csv":
inserted, updated, skipped, errors = c.importCSV(collection, file, mode, skipInvalid, stopOnError)
default:
c.RenderError(w, r, fmt.Errorf("Unsupported format: %s", format))
return
}
// Audit log
helpers.AuditUpdate(r, user.ID, helpers.ResourceCollection, collectionID,
fmt.Sprintf("%s import (%s): %d inserted, %d updated, %d skipped",
collection.Name, format, inserted, updated, skipped), nil)
if len(errors) > 0 && !skipInvalid {
c.RenderError(w, r, fmt.Errorf("Import failed: %s", errors[0]))
return
}
msg := fmt.Sprintf("Import complete: %d inserted, %d updated, %d skipped", inserted, updated, skipped)
if len(errors) > 0 {
msg += fmt.Sprintf(" (%d errors)", len(errors))
}
w.Header().Set("HX-Trigger", fmt.Sprintf(`{"showToast":%q}`, msg))
c.Refresh(w, r)
}
func (c *CollectionsController) importJSON(collection *models.Collection, file io.Reader, mode string, skipInvalid, stopOnError bool) (inserted, updated, skipped int, errors []string) {
// Decode JSON array
var items []map[string]any
if err := json.NewDecoder(file).Decode(&items); err != nil {
return 0, 0, 0, []string{fmt.Sprintf("Invalid JSON: %v", err)}
}
// Check row limit
if len(items) > MaxImportRows {
return 0, 0, 0, []string{fmt.Sprintf("Too many rows: %d (max %d)", len(items), MaxImportRows)}
}
// If replace mode, delete existing first
if mode == "replace" {
existing, _ := models.Documents.Search("WHERE CollectionID = ?", collection.ID)
for _, doc := range existing {
models.Documents.Delete(doc)
}
}
for i, item := range items {
// Extract system fields if present
id, _ := item["id"].(string)
delete(item, "id")
delete(item, "created")
delete(item, "updated")
delete(item, "collectionId")
delete(item, "collectionName")
// Validate against schema
if err := content.ValidateDocument(collection, item); err != nil {
if skipInvalid {
skipped++
continue
}
errors = append(errors, fmt.Sprintf("Row %d: %v", i+1, err))
if stopOnError {
return // Stop at first error
}
continue // Collect all errors
}
dataBytes, _ := json.Marshal(item)
if mode == "upsert" && id != "" {
// Try to find existing
existing, _ := models.Documents.Get(id)
if existing != nil && existing.CollectionID == collection.ID {
existing.Data = string(dataBytes)
models.Documents.Update(existing)
updated++
continue
}
}
// Insert new
doc := &models.Document{
CollectionID: collection.ID,
Data: string(dataBytes),
}
models.Documents.Insert(doc)
inserted++
}
return
}
func (c *CollectionsController) importCSV(collection *models.Collection, file interface{ Read([]byte) (int, error) }, mode string, skipInvalid, stopOnError bool) (inserted, updated, skipped int, errors []string) {
// Read all data from file
csvData := make([]byte, 0)
buf := make([]byte, 4096)
for {
n, err := file.Read(buf)
if n > 0 {
csvData = append(csvData, buf[:n]...)
}
if err != nil {
break
}
}
// Remove BOM if present
if len(csvData) >= 3 && csvData[0] == 0xEF && csvData[1] == 0xBB && csvData[2] == 0xBF {
csvData = csvData[3:]
}
// Split into lines
lines := strings.Split(string(csvData), "\n")
if len(lines) < 2 {
return 0, 0, 0, []string{"CSV must have header row and at least one data row"}
}
// Parse header
headers := parseCSVLine(lines[0])
if len(headers) == 0 {
return 0, 0, 0, []string{"Invalid CSV header"}
}
// Count data rows (excluding empty lines)
dataRowCount := 0
for i := 1; i < len(lines); i++ {
if strings.TrimSpace(lines[i]) != "" {
dataRowCount++
}
}
// Check row limit
if dataRowCount > MaxImportRows {
return 0, 0, 0, []string{fmt.Sprintf("Too many rows: %d (max %d)", dataRowCount, MaxImportRows)}
}
// If replace mode, delete existing first
if mode == "replace" {
existing, _ := models.Documents.Search("WHERE CollectionID = ?", collection.ID)
for _, doc := range existing {
models.Documents.Delete(doc)
}
}
// Process data rows
rowNum := 0
for i := 1; i < len(lines); i++ {
line := strings.TrimSpace(lines[i])
if line == "" {
continue
}
rowNum++
values := parseCSVLine(line)
if len(values) != len(headers) {
if skipInvalid {
skipped++
continue
}
errors = append(errors, fmt.Sprintf("Row %d: column count mismatch", rowNum))
if stopOnError {
return
}
continue
}
// Build data map
data := make(map[string]any)
var id string
for j, header := range headers {
header = strings.TrimSpace(header)
value := values[j]
switch header {
case "id":
id = value
case "created", "updated", "collectionId", "collectionName":
// Skip system fields
default:
data[header] = parseCSVValue(value)
}
}
// Validate against schema
if err := content.ValidateDocument(collection, data); err != nil {
if skipInvalid {
skipped++
continue
}
errors = append(errors, fmt.Sprintf("Row %d: %v", rowNum, err))
if stopOnError {
return
}
continue
}
dataBytes, _ := json.Marshal(data)
if mode == "upsert" && id != "" {
existing, _ := models.Documents.Get(id)
if existing != nil && existing.CollectionID == collection.ID {
existing.Data = string(dataBytes)
models.Documents.Update(existing)
updated++
continue
}
}
doc := &models.Document{
CollectionID: collection.ID,
Data: string(dataBytes),
}
models.Documents.Insert(doc)
inserted++
}
return
}
// parseCSVLine parses a CSV line, handling quoted fields.
func parseCSVLine(line string) []string {
var result []string
var current strings.Builder
inQuotes := false
for i := 0; i < len(line); i++ {
ch := line[i]
if ch == '"' {
if inQuotes && i+1 < len(line) && line[i+1] == '"' {
// Escaped quote
current.WriteByte('"')
i++
} else {
// Toggle quote mode
inQuotes = !inQuotes
}
} else if ch == ',' && !inQuotes {
// Field separator
result = append(result, current.String())
current.Reset()
} else if ch == '\r' {
// Ignore carriage return
} else {
current.WriteByte(ch)
}
}
result = append(result, current.String())
return result
}
// Import limits
const (
MaxImportRows = 10000
MaxImportFileSize = 50 << 20 // 50MB
)
// ImportPreviewResponse is the response format for import preview.
type ImportPreviewResponse struct {
TotalRows int `json:"totalRows"`
ValidRows int `json:"validRows"`
InvalidRows int `json:"invalidRows"`
Sample []map[string]any `json:"sample"`
Errors []string `json:"errors"`
Headers []string `json:"headers,omitempty"` // For CSV only
}
// ImportPreview parses an import file and returns a preview of the data.
// POST /admin/collections/{id}/import/preview
func (c *CollectionsController) ImportPreview(w http.ResponseWriter, r *http.Request) {
collectionID := r.PathValue("id")
// Check authorization
user := access.GetUserFromJWT(r)
if !access.CheckAccess(user, access.ResourceCollection, collectionID, access.PermWrite) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusForbidden)
json.NewEncoder(w).Encode(map[string]string{"error": "Access denied"})
return
}
collection, err := models.Collections.Get(collectionID)
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusNotFound)
json.NewEncoder(w).Encode(map[string]string{"error": "Collection not found"})
return
}
// Get uploaded file
r.ParseMultipartForm(MaxImportFileSize)
file, header, err := r.FormFile("file")
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]string{"error": "No import file provided"})
return
}
defer file.Close()
// Check file size
if header.Size > MaxImportFileSize {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]string{"error": fmt.Sprintf("File too large (max %dMB)", MaxImportFileSize>>20)})
return
}
// Detect format
format := r.FormValue("format")
if format == "" {
if strings.HasSuffix(strings.ToLower(header.Filename), ".csv") {
format = "csv"
} else {
format = "json"
}
}
// Generate preview based on format
var response ImportPreviewResponse
switch format {
case "json":
response = c.previewJSON(collection, file)
case "csv":
response = c.previewCSV(collection, file)
default:
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]string{"error": "Unsupported format: " + format})
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (c *CollectionsController) previewJSON(collection *models.Collection, file io.Reader) ImportPreviewResponse {
var items []map[string]any
if err := json.NewDecoder(file).Decode(&items); err != nil {
return ImportPreviewResponse{
Errors: []string{fmt.Sprintf("Invalid JSON: %v", err)},
}
}
// Check row limit
if len(items) > MaxImportRows {
return ImportPreviewResponse{
TotalRows: len(items),
Errors: []string{fmt.Sprintf("Too many rows: %d (max %d)", len(items), MaxImportRows)},
}
}
response := ImportPreviewResponse{
TotalRows: len(items),
Sample: make([]map[string]any, 0, 10),
Errors: make([]string, 0),
}
for i, item := range items {
// Remove system fields for validation
itemCopy := make(map[string]any)
for k, v := range item {
if k != "id" && k != "created" && k != "updated" && k != "collectionId" && k != "collectionName" {
itemCopy[k] = v
}
}
// Validate against schema
if err := content.ValidateDocument(collection, itemCopy); err != nil {
response.InvalidRows++
if len(response.Errors) < 10 { // Limit errors shown
response.Errors = append(response.Errors, fmt.Sprintf("Row %d: %v", i+1, err))
}
} else {
response.ValidRows++
}
// Add to sample (first 10 rows)
if len(response.Sample) < 10 {
response.Sample = append(response.Sample, item)
}
}
return response
}
func (c *CollectionsController) previewCSV(collection *models.Collection, file interface{ Read([]byte) (int, error) }) ImportPreviewResponse {
// Read all data from file
csvData := make([]byte, 0)
buf := make([]byte, 4096)
for {
n, err := file.Read(buf)
if n > 0 {
csvData = append(csvData, buf[:n]...)
}
if err != nil {
break
}
}
// Remove BOM if present
if len(csvData) >= 3 && csvData[0] == 0xEF && csvData[1] == 0xBB && csvData[2] == 0xBF {
csvData = csvData[3:]
}
// Split into lines
lines := strings.Split(string(csvData), "\n")
if len(lines) < 2 {
return ImportPreviewResponse{
Errors: []string{"CSV must have header row and at least one data row"},
}
}
// Parse header
headers := parseCSVLine(lines[0])
if len(headers) == 0 {
return ImportPreviewResponse{
Errors: []string{"Invalid CSV header"},
}
}
// Count data rows (excluding empty lines)
dataRowCount := 0
for i := 1; i < len(lines); i++ {
if strings.TrimSpace(lines[i]) != "" {
dataRowCount++
}
}
// Check row limit
if dataRowCount > MaxImportRows {
return ImportPreviewResponse{
TotalRows: dataRowCount,
Headers: headers,
Errors: []string{fmt.Sprintf("Too many rows: %d (max %d)", dataRowCount, MaxImportRows)},
}
}
response := ImportPreviewResponse{
TotalRows: dataRowCount,
Headers: headers,
Sample: make([]map[string]any, 0, 10),
Errors: make([]string, 0),
}
rowNum := 0
for i := 1; i < len(lines); i++ {
line := strings.TrimSpace(lines[i])
if line == "" {
continue
}
rowNum++
values := parseCSVLine(line)
if len(values) != len(headers) {
response.InvalidRows++
if len(response.Errors) < 10 {
response.Errors = append(response.Errors, fmt.Sprintf("Row %d: column count mismatch (expected %d, got %d)", rowNum, len(headers), len(values)))
}
continue
}
// Build data map
data := make(map[string]any)
for j, header := range headers {
header = strings.TrimSpace(header)
value := values[j]
// Skip system fields
if header == "id" || header == "created" || header == "updated" || header == "collectionId" || header == "collectionName" {
continue
}
// Parse value according to type
data[header] = parseCSVValue(value)
}
// Validate against schema
if err := content.ValidateDocument(collection, data); err != nil {
response.InvalidRows++
if len(response.Errors) < 10 {
response.Errors = append(response.Errors, fmt.Sprintf("Row %d: %v", rowNum, err))
}
} else {
response.ValidRows++
}
// Add to sample (first 10 rows)
if len(response.Sample) < 10 {
// Include all fields in sample
sample := make(map[string]any)
for j, header := range headers {
sample[strings.TrimSpace(header)] = parseCSVValue(values[j])
}
response.Sample = append(response.Sample, sample)
}
}
return response
}
// parseCSVValue parses a CSV cell value to the appropriate type.
func parseCSVValue(value string) any {
// Try to parse as JSON for complex types
if strings.HasPrefix(value, "[") || strings.HasPrefix(value, "{") {
var parsed any
if err := json.Unmarshal([]byte(value), &parsed); err == nil {
return parsed
}
}
// Try to parse as number
if value != "" && (value[0] >= '0' && value[0] <= '9' || value[0] == '-') {
var f float64
if _, err := fmt.Sscanf(value, "%f", &f); err == nil {
return f
}
}
// Try to parse as boolean
if value == "true" {
return true
}
if value == "false" {
return false
}
// Return as string
return value
}