Artifact and media metadata were previously reconstructed from job result refs and JSON sidecar files. This change regularizes metadata into SQLite, keeps filesystem blobs in place, and preserves backward compatibility via lazy fallback and backfill from existing job refs and JSON metadata. Constraint: Blob storage remains on the local filesystem in this phase Rejected: Migrate blobs into SQLite | larger scope and worse operational profile for current media sizes Rejected: Hard cutover without fallback | unsafe for historical data already on the test server Confidence: medium Scope-risk: moderate Directive: Treat SQLite as the metadata source of truth; JSON sidecars are compatibility fallback only Tested: go test ./...; deployed to test server 101.42.99.35; verified /v1/artifacts, /v1/artifacts/:id, signed media URL 200, unsigned content 401 Not-tested: Full historical backfill sweep over all existing artifact rows under production-sized data volume
285 lines
7.5 KiB
Go
285 lines
7.5 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/hmac"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type mediaRecord struct {
|
|
media
|
|
UserID string `json:"user_id"`
|
|
LocalPath string `json:"local_path"`
|
|
}
|
|
|
|
func mediaDir(cfg Config) string {
|
|
return filepath.Join(cfg.DataDir, "media")
|
|
}
|
|
|
|
func mediaBlobDir(cfg Config) string {
|
|
return filepath.Join(mediaDir(cfg), "blobs")
|
|
}
|
|
|
|
func mediaMetaDir(cfg Config) string {
|
|
return filepath.Join(mediaDir(cfg), "meta")
|
|
}
|
|
|
|
func mediaMetaPath(cfg Config, mediaID string) string {
|
|
return filepath.Join(mediaMetaDir(cfg), mediaID+".json")
|
|
}
|
|
|
|
func mediaBlobPath(cfg Config, mediaID, filename string) string {
|
|
ext := filepath.Ext(filename)
|
|
if ext == "" {
|
|
ext = extensionFromContentType("")
|
|
}
|
|
return filepath.Join(mediaBlobDir(cfg), mediaID+ext)
|
|
}
|
|
|
|
func publicBaseURL(cfg Config) string {
|
|
base := strings.TrimSpace(cfg.PublicBaseURL)
|
|
if base == "" {
|
|
base = strings.TrimSpace(os.Getenv("POPIART_SERVER_ADDR"))
|
|
}
|
|
if base == "" {
|
|
base = "127.0.0.1:8080"
|
|
}
|
|
if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") {
|
|
base = "http://" + base
|
|
}
|
|
base = strings.Replace(base, "0.0.0.0", "127.0.0.1", 1)
|
|
return strings.TrimRight(base, "/")
|
|
}
|
|
|
|
func mediaContentPath(mediaID string) string {
|
|
return "/v1/media/" + url.PathEscape(strings.TrimSpace(mediaID)) + "/content"
|
|
}
|
|
|
|
func mediaContentURL(cfg Config, mediaID string) string {
|
|
return publicBaseURL(cfg) + mediaContentPath(mediaID)
|
|
}
|
|
|
|
func mediaURLTTL() time.Duration {
|
|
return 24 * time.Hour
|
|
}
|
|
|
|
func mediaURLExpiry(now time.Time) time.Time {
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
return now.UTC().Add(mediaURLTTL())
|
|
}
|
|
|
|
func mediaURLSecret(cfg Config) []byte {
|
|
return []byte(defaultString(strings.TrimSpace(cfg.SessionSecret), "popiart-dev-session-secret"))
|
|
}
|
|
|
|
func signedMediaContentURL(cfg Config, mediaID string, now time.Time) string {
|
|
expiresAt := mediaURLExpiry(now)
|
|
baseURL := mediaContentURL(cfg, mediaID)
|
|
exp := strconv.FormatInt(expiresAt.Unix(), 10)
|
|
sig := signMediaAccess(cfg, mediaID, exp)
|
|
return baseURL + "?exp=" + url.QueryEscape(exp) + "&sig=" + url.QueryEscape(sig)
|
|
}
|
|
|
|
func signMediaAccess(cfg Config, mediaID, exp string) string {
|
|
mac := hmac.New(sha256.New, mediaURLSecret(cfg))
|
|
_, _ = mac.Write([]byte(strings.TrimSpace(mediaID)))
|
|
_, _ = mac.Write([]byte("\n"))
|
|
_, _ = mac.Write([]byte(strings.TrimSpace(exp)))
|
|
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
|
}
|
|
|
|
func validateSignedMediaAccess(cfg Config, mediaID, exp, sig string, now time.Time) bool {
|
|
exp = strings.TrimSpace(exp)
|
|
sig = strings.TrimSpace(sig)
|
|
if exp == "" || sig == "" {
|
|
return false
|
|
}
|
|
unix, err := strconv.ParseInt(exp, 10, 64)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
if now.UTC().After(time.Unix(unix, 0).UTC()) {
|
|
return false
|
|
}
|
|
expected := signMediaAccess(cfg, mediaID, exp)
|
|
return hmac.Equal([]byte(expected), []byte(sig))
|
|
}
|
|
|
|
func mediaView(cfg Config, record mediaRecord, now time.Time) media {
|
|
view := record.media
|
|
view.URL = signedMediaContentURL(cfg, record.ID, now)
|
|
return view
|
|
}
|
|
|
|
func artifactView(cfg Config, item artifact, now time.Time) artifact {
|
|
view := item
|
|
if strings.TrimSpace(view.MediaID) != "" {
|
|
view.URL = signedMediaContentURL(cfg, view.MediaID, now)
|
|
}
|
|
return view
|
|
}
|
|
|
|
func artifactViews(cfg Config, items []artifact, now time.Time) []artifact {
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]artifact, 0, len(items))
|
|
for _, item := range items {
|
|
out = append(out, artifactView(cfg, item, now))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mediaContentURLCanonical(cfg Config, mediaID string) string {
|
|
return publicBaseURL(cfg) + "/v1/media/" + url.PathEscape(strings.TrimSpace(mediaID)) + "/content"
|
|
}
|
|
|
|
func persistMediaContent(cfg Config, repo MediaRepository, userID, projectID, artifactID, filename, contentType, visibility string, content []byte, mediaID string) (mediaRecord, error) {
|
|
if len(content) == 0 {
|
|
return mediaRecord{}, fmt.Errorf("media content is required")
|
|
}
|
|
filename = sanitizeFilename(strings.TrimSpace(filename))
|
|
if filename == "" || filename == "artifact" {
|
|
filename = "artifact" + extensionFromContentType(contentType)
|
|
}
|
|
contentType = strings.TrimSpace(contentType)
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
visibility = strings.TrimSpace(visibility)
|
|
if visibility == "" {
|
|
visibility = "unlisted"
|
|
}
|
|
mediaID = strings.TrimSpace(mediaID)
|
|
if mediaID == "" {
|
|
mediaID = "med_" + randomID()
|
|
}
|
|
if err := os.MkdirAll(mediaBlobDir(cfg), 0o755); err != nil {
|
|
return mediaRecord{}, err
|
|
}
|
|
if err := os.MkdirAll(mediaMetaDir(cfg), 0o755); err != nil {
|
|
return mediaRecord{}, err
|
|
}
|
|
|
|
sum := sha256.Sum256(content)
|
|
blobPath := mediaBlobPath(cfg, mediaID, filename)
|
|
if err := os.WriteFile(blobPath, content, 0o644); err != nil {
|
|
return mediaRecord{}, err
|
|
}
|
|
|
|
record := mediaRecord{
|
|
media: media{
|
|
ID: mediaID,
|
|
ArtifactID: strings.TrimSpace(artifactID),
|
|
ProjectID: strings.TrimSpace(projectID),
|
|
Filename: filename,
|
|
ContentType: contentType,
|
|
SizeBytes: int64(len(content)),
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
URL: mediaContentURLCanonical(cfg, mediaID),
|
|
Visibility: visibility,
|
|
SHA256: hex.EncodeToString(sum[:]),
|
|
},
|
|
UserID: strings.TrimSpace(userID),
|
|
LocalPath: blobPath,
|
|
}
|
|
|
|
metaBytes, err := json.Marshal(record)
|
|
if err != nil {
|
|
return mediaRecord{}, err
|
|
}
|
|
if err := os.WriteFile(mediaMetaPath(cfg, mediaID), metaBytes, 0o644); err != nil {
|
|
return mediaRecord{}, err
|
|
}
|
|
if repo != nil {
|
|
if err := repo.UpsertMedia(record); err != nil {
|
|
return mediaRecord{}, err
|
|
}
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func loadMediaRecordFromJSON(cfg Config, mediaID string) (*mediaRecord, bool, error) {
|
|
mediaID = strings.TrimSpace(mediaID)
|
|
if mediaID == "" {
|
|
return nil, false, nil
|
|
}
|
|
data, err := os.ReadFile(mediaMetaPath(cfg, mediaID))
|
|
if os.IsNotExist(err) {
|
|
return nil, false, nil
|
|
}
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
var record mediaRecord
|
|
if err := json.Unmarshal(data, &record); err != nil {
|
|
return nil, false, err
|
|
}
|
|
return &record, true, nil
|
|
}
|
|
|
|
func (s *Server) persistResultRefs(ctx context.Context, record *job, refs []resultRef) ([]resultRef, error) {
|
|
if record == nil || len(refs) == 0 {
|
|
return refs, nil
|
|
}
|
|
items := make([]resultRef, 0, len(refs))
|
|
for idx, ref := range refs {
|
|
contentType, _, reader, err := s.newapi.openResultRef(ctx, record.UpstreamKey, ref)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
content, readErr := io.ReadAll(reader)
|
|
reader.Close()
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
filename := strings.TrimSpace(ref.Filename)
|
|
if filename == "" {
|
|
filename = inferFilenameFromRef(ref, buildArtifactID(record.JobID, idx))
|
|
}
|
|
artifactID := buildArtifactID(record.JobID, idx)
|
|
mediaRecord, err := s.store.persistMediaContent(
|
|
record.UserID,
|
|
record.ProjectID,
|
|
artifactID,
|
|
filename,
|
|
defaultString(strings.TrimSpace(contentType), strings.TrimSpace(ref.ContentType)),
|
|
defaultString(strings.TrimSpace(ref.Visibility), "unlisted"),
|
|
content,
|
|
"",
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, resultRef{
|
|
Kind: "local_path",
|
|
URL: mediaRecord.URL,
|
|
LocalPath: mediaRecord.LocalPath,
|
|
MediaID: mediaRecord.ID,
|
|
Filename: mediaRecord.Filename,
|
|
ContentType: mediaRecord.ContentType,
|
|
SizeBytes: mediaRecord.SizeBytes,
|
|
Visibility: mediaRecord.Visibility,
|
|
SHA256: mediaRecord.SHA256,
|
|
StorageStatus: "ready",
|
|
})
|
|
}
|
|
return items, nil
|
|
}
|