package persistence
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"slices"
"strings"
"time"
. "github.com/Masterminds/squirrel"
"github.com/deluan/rest"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
"github.com/navidrome/navidrome/utils/str"
"github.com/pocketbase/dbx"
)
type artistRepository struct {
sqlRepository
indexGroups utils.IndexGroups
}
type dbArtist struct {
*model.Artist `structs:",flatten"`
SimilarArtists string `structs:"-" json:"-"`
LibraryStatsJSON string `structs:"-" json:"-"`
}
type dbSimilarArtist struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}
func (a *dbArtist) PostScan() error {
a.Artist.Stats = make(map[model.Role]model.ArtistStats)
if a.LibraryStatsJSON != "" {
var rawLibStats map[string]map[string]map[string]int64
if err := json.Unmarshal([]byte(a.LibraryStatsJSON), &rawLibStats); err != nil {
return fmt.Errorf("parsing artist stats from db: %w", err)
}
for _, stats := range rawLibStats {
for key, stat := range stats {
artistStats := model.ArtistStats{
SongCount: int(stat["m"]),
AlbumCount: int(stat["a"]),
Size: stat["s"],
}
if key == "total" {
a.Artist.Size += artistStats.Size
a.Artist.SongCount += artistStats.SongCount
a.Artist.AlbumCount += artistStats.AlbumCount
}
role := model.RoleFromString(key)
if role == model.RoleInvalid {
continue
}
current := a.Artist.Stats[role]
current.Size += artistStats.Size
current.SongCount += artistStats.SongCount
current.AlbumCount += artistStats.AlbumCount
a.Artist.Stats[role] = current
}
}
}
a.Artist.SimilarArtists = nil
if a.SimilarArtists == "" {
return nil
}
var sa []dbSimilarArtist
if err := json.Unmarshal([]byte(a.SimilarArtists), &sa); err != nil {
return fmt.Errorf("parsing similar artists from db: %w", err)
}
for _, s := range sa {
a.Artist.SimilarArtists = append(a.Artist.SimilarArtists, model.Artist{
ID: s.ID,
Name: s.Name,
})
}
return nil
}
func (a *dbArtist) PostMapArgs(m map[string]any) error {
sa := make([]dbSimilarArtist, 0)
for _, s := range a.Artist.SimilarArtists {
sa = append(sa, dbSimilarArtist{ID: s.ID, Name: s.Name})
}
similarArtists, _ := json.Marshal(sa)
m["similar_artists"] = string(similarArtists)
m["full_text"] = formatFullText(a.Name, a.SortArtistName)
m["search_normalized"] = str.NormalizeForFTS(a.Name)
if v, ok := m["sort_artist_name"]; !ok || v.(string) == "" {
delete(m, "sort_artist_name")
}
if v, ok := m["mbz_artist_id"]; !ok || v.(string) == "" {
delete(m, "mbz_artist_id")
}
return nil
}
type dbArtists []dbArtist
func (dba dbArtists) toModels() model.Artists {
res := make(model.Artists, len(dba))
for i := range dba {
res[i] = *dba[i].Artist
}
return res
}
func NewArtistRepository(db dbx.Builder) model.ArtistRepository {
r := &artistRepository{}
r.db = db
r.indexGroups = utils.ParseIndexGroups(conf.Server.IndexGroups)
r.tableName = "artist"
r.registerModel(&model.Artist{}, map[string]filterFunc{
"id": idFilter(r.tableName),
"name": fullTextFilter(r.tableName, "mbz_artist_id"),
"starred": annotationBoolFilter("starred"),
"has_rating": annotationBoolFilter("rating"),
"role": roleFilter,
"missing": booleanFilter,
"library_id": artistLibraryIdFilter,
})
r.setSortMappings(map[string]string{
"name": "order_artist_name",
"starred_at": "starred, starred_at",
"rated_at": "rating, rated_at",
"song_count": "stats->>'total'->>'m'",
"album_count": "stats->>'total'->>'a'",
"size": "stats->>'total'->>'s'",
"maincredit_song_count": "sum(stats->>'maincredit'->>'m')",
"maincredit_album_count": "sum(stats->>'maincredit'->>'a')",
"maincredit_size": "sum(stats->>'maincredit'->>'s')",
})
return r
}
func roleFilter(_ string, role any) Sqlizer {
if role, ok := role.(string); ok {
if safe, ok := sanitizeArtistStatsRole(role); ok && safe != "total" {
return Expr("JSON_EXTRACT(library_artist.stats, '$." + safe + ".m') IS NOT NULL")
}
}
return Eq{"1": 2}
}
func sanitizeArtistStatsRole(role string) (string, bool) {
if role == "" || role == "total" {
return "total", true
}
if _, ok := model.AllRoles[role]; ok {
return role, true
}
return "", false
}
func artistLibraryIdFilter(_ string, value any) Sqlizer {
return Eq{"library_artist.library_id": value}
}
func (r *artistRepository) applyLibraryFilterToArtistQuery(ctx context.Context, query SelectBuilder) SelectBuilder {
user := loggedUser(ctx)
query = query.Join("library_artist on library_artist.artist_id = artist.id")
if user.ID != invalidUserId && !user.IsAdmin {
query = query.Join("user_library on user_library.library_id = library_artist.library_id AND user_library.user_id = ?", user.ID)
}
return query
}
func (r *artistRepository) selectArtist(ctx context.Context, options ...model.QueryOptions) SelectBuilder {
query := r.newSelect(ctx, options...).Columns("artist.*",
"JSON_GROUP_OBJECT(library_artist.library_id, JSONB(library_artist.stats)) as library_stats_json")
query = r.applyLibraryFilterToArtistQuery(ctx, query)
query = query.GroupBy("artist.id")
return r.withAnnotation(ctx, query, "artist.id")
}
func (r *artistRepository) CountAll(ctx context.Context, options ...model.QueryOptions) (int64, error) {
query := r.newSelect(ctx)
query = r.applyLibraryFilterToArtistQuery(ctx, query)
if filtersNeedAnnotation(r.applyFilters(query, options...)) {
query = r.withAnnotation(ctx, query, "artist.id")
}
return r.count(ctx, query, options...)
}
func (r *artistRepository) Exists(ctx context.Context, id string) (bool, error) {
query := r.newSelect(ctx).Columns("count(distinct artist.id) as exist").Where(Eq{"artist.id": id})
query = r.applyLibraryFilterToArtistQuery(ctx, query)
var res struct{ Exist int64 }
err := r.queryOne(ctx, query, &res)
return res.Exist > 0, err
}
func (r *artistRepository) Put(ctx context.Context, a *model.Artist, colsToUpdate ...string) error {
dba := &dbArtist{Artist: a}
dba.CreatedAt = new(time.Now())
dba.UpdatedAt = dba.CreatedAt
_, err := r.put(ctx, dba.ID, dba, colsToUpdate...)
return err
}
func (r *artistRepository) UpdateExternalInfo(ctx context.Context, a *model.Artist) error {
dba := &dbArtist{Artist: a}
_, err := r.put(ctx, a.ID, dba,
"biography", "small_image_url", "medium_image_url", "large_image_url",
"similar_artists", "external_url", "external_info_updated_at")
return err
}
func (r *artistRepository) Get(ctx context.Context, id string) (*model.Artist, error) {
sel := r.selectArtist(ctx).Where(Eq{"artist.id": id})
var dba dbArtists
if err := r.queryAll(ctx, sel, &dba); err != nil {
return nil, err
}
if len(dba) == 0 {
return nil, model.ErrNotFound
}
res := dba.toModels()
r.hydrateArtwork(ctx, res)
return &res[0], nil
}
func (r *artistRepository) GetAll(ctx context.Context, options ...model.QueryOptions) (model.Artists, error) {
sel := r.selectArtist(ctx, options...)
var dba dbArtists
err := r.queryAll(ctx, sel, &dba)
if err != nil {
return nil, err
}
res := dba.toModels()
r.hydrateArtwork(ctx, res)
return res, err
}
func (r *artistRepository) getAllIDs(ctx context.Context, options ...model.QueryOptions) ([]string, error) {
sq := r.applyLibraryFilterToArtistQuery(ctx, r.newSelect(ctx, options...).Columns("artist.id")).GroupBy("artist.id")
if filtersNeedAnnotation(sq) {
sq = r.withAnnotation(ctx, sq, "artist.id")
}
ids := []string{}
err := r.queryAllSlice(ctx, sq, &ids)
return ids, err
}
func (r *artistRepository) hydrateArtwork(ctx context.Context, artists model.Artists) {
hydrateItems(ctx, r.db, model.KindArtistArtwork, artists,
func(a *model.Artist) (string, *model.ItemImage) { return a.ID, &a.ItemImage })
}
func (r *artistRepository) GetCursor(ctx context.Context, options ...model.QueryOptions) (model.ArtistCursor, error) {
ids, err := r.getAllIDs(ctx, options...)
if err != nil {
return nil, err
}
opts := chunkOptions(options, "artist.id")
return model.ArtistCursor(streamByIDs(ids, func(chunk []string) (model.Artists, error) {
return r.GetAll(ctx, opts(chunk))
})), nil
}
func (r *artistRepository) getIndexKey(a model.Artist) string {
source := a.OrderArtistName
if conf.Server.PreferSortTags {
source = cmp.Or(a.SortArtistName, a.OrderArtistName)
}
name := strings.ToLower(source)
for k, v := range r.indexGroups {
if strings.HasPrefix(name, strings.ToLower(k)) {
return v
}
}
return "#"
}
func (r *artistRepository) GetIndex(ctx context.Context, includeMissing bool, libraryIds []int, roles ...model.Role) (model.ArtistIndexes, error) {
if len(libraryIds) == 0 {
return nil, nil
}
options := model.QueryOptions{Sort: "name"}
if len(roles) > 0 {
roleFilters := slice.Map(roles, func(r model.Role) Sqlizer {
return roleFilter("role", r.String())
})
options.Filters = Or(roleFilters)
}
if !includeMissing {
if options.Filters == nil {
options.Filters = Eq{"artist.missing": false}
} else {
options.Filters = And{options.Filters, Eq{"artist.missing": false}}
}
}
libFilter := artistLibraryIdFilter("library_id", libraryIds)
if options.Filters == nil {
options.Filters = libFilter
} else {
options.Filters = And{options.Filters, libFilter}
}
artists, err := r.GetAll(ctx, options)
if err != nil {
return nil, err
}
var result model.ArtistIndexes
for k, v := range slice.Group(artists, r.getIndexKey) {
result = append(result, model.ArtistIndex{ID: k, Artists: v})
}
slices.SortFunc(result, func(a, b model.ArtistIndex) int {
return cmp.Compare(a.ID, b.ID)
})
return result, nil
}
func (r *artistRepository) purgeEmpty(ctx context.Context) error {
orphanFilter := "id not in (select artist_id from album_artists)"
sel := Select("uploaded_image").From(r.tableName).
Where(orphanFilter).
Where("uploaded_image != ''")
var imageFiles []string
if err := r.queryAllSlice(ctx, sel, &imageFiles); err != nil && !errors.Is(err, model.ErrNotFound) {
return fmt.Errorf("collecting artist images for cleanup: %w", err)
}
del := Delete(r.tableName).Where(orphanFilter)
c, err := r.executeSQL(ctx, del)
if err != nil {
return fmt.Errorf("purging empty artists: %w", err)
}
if c > 0 {
log.Debug(ctx, "Purged empty artists", "totalDeleted", c)
}
if len(imageFiles) == 0 {
return nil
}
log.Debug(ctx, "Cleaning up artist images", "totalImages", len(imageFiles))
for _, filename := range imageFiles {
path := model.UploadedImagePath(consts.EntityArtist, filename)
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
log.Warn(ctx, "Failed to remove artist image during GC", "path", path, err)
}
}
return nil
}
func (r *artistRepository) markOrphansMissing(ctx context.Context) error {
_, err := r.executeSQL(ctx, Expr(
"update artist set missing = true where missing = false "+
"and not exists (select 1 from library_artist where library_artist.artist_id = artist.id)"))
if err != nil {
return fmt.Errorf("marking orphaned artists missing: %w", err)
}
return nil
}
func (r *artistRepository) markMissing(ctx context.Context) error {
q := Expr(`
with artists_with_non_missing_albums as (
select distinct aa.artist_id
from album_artists aa
join album a on aa.album_id = a.id
where a.missing = false
)
update artist
set missing = (artist.id not in (select artist_id from artists_with_non_missing_albums));
`)
_, err := r.executeSQL(ctx, q)
if err != nil {
return fmt.Errorf("marking missing artists: %w", err)
}
return nil
}
func (r *artistRepository) RefreshPlayCounts(ctx context.Context) (int64, error) {
query := Expr(`
with play_counts as (
select user_id, atom as artist_id, sum(play_count) as total_play_count, max(play_date) as last_play_date
from media_file
join annotation on item_id = media_file.id
left join json_tree(participants, '$.artist') as jt
where atom is not null and key = 'id'
group by user_id, atom
)
insert into annotation (user_id, item_id, item_type, play_count, play_date)
select user_id, artist_id, 'artist', total_play_count, last_play_date
from play_counts
where total_play_count > 0
on conflict (user_id, item_id, item_type) do update
set play_count = excluded.play_count,
play_date = excluded.play_date;
`)
return r.executeSQL(ctx, query)
}
func (r *artistRepository) RefreshStats(ctx context.Context, allArtists bool) (int64, error) {
var allTouchedArtistIDs []string
if allArtists {
allArtistsQuerySQL := `SELECT DISTINCT id FROM artist WHERE id <> ''`
if err := r.db.NewQuery(allArtistsQuerySQL).Column(&allTouchedArtistIDs); err != nil {
return 0, fmt.Errorf("fetching all artist IDs: %w", err)
}
log.Debug(ctx, "RefreshStats: Refreshing all artists.", "count", len(allTouchedArtistIDs))
} else {
touchedArtistsQuerySQL := `
SELECT DISTINCT id
FROM artist
WHERE updated_at > (SELECT last_scan_at FROM library ORDER BY last_scan_at ASC LIMIT 1)
`
if err := r.db.NewQuery(touchedArtistsQuerySQL).Column(&allTouchedArtistIDs); err != nil {
return 0, fmt.Errorf("fetching touched artist IDs: %w", err)
}
log.Debug(ctx, "RefreshStats: Refreshing touched artists.", "count", len(allTouchedArtistIDs))
}
if len(allTouchedArtistIDs) == 0 {
log.Debug(ctx, "RefreshStats: No artists to update.")
return 0, nil
}
batchUpdateStatsSQL := `
WITH artist_role_counters AS (
SELECT mfa.artist_id,
mf.library_id,
mfa.role,
count(DISTINCT mf.album_id) AS album_count,
count(DISTINCT mf.id) AS count,
sum(mf.size) AS size
FROM media_file_artists mfa
JOIN media_file mf ON mfa.media_file_id = mf.id
WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders
GROUP BY mfa.artist_id, mf.library_id, mfa.role
),
artist_total_counters AS (
SELECT mfa.artist_id,
mf.library_id,
'total' AS role,
count(DISTINCT mf.album_id) AS album_count,
count(DISTINCT mf.id) AS count,
sum(mf.size) AS size
FROM media_file_artists mfa
JOIN media_file mf ON mfa.media_file_id = mf.id
WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders
GROUP BY mfa.artist_id, mf.library_id
),
artist_participant_counter AS (
SELECT mfa.artist_id,
mf.library_id,
'maincredit' AS role,
count(DISTINCT mf.album_id) AS album_count,
count(DISTINCT mf.id) AS count,
sum(mf.size) AS size
FROM media_file_artists mfa
JOIN media_file mf ON mfa.media_file_id = mf.id
WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders
AND mfa.role IN ('albumartist', 'artist')
GROUP BY mfa.artist_id, mf.library_id
),
combined_counters AS (
SELECT artist_id, library_id, role, album_count, count, size FROM artist_role_counters
UNION ALL
SELECT artist_id, library_id, role, album_count, count, size FROM artist_total_counters
UNION ALL
SELECT artist_id, library_id, role, album_count, count, size FROM artist_participant_counter
),
library_artist_counters AS (
SELECT artist_id,
library_id,
json_group_object(
role,
json_object('a', album_count, 'm', count, 's', size)
) AS counters
FROM combined_counters
GROUP BY artist_id, library_id
)
UPDATE library_artist
SET stats = coalesce((SELECT counters FROM library_artist_counters lac
WHERE lac.artist_id = library_artist.artist_id
AND lac.library_id = library_artist.library_id), '{}')
WHERE library_artist.artist_id IN (ROLE_IDS_PLACEHOLDER);`
var totalRowsAffected int64 = 0
const batchSize = 1000
batchCounter := 0
for artistIDBatch := range slice.CollectChunks(slices.Values(allTouchedArtistIDs), batchSize) {
batchCounter++
log.Trace(ctx, "RefreshStats: Processing batch", "batchNum", batchCounter, "batchSize", len(artistIDBatch))
placeholders := make([]string, len(artistIDBatch))
for i := range artistIDBatch {
placeholders[i] = "?"
}
inClause := strings.Join(placeholders, ",")
batchSQL := strings.Replace(batchUpdateStatsSQL, "ROLE_IDS_PLACEHOLDER", inClause, 4)
args := make([]any, 4*len(artistIDBatch))
for idx, id := range artistIDBatch {
for i := range 4 {
startIdx := i * len(artistIDBatch)
args[startIdx+idx] = id
}
}
sqlizer := Expr(batchSQL, args...)
rowsAffected, err := r.executeSQL(ctx, sqlizer)
if err != nil {
return totalRowsAffected, fmt.Errorf("executing batch update for artist stats (batch %d): %w", batchCounter, err)
}
totalRowsAffected += rowsAffected
}
cleanupSQL := Delete("library_artist").Where("stats = '{}'")
cleanupRows, err := r.executeSQL(ctx, cleanupSQL)
if err != nil {
log.Warn(ctx, "Failed to cleanup empty library_artist entries", err)
} else {
if cleanupRows > 0 {
log.Debug(ctx, "Cleaned up empty library_artist entries", "rowsDeleted", cleanupRows)
}
if cleanupRows > 0 || allArtists {
if err := r.markOrphansMissing(ctx); err != nil {
log.Warn(ctx, "Failed to mark orphaned artists missing after library_artist cleanup", err)
}
}
}
log.Debug(ctx, "RefreshStats: Successfully updated stats.", "totalArtistsProcessed", len(allTouchedArtistIDs), "totalDBRowsAffected", totalRowsAffected)
return totalRowsAffected, nil
}
func (r *artistRepository) searchCfg(scope []int) searchConfig {
return searchConfig{
NaturalOrder: "artist.id",
OrderBy: []string{"sum(json_extract(stats, '$.total.m')) desc", "name"},
MBIDFields: []string{"mbz_artist_id"},
LibraryFilter: func(query SelectBuilder) SelectBuilder {
if scope == nil {
return query
}
return query.Where(artistLibraryFilter(scope))
},
}
}
func artistLibraryFilter(libraryIDs []int) Sqlizer {
if len(libraryIDs) == 0 {
return Eq{"1": 2}
}
sub, args, _ := Select("1").From("library_artist").
Where(And{
Expr("library_artist.artist_id = artist.id"),
Eq{"library_artist.library_id": libraryIDs},
}).Limit(1).ToSql()
return Expr("EXISTS ("+sub+")", args...)
}
func (r *artistRepository) Search(ctx context.Context, q string, options ...model.QueryOptions) (model.Artists, error) {
var opts model.QueryOptions
if len(options) > 0 {
opts = options[0]
}
scope := r.searchScope(ctx, opts.Filters)
if isLibraryIDFilter(opts.Filters) {
opts.Filters = nil
}
var res dbArtists
err := r.doSearch(ctx, r.selectArtist(ctx, opts), q, &res, r.searchCfg(scope), opts)
if err != nil {
return nil, fmt.Errorf("searching artist %q: %w", q, err)
}
artists := res.toModels()
r.hydrateArtwork(ctx, artists)
return artists, nil
}
func (r *artistRepository) searchScope(ctx context.Context, filter Sqlizer) []int {
visible, err := r.visibleLibraryIDs(ctx)
if err != nil {
return r.requestedLibraryIDs(filter)
}
requested := r.requestedLibraryIDs(filter)
if requested == nil {
if r.userSeesAllLibraries(ctx, visible) {
return nil
}
return visible
}
requestedSet := slice.ToSet(requested)
if slices.ContainsFunc(visible, func(id int) bool { _, ok := requestedSet[id]; return !ok }) {
return requested
}
return nil
}
func (r *artistRepository) requestedLibraryIDs(filter Sqlizer) []int {
eq, ok := filter.(Eq)
if !ok {
return nil
}
ids, _ := eq["library_id"].([]int)
return ids
}
func isLibraryIDFilter(filter Sqlizer) bool {
eq, ok := filter.(Eq)
if !ok {
return false
}
_, ok = eq["library_id"]
return ok
}
func (r *artistRepository) Count(ctx context.Context, options ...rest.QueryOptions) (int64, error) {
return r.CountAll(ctx, r.parseRestOptions(ctx, options...))
}
func (r *artistRepository) Read(ctx context.Context, id string) (*model.Artist, error) {
return r.Get(ctx, id)
}
func (r *artistRepository) sortMappingsForRole(options ...rest.QueryOptions) map[string]string {
role := "total"
if len(options) > 0 {
if v, ok := options[0].Filters["role"].(string); ok {
if safe, ok := sanitizeArtistStatsRole(v); ok {
role = safe
}
}
}
mappings := maps.Clone(r.sortMappings)
mappings["song_count"] = "sum(stats->>'" + role + "'->>'m')"
mappings["album_count"] = "sum(stats->>'" + role + "'->>'a')"
mappings["size"] = "sum(stats->>'" + role + "'->>'s')"
return mappings
}
func (r *artistRepository) ReadAll(ctx context.Context, options ...rest.QueryOptions) ([]model.Artist, error) {
scoped := *r
scoped.sortMappings = r.sortMappingsForRole(options...)
return scoped.GetAll(ctx, scoped.parseRestOptions(ctx, options...))
}
var _ model.ArtistRepository = (*artistRepository)(nil)
var _ rest.Repository[model.Artist] = (*artistRepository)(nil)