refactor: Major overhaul to audio clip retention methods, age and usa…

internal/analysis/realtime.go

@@ -112,7 +112,7 @@ func RealtimeAnalysis(settings *conf.Settings) error {
 	startAudioCapture(&wg, settings, quitChan, restartChan, audioBuffer)
 
 	// start cleanup of clips
-	if conf.Setting().Realtime.Audio.Export.Retention.Enabled {
+	if conf.Setting().Realtime.Audio.Export.Retention.Policy != "none" {
 		startClipCleanupMonitor(&wg, settings, dataStore, quitChan)
 	}
 
@@ -206,33 +206,25 @@ func clipCleanupMonitor(wg *sync.WaitGroup, dataStore datastore.Interface, quitC
 	ticker := time.NewTicker(5 * time.Minute)
 	defer ticker.Stop() // Ensure the ticker is stopped to prevent leaks
 
+	log.Println("Clip retention policy:", conf.Setting().Realtime.Audio.Export.Retention.Policy)
+
 	for {
 		select {
 		case <-quitChan:
 			// Handle quit signal to stop the monitor
 			return
 
 		case <-ticker.C:
-			if conf.Setting().Realtime.Audio.Export.Debug {
-				log.Println("Cleanup ticker triggered")
-			}
-
 			// age based cleanup method
-			if conf.Setting().Realtime.Audio.Export.Retention.Mode == "age" {
-				if conf.Setting().Realtime.Audio.Export.Debug {
-					log.Println("Running age based cleanup")
-				}
-				if err := diskmanager.AgeBasedCleanup(dataStore); err != nil {
+			if conf.Setting().Realtime.Audio.Export.Retention.Policy == "age" {
+				if err := diskmanager.AgeBasedCleanup(quitChan); err != nil {
 					log.Println("Error cleaning up clips: ", err)
 				}
 			}
 
 			// priority based cleanup method
-			if conf.Setting().Realtime.Audio.Export.Retention.Mode == "priority" {
-				if conf.Setting().Realtime.Audio.Export.Debug {
-					log.Println("Running priority based cleanup")
-				}
-				if err := diskmanager.PriorityBasedCleanup(quitChan); err != nil {
+			if conf.Setting().Realtime.Audio.Export.Retention.Policy == "usage" {
+				if err := diskmanager.UsageBasedCleanup(quitChan); err != nil {
 					log.Println("Error cleaning up clips: ", err)
 				}
 			}

internal/conf/config.go

@@ -54,11 +54,11 @@ type Settings struct {
 				Path      string // path to audio clip export directory
 				Type      string // audio file type, wav, mp3 or flac
 				Retention struct {
-					Enabled            bool   // true to enable audio clip retention
-					Mode               string // retention mode, "age" or "priority"
-					DiskUsageLimit     string // maximum disk usage percentage before retention
-					MinEvictionHours   int    // minimum number of hours to keep audio clips
-					MinClipsPerSpecies int    // minimum number of clips per species to keep
+					Debug    bool   // true to enable retention debug
+					Policy   string // retention policy, "none", "age" or "usage"
+					MaxAge   string // maximum age of audio clips to keep
+					MaxUsage string // maximum disk usage percentage before cleanup
+					MinClips int    // minimum number of clips per species to keep
 				}
 			}
 		}

internal/conf/config.yaml

@@ -36,12 +36,11 @@ realtime:
       debug: false        # true to enable audio export debug messages
       path: clips/        # path to audio clip export directory
       type: wav           # only wav supported for now
-      retention:
-        enabled: true           # true to enable retention policy of clips
-        mode: priority          # age or priority
-        diskusagelimit: 80%     # percentage of disk usage to trigger eviction        
-        minClipsPerSpecies: 10  # minumum number of clips per species to keep before starting evictions
-        minEvictionHours: 0     # minumum number of hours before considering clip for eviction
+      retention:        
+        policy: usage     # retention policy: none, age or usage
+        maxage: 30d       # age policy: maximum age of clips to keep before starting evictions
+        maxusage: 80%     # usage policy: percentage of disk usage to trigger eviction        
+        minclips: 10      # minumum number of clips per species to keep before starting evictions
 
   log:
     enabled: false      # true to enable OBS chat log

internal/conf/defaults.go

@@ -40,10 +40,10 @@ func setDefaultConfig() {
 
 	viper.SetDefault("realtime.audio.export.retention.enabled", true)
 	viper.SetDefault("realtime.audio.export.retention.debug", false)
-	viper.SetDefault("realtime.audio.export.retention.mode", "priority")
-	viper.SetDefault("realtime.audio.export.retention.diskusagelimit", "80%")
-	viper.SetDefault("realtime.audio.export.retention.minClipsPerSpecies", 10)
-	viper.SetDefault("realtime.audio.export.retention.minEvictionHours", 0)
+	viper.SetDefault("realtime.audio.export.retention.policy", "usage")
+	viper.SetDefault("realtime.audio.export.retention.maxusage", "80%")
+	viper.SetDefault("realtime.audio.export.retention.maxage", "30d")
+	viper.SetDefault("realtime.audio.export.retention.minclips", 10)
 
 	viper.SetDefault("realtime.log.enabled", false)
 	viper.SetDefault("realtime.log.path", "birdnet.txt")

internal/conf/utils.go

@@ -191,3 +191,43 @@ func ParsePercentage(percentage string) (float64, error) {
 	}
 	return 0, errors.New("invalid percentage format")
 }
+
+// ParseRetentionPeriod converts a string like "24h", "7d", "1w", "3m", "1y" to hours.
+func ParseRetentionPeriod(retention string) (int, error) {
+	if len(retention) == 0 {
+		return 0, fmt.Errorf("retention period cannot be empty")
+	}
+
+	// Try to parse the retention period
+	lastChar := retention[len(retention)-1]
+	numberPart := retention[:len(retention)-1]
+
+	// Handle case where the input is a plain integer
+	if lastChar >= '0' && lastChar <= '9' {
+		hours, err := strconv.Atoi(retention)
+		if err != nil {
+			return 0, fmt.Errorf("invalid retention period format: %s", retention)
+		}
+		return hours, nil
+	}
+
+	number, err := strconv.Atoi(numberPart)
+	if err != nil {
+		return 0, fmt.Errorf("invalid retention period format: %s", retention)
+	}
+
+	switch lastChar {
+	case 'h':
+		return number, nil
+	case 'd':
+		return number * 24, nil
+	case 'w':
+		return number * 24 * 7, nil
+	case 'm':
+		return number * 24 * 30, nil // Approximation, as months can vary in length
+	case 'y':
+		return number * 24 * 365, nil // Ignoring leap years for simplicity
+	default:
+		return 0, fmt.Errorf("invalid suffix for retention period: %c", lastChar)
+	}
+}

internal/diskmanager/file_utils.go

@@ -0,0 +1,147 @@
+// file_utils.go - shared file management code
+package diskmanager
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// FileInfo holds information about a file
+type FileInfo struct {
+	Path       string
+	Species    string
+	Confidence int
+	Timestamp  time.Time
+	Size       int64
+}
+
+// LoadPolicy loads the cleanup policies from a CSV file
+func LoadPolicy(policyFile string) (*Policy, error) {
+	file, err := os.Open(policyFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	reader := csv.NewReader(file)
+	records, err := reader.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	policy := &Policy{
+		AlwaysCleanupFirst: make(map[string]bool),
+		NeverCleanup:       make(map[string]bool),
+	}
+
+	for _, record := range records {
+		if len(record) != 2 {
+			return nil, errors.New("invalid policy record")
+		}
+		if record[1] == "always" {
+			policy.AlwaysCleanupFirst[record[0]] = true
+		} else if record[1] == "never" {
+			policy.NeverCleanup[record[0]] = true
+		}
+	}
+
+	return policy, nil
+}
+
+// GetAudioFiles returns a list of audio files in the directory and its subdirectories
+func GetAudioFiles(baseDir string, allowedExts []string, debug bool) ([]FileInfo, error) {
+	var files []FileInfo
+
+	err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			ext := filepath.Ext(info.Name())
+			if contains(allowedExts, ext) {
+				fileInfo, err := parseFileInfo(path, info)
+				if err != nil {
+					return err
+				}
+				files = append(files, fileInfo)
+			}
+		}
+		return nil
+	})
+
+	return files, err
+}
+
+// parseFileInfo parses the file information from the file path and os.FileInfo
+func parseFileInfo(path string, info os.FileInfo) (FileInfo, error) {
+	name := filepath.Base(info.Name())
+	parts := strings.Split(name, "_")
+	if len(parts) < 3 {
+		return FileInfo{}, errors.New("invalid file name format")
+	}
+
+	// The species name might contain underscores, so we need to handle the last two parts separately
+	confidenceStr := parts[len(parts)-2]
+	timestampStr := parts[len(parts)-1]
+	species := strings.Join(parts[:len(parts)-2], "_")
+
+	confidence, err := strconv.Atoi(strings.TrimSuffix(confidenceStr, "p"))
+	if err != nil {
+		return FileInfo{}, err
+	}
+
+	timestamp, err := time.Parse("20060102T150405Z", strings.TrimSuffix(timestampStr, ".wav"))
+	if err != nil {
+		return FileInfo{}, err
+	}
+
+	return FileInfo{
+		Path:       path,
+		Species:    species,
+		Confidence: confidence,
+		Timestamp:  timestamp,
+		Size:       info.Size(),
+	}, nil
+}
+
+// contains checks if a string is in a slice
+func contains(slice []string, item string) bool {
+	for _, s := range slice {
+		if s == item {
+			return true
+		}
+	}
+	return false
+}
+
+// WriteSortedFilesToFile writes the sorted list of files to a text file for investigation
+func WriteSortedFilesToFile(files []FileInfo, filePath string) error {
+	file, err := os.Create(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to create file: %w", err)
+	}
+	defer file.Close()
+
+	for _, fileInfo := range files {
+		line := fmt.Sprintf("Path: %s, Species: %s, Confidence: %d, Timestamp: %s, Size: %d\n",
+			fileInfo.Path, fileInfo.Species, fileInfo.Confidence, fileInfo.Timestamp.Format(time.RFC3339), fileInfo.Size)
+		_, err := file.WriteString(line)
+		if err != nil {
+			return fmt.Errorf("failed to write to file: %w", err)
+		}
+	}
+
+	if err := file.Sync(); err != nil {
+		return fmt.Errorf("failed to sync file: %w", err)
+	}
+
+	log.Printf("Sorted files have been written to %s", filePath)
+	return nil
+}