Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add initial retention policy #121

Merged
merged 1 commit into from
Apr 16, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions internal/analysis/realtime.go
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
"os/signal"
"sync"
"syscall"
"time"

"github.com/tphakala/birdnet-go/internal/analysis/processor"
"github.com/tphakala/birdnet-go/internal/analysis/queue"
@@ -82,6 +83,8 @@
startBufferMonitor(&wg, bn, quitChan)
// start audio capture
startAudioCapture(&wg, settings, quitChan, restartChan, audioBuffer)
// start cleanup of clips
startClipCleanupMonitor(&wg, settings, dataStore, quitChan)

// start quit signal monitor
monitorCtrlC(quitChan)
@@ -105,6 +108,7 @@
startAudioCapture(&wg, settings, quitChan, restartChan, audioBuffer)
}
}

}

// startAudioCapture initializes and starts the audio capture routine in a new goroutine.
@@ -119,6 +123,12 @@
go myaudio.BufferMonitor(wg, bn, quitChan)
}

// startClipCleanupMonitor initializes and starts the clip cleanup monitoring routine in a new goroutine.
func startClipCleanupMonitor(wg *sync.WaitGroup, settings *conf.Settings, dataStore datastore.Interface, quitChan chan struct{}) {
wg.Add(1)
go ClipCleanupMonitor(wg, settings, dataStore, quitChan)
}

// monitorCtrlC listens for the SIGINT (Ctrl+C) signal and triggers the application shutdown process.
func monitorCtrlC(quitChan chan struct{}) {
go func() {
@@ -140,3 +150,34 @@
log.Println("Successfully closed database")
}
}

// ClipCleanupMonitor monitors the database and deletes clips that meets the retention policy.
func ClipCleanupMonitor(wg *sync.WaitGroup, settings *conf.Settings, dataStore datastore.Interface, quitChan chan struct{}) {
defer wg.Done()

// Creating a ticker that ticks every 1 minute
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()

for {
select {
case <-quitChan:
// Quit signal received, stop the clip cleanup monitor
return

case <-ticker.C: // Wait for the next tick
clipsForRemoval, _ := dataStore.GetClipsQualifyingForRemoval(settings.Realtime.Retention.MinEvictionHours, settings.Realtime.Retention.MinClipsPerSpecies)

log.Printf("Found %d clips to remove\n", len(clipsForRemoval))

for _, clip := range clipsForRemoval {
if err := os.Remove(clip.ClipName); err != nil {
log.Printf("Failed to remove %s: %s\n", clip.ClipName, err)
} else {
log.Printf("Removed %s\n", clip.ClipName)
}
dataStore.DeleteNoteClipPath(clip.ID)

Check failure on line 179 in internal/analysis/realtime.go

GitHub Actions / lint

Error return value of `dataStore.DeleteNoteClipPath` is not checked (errcheck)
}
}
}
}
10 changes: 10 additions & 0 deletions internal/conf/config.go
Original file line number Diff line number Diff line change
@@ -66,6 +66,11 @@ type Settings struct {
Enabled bool // true to enable dog bark filter
}

Retention struct {
MinEvictionHours int // minimum number of hours to keep audio clips
MinClipsPerSpecies int // minimum number of clips per species to keep
}

RTSP struct {
Url string // RTSP stream URL
Transport string // RTSP Transport Protocol
@@ -266,6 +271,11 @@ realtime:
dogbarkfilter:
enabled: true
retention:
enabled: true # true to enable retention policy of clips
minEvictionHours: 72 # minumum number of hours before considering clip for eviction
minClipsPerSpecies: 10 # minumum number of clips per species to keep before starting evictions
webserver:
enabled: true # true to enable web server
port: 8080 # port for web server
45 changes: 45 additions & 0 deletions internal/datastore/interfaces.go
Original file line number Diff line number Diff line change
@@ -26,6 +26,8 @@ type Interface interface {
GetLastDetections(numDetections int) ([]Note, error)
SearchNotes(query string, sortAscending bool, limit int, offset int) ([]Note, error)
GetNoteClipPath(noteID string) (string, error)
DeleteNoteClipPath(noteID string) error
GetClipsQualifyingForRemoval(minHours int, minClips int) ([]ClipForRemoval, error)
}

// DataStore implements StoreInterface using a GORM database.
@@ -148,6 +150,19 @@ func (ds *DataStore) GetNoteClipPath(noteID string) (string, error) {
return clipPath.ClipName, nil
}

// DeleteNoteClipPath deletes the field representing the path to the audio clip associated with a note.
func (ds *DataStore) DeleteNoteClipPath(noteID string) error {
err := ds.DB.Model(&Note{}).
Where("id = ?", noteID).
Update("clip_name", "").Error

if err != nil {
return fmt.Errorf("failed to delete clip path: %w", err)
}

return nil
}

// GetAllNotes retrieves all notes from the database.
func (ds *DataStore) GetAllNotes() ([]Note, error) {
var notes []Note
@@ -174,6 +189,36 @@ func (ds *DataStore) GetTopBirdsData(selectedDate string, minConfidenceNormalize
return results, err
}

type ClipForRemoval struct {
ID string
ScientificName string
ClipName string
NumRecordings int
}

// GetClipsQualifyingForRemoval returns the list of clips that qualify for removal based on retention policy.
func (ds *DataStore) GetClipsQualifyingForRemoval(minHours int, minClips int) ([]ClipForRemoval, error) {

if minHours <= 0 || minClips <= 0 {
return []ClipForRemoval{}, nil
}

var results []ClipForRemoval

subquery := ds.DB.Model(&Note{}).
Select("ID, scientific_name, ROW_NUMBER () OVER ( PARTITION BY scientific_name ) num_recordings").
Where("clip_name != ''")

ds.DB.Model(&Note{}).
Select("n.ID, n.scientific_name, n.clip_name, sub.num_recordings").
Joins("n INNER JOIN (?) AS sub ON n.ID = sub.ID", subquery).
Where("(strftime('%s', 'now') - strftime('%s', begin_time)) / 3600 > ?", minHours).
Where("sub.num_recordings > ?", minClips).
Scan(&results)

return results, nil
}

// GetHourFormat returns the database-specific SQL fragment for formatting a time column as hour.
func (ds *DataStore) GetHourFormat() string {
// Handling for supported databases: SQLite and MySQL
88 changes: 88 additions & 0 deletions internal/datastore/interfaces_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package datastore

import (
"testing"
"time"

"github.com/tphakala/birdnet-go/internal/conf"
"github.com/tphakala/birdnet-go/internal/logger"
)

func createDatabase(t testing.TB, settings *conf.Settings) Interface {
tempDir := t.TempDir()
settings.Output.SQLite.Enabled = true
settings.Output.SQLite.Path = tempDir + "/test.db"

dataStore := New(settings)

// Open a connection to the database and handle possible errors.
if err := dataStore.Open(); err != nil {
logger.Error("main", "Failed to open database: %v", err)
} else {
t.Cleanup(func() { dataStore.Close() })
}

return dataStore
}

func TestGetClipsQualifyingForRemoval(t *testing.T) {

settings := &conf.Settings{}

dataStore := createDatabase(t, settings)

// One Cool bird should be removed since there is one too many
// No Amazing bird should be removed since there is only one
// While there are two Wonderful birds, only one of them are old enough, but too few to be removed
// While there are two Magnificent birds, only one of them have a clip, meaning that the remaining one should be kept
dataStore.Save(&Note{

Check failure on line 38 in internal/datastore/interfaces_test.go

GitHub Actions / lint

Error return value of `dataStore.Save` is not checked (errcheck)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Error return value of dataStore.Save is not checked. Consider handling this to avoid uncaught errors during tests.

- dataStore.Save(&Note{...}, []Results{})
+ if err := dataStore.Save(&Note{...}, []Results{}); err != nil {
+     t.Fatalf("Failed to save note: %v", err)
+ }

Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation.

Suggested change
dataStore.Save(&Note{
if err := dataStore.Save(&Note{

ClipName: "test.wav",
ScientificName: "Cool bird",
BeginTime: time.Now().Add(-2 * time.Hour),
}, []Results{})
dataStore.Save(&Note{

Check failure on line 43 in internal/datastore/interfaces_test.go

GitHub Actions / lint

Error return value of `dataStore.Save` is not checked (errcheck)
ClipName: "test2.wav",
ScientificName: "Amazing bird",
BeginTime: time.Now().Add(-2 * time.Hour),
}, []Results{})
dataStore.Save(&Note{

Check failure on line 48 in internal/datastore/interfaces_test.go

GitHub Actions / lint

Error return value of `dataStore.Save` is not checked (errcheck)
ClipName: "test3.wav",
ScientificName: "Cool bird",
BeginTime: time.Now().Add(-2 * time.Hour),
}, []Results{})
dataStore.Save(&Note{
ClipName: "test4.wav",
ScientificName: "Wonderful bird",
BeginTime: time.Now().Add(-2 * time.Hour),
}, []Results{})
dataStore.Save(&Note{
ClipName: "test5.wav",
ScientificName: "Magnificent bird",
BeginTime: time.Now().Add(-2 * time.Hour),
}, []Results{})
dataStore.Save(&Note{
ClipName: "",
ScientificName: "Magnificent bird",
BeginTime: time.Now(),
}, []Results{})
dataStore.Save(&Note{
ClipName: "test7.wav",
ScientificName: "Wonderful bird",
BeginTime: time.Now(),
}, []Results{})

minHours := 1
minClips := 1

clipsForRemoval, err := dataStore.GetClipsQualifyingForRemoval(minHours, minClips)

if err != nil {
t.Errorf("Expected no error, got %v", err)
}
if len(clipsForRemoval) != 1 {
t.Errorf("Expected one entry in clipsForRemoval, got %d", len(clipsForRemoval))
}
if clipsForRemoval[0].ScientificName != "Cool bird" {
t.Errorf("Expected ScientificName to be 'Cool bird', got '%s'", clipsForRemoval[0].ScientificName)
}
}