Merge branch 'queue' of github.com:CorentinB/Zeno into queue

internetarchive · Jul 13, 2024 · 269b258 · 269b258
2 parents 5a00379 + 53e4a54
commit 269b258
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 14 deletions.
diff --git a/internal/pkg/queue/dequeue.go b/internal/pkg/queue/dequeue.go
@@ -9,7 +9,7 @@ import (
 
 // Dequeue removes and returns the next item from the queue
 // It blocks until an item is available
-func (q *PersistentGroupedQueue) Dequeue() (*Item, error) {
+func (q *PersistentGroupedQueue) Dequeue() (item *Item, err error) {
 	if q.closed {
 		return nil, ErrQueueClosed
 	}
@@ -104,7 +104,7 @@ func (q *PersistentGroupedQueue) Dequeue() (*Item, error) {
 			return nil, fmt.Errorf("failed to save metadata: %w", err)
 		}
 
-		return &item, nil
+		break
 	}
 
 	// If we've checked all hosts and found no items, return queue empty error

diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go
@@ -57,16 +57,16 @@ type PersistentGroupedQueue struct {
 }
 
 type Item struct {
+	URL             *url.URL
+	ParentItem      *Item
 	ID              string
-	Hash            uint64
-	Hop             uint8
 	Host            string
 	Type            string
+	BypassSeencheck string
+	Hash            uint64
 	Redirect        int
-	URL             *url.URL
-	ParentItem      *Item
 	LocallyCrawled  uint64
-	BypassSeencheck string
+	Hop             uint8
 }
 
 func init() {

diff --git a/internal/pkg/queue/stats.go b/internal/pkg/queue/stats.go
@@ -6,19 +6,19 @@ import (
 )
 
 type QueueStats struct {
-	TotalElements              int                `json:"total_elements"`
-	UniqueHosts                int                `json:"unique_hosts"`
-	ElementsPerHost            map[string]int     `json:"elements_per_host"`
-	EnqueueCount               int                `json:"enqueue_count"`
-	DequeueCount               int                `json:"dequeue_count"`
 	FirstEnqueueTime           time.Time          `json:"first_enqueue_time"`
 	LastEnqueueTime            time.Time          `json:"last_enqueue_time"`
 	FirstDequeueTime           time.Time          `json:"first_dequeue_time"`
 	LastDequeueTime            time.Time          `json:"last_dequeue_time"`
+	ElementsPerHost            map[string]int     `json:"elements_per_host"`
+	HostDistribution           map[string]float64 `json:"host_distribution"`
+	TopHosts                   []HostStat         `json:"top_hosts"`
+	TotalElements              int                `json:"total_elements"`
+	UniqueHosts                int                `json:"unique_hosts"`
+	EnqueueCount               int                `json:"enqueue_count"`
+	DequeueCount               int                `json:"dequeue_count"`
 	AverageTimeBetweenEnqueues time.Duration      `json:"average_time_between_enqueues"`
 	AverageTimeBetweenDequeues time.Duration      `json:"average_time_between_dequeues"`
-	TopHosts                   []HostStat         `json:"top_hosts"`
-	HostDistribution           map[string]float64 `json:"host_distribution"`
 	AverageElementsPerHost     float64            `json:"average_elements_per_host"`
 	Utilization                float64            `json:"utilization"`
 }