diff --git a/internal/pkg/queue/dequeue.go b/internal/pkg/queue/dequeue.go index 992ecc3d..ca2f5089 100644 --- a/internal/pkg/queue/dequeue.go +++ b/internal/pkg/queue/dequeue.go @@ -9,7 +9,7 @@ import ( // Dequeue removes and returns the next item from the queue // It blocks until an item is available -func (q *PersistentGroupedQueue) Dequeue() (*Item, error) { +func (q *PersistentGroupedQueue) Dequeue() (item *Item, err error) { if q.closed { return nil, ErrQueueClosed } @@ -104,7 +104,7 @@ func (q *PersistentGroupedQueue) Dequeue() (*Item, error) { return nil, fmt.Errorf("failed to save metadata: %w", err) } - return &item, nil + break } // If we've checked all hosts and found no items, return queue empty error diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index 8e6f909a..2eb3132f 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -57,16 +57,16 @@ type PersistentGroupedQueue struct { } type Item struct { + URL *url.URL + ParentItem *Item ID string - Hash uint64 - Hop uint8 Host string Type string + BypassSeencheck string + Hash uint64 Redirect int - URL *url.URL - ParentItem *Item LocallyCrawled uint64 - BypassSeencheck string + Hop uint8 } func init() { diff --git a/internal/pkg/queue/stats.go b/internal/pkg/queue/stats.go index 15c4f2df..d16945e8 100644 --- a/internal/pkg/queue/stats.go +++ b/internal/pkg/queue/stats.go @@ -6,19 +6,19 @@ import ( ) type QueueStats struct { - TotalElements int `json:"total_elements"` - UniqueHosts int `json:"unique_hosts"` - ElementsPerHost map[string]int `json:"elements_per_host"` - EnqueueCount int `json:"enqueue_count"` - DequeueCount int `json:"dequeue_count"` FirstEnqueueTime time.Time `json:"first_enqueue_time"` LastEnqueueTime time.Time `json:"last_enqueue_time"` FirstDequeueTime time.Time `json:"first_dequeue_time"` LastDequeueTime time.Time `json:"last_dequeue_time"` + ElementsPerHost map[string]int `json:"elements_per_host"` + HostDistribution map[string]float64 `json:"host_distribution"` + TopHosts []HostStat `json:"top_hosts"` + TotalElements int `json:"total_elements"` + UniqueHosts int `json:"unique_hosts"` + EnqueueCount int `json:"enqueue_count"` + DequeueCount int `json:"dequeue_count"` AverageTimeBetweenEnqueues time.Duration `json:"average_time_between_enqueues"` AverageTimeBetweenDequeues time.Duration `json:"average_time_between_dequeues"` - TopHosts []HostStat `json:"top_hosts"` - HostDistribution map[string]float64 `json:"host_distribution"` AverageElementsPerHost float64 `json:"average_elements_per_host"` Utilization float64 `json:"utilization"` }