Merge pull request #3901 from oasisprotocol/kostko/stable/21.1.x/back…

…port-multi-2

[BACKPORT/21.1.x] Minor runtime worker fixes

.changelog/3889.bugfix.md

@@ -0,0 +1,7 @@
+Runtime workers should wait for history reindex
+
+If a node first syncs consensus without any runtimes configured and a runtime
+is configured later, the workers should wait for historic runtime block
+reindexing to complete before continuing with initialization. Otherwise
+historic block queries may fail and prevent the worker from operating
+normally.

.changelog/3899.bugfix.md

@@ -0,0 +1 @@
+worker/storage: Fix heartbeat stall

go/worker/common/committee/node.go

@@ -413,9 +413,7 @@ func (n *Node) worker() {
 	}
 	defer nodeUpsSub.Close()
 
-	// We are initialized.
-	close(n.initCh)
-
+	initialized := false
 	for {
 		select {
 		case <-n.stopCh:
@@ -431,6 +429,13 @@ func (n *Node) worker() {
 				n.Height = blk.Height
 			}()
 		case blk := <-blocks:
+			// We are initialized after we have received the first block. This makes sure that any
+			// history reindexing has been completed.
+			if !initialized {
+				close(n.initCh)
+				initialized = true
+			}
+
 			// Received a block (annotated).
 			func() {
 				n.CrossNode.Lock()

go/worker/storage/committee/utils.go

@@ -97,5 +97,10 @@ func (h *heartbeat) reset() {
 
 	boff := backoff.NewExponentialBackOff()
 	boff.InitialInterval = 5 * time.Second
+	boff.MaxElapsedTime = 0
+	boff.MaxInterval = 20 * time.Second
 	h.Ticker = backoff.NewTicker(boff)
+
+	// Gobble the first tick, which is immediate.
+	<-h.C
 }