Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherrypick/v0.3.3-RC6 #2702

Merged
merged 6 commits into from
Oct 23, 2023
Merged

Cherrypick/v0.3.3-RC6 #2702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
778d687
Add UpdateGER entry to stream (#2695)
ToniRamirezM Oct 20, 2023
cc7fbc2
start metrics http server before node components (#2697)
agnusmor Oct 20, 2023
ab018eb
start streamserver before update the file (#2698)
agnusmor Oct 20, 2023
97243c5
add entries decode (#2699)
ToniRamirezM Oct 21, 2023
d6aa1d0
update to streamer v0.1.2
ToniRamirezM Oct 21, 2023
42aa672
Fix/update ger open batch (#2701)
ToniRamirezM Oct 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/0xPolygonHermez/zkevm-node
go 1.21

require (
github.com/0xPolygonHermez/zkevm-data-streamer v0.1.0
github.com/0xPolygonHermez/zkevm-data-streamer v0.1.3
github.com/didip/tollbooth/v6 v6.1.2
github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127
github.com/ethereum/go-ethereum v1.13.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/0xPolygonHermez/zkevm-data-streamer v0.1.0 h1:fYsPST5nwgE+IAQMa53kk1jgGJcKwUAmtOd4gABm92I=
github.com/0xPolygonHermez/zkevm-data-streamer v0.1.0/go.mod h1:853hkExOKPpMqbyEGqF8cuobSlbVd2OLVEZoRFTGRzo=
github.com/0xPolygonHermez/zkevm-data-streamer v0.1.3 h1:J4R0hBFmcPFf3SeeATgM0CDPOIhYU85DxhgRXAjorWg=
github.com/0xPolygonHermez/zkevm-data-streamer v0.1.3/go.mod h1:853hkExOKPpMqbyEGqF8cuobSlbVd2OLVEZoRFTGRzo=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
Expand Down
3 changes: 2 additions & 1 deletion sequencer/interfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ type stateInterface interface {
GetStoredFlushID(ctx context.Context) (uint64, string, error)
GetForkIDByBatchNumber(batchNumber uint64) uint64
GetDSGenesisBlock(ctx context.Context, dbTx pgx.Tx) (*state.DSL2Block, error)
GetDSL2Blocks(ctx context.Context, limit, offset uint64, dbTx pgx.Tx) ([]*state.DSL2Block, error)
GetDSBatch(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.DSBatch, error)
GetDSL2Blocks(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]*state.DSL2Block, error)
GetDSL2Transactions(ctx context.Context, minL2Block, maxL2Block uint64, dbTx pgx.Tx) ([]*state.DSL2Transaction, error)
}

Expand Down
44 changes: 35 additions & 9 deletions sequencer/mock_state.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

220 changes: 5 additions & 215 deletions sequencer/sequencer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,8 @@ package sequencer

import (
"context"
"encoding/binary"
"errors"
"fmt"
"reflect"
"time"

"github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer"
Expand Down Expand Up @@ -94,34 +92,13 @@ func (s *Sequencer) Start(ctx context.Context) {
log.Fatalf("failed to create stream server, err: %v", err)
}

// Set entities definition
entriesDefinition := map[datastreamer.EntryType]datastreamer.EntityDefinition{
state.EntryTypeL2BlockStart: {
Name: "L2BlockStart",
StreamType: state.StreamTypeSequencer,
Definition: reflect.TypeOf(state.DSL2BlockStart{}),
},
state.EntryTypeL2Tx: {
Name: "L2Transaction",
StreamType: state.StreamTypeSequencer,
Definition: reflect.TypeOf(state.DSL2Transaction{}),
},
state.EntryTypeL2BlockEnd: {
Name: "L2BlockEnd",
StreamType: state.StreamTypeSequencer,
Definition: reflect.TypeOf(state.DSL2BlockEnd{}),
},
}

streamServer.SetEntriesDef(entriesDefinition)

s.updateDataStreamerFile(ctx, &streamServer)

dbManager.streamServer = &streamServer
err = dbManager.streamServer.Start()
if err != nil {
log.Fatalf("failed to start stream server, err: %v", err)
}

s.updateDataStreamerFile(ctx, &streamServer)
}

go dbManager.Start()
Expand Down Expand Up @@ -159,197 +136,10 @@ func (s *Sequencer) Start(ctx context.Context) {
}

func (s *Sequencer) updateDataStreamerFile(ctx context.Context, streamServer *datastreamer.StreamServer) {
var skipBookMark, skipL2BlockStart bool
var currentL2Block uint64
var currentTxIndex uint64
var err error

header := streamServer.GetHeader()

if header.TotalEntries == 0 {
// Get Genesis block
genesisL2Block, err := s.state.GetDSGenesisBlock(ctx, nil)
if err != nil {
log.Fatal(err)
}

err = streamServer.StartAtomicOp()
if err != nil {
log.Fatal(err)
}

bookMark := state.DSBookMark{
Type: state.BookMarkTypeL2Block,
L2BlockNumber: genesisL2Block.L2BlockNumber,
}

_, err = streamServer.AddStreamBookmark(bookMark.Encode())
if err != nil {
log.Fatal(err)
}

genesisBlock := state.DSL2BlockStart{
BatchNumber: genesisL2Block.BatchNumber,
L2BlockNumber: genesisL2Block.L2BlockNumber,
Timestamp: genesisL2Block.Timestamp,
GlobalExitRoot: genesisL2Block.GlobalExitRoot,
Coinbase: genesisL2Block.Coinbase,
ForkID: genesisL2Block.ForkID,
}

log.Infof("Genesis block: %+v", genesisBlock)

_, err = streamServer.AddStreamEntry(1, genesisBlock.Encode())
if err != nil {
log.Fatal(err)
}

genesisBlockEnd := state.DSL2BlockEnd{
L2BlockNumber: genesisL2Block.L2BlockNumber,
BlockHash: genesisL2Block.BlockHash,
StateRoot: genesisL2Block.StateRoot,
}

_, err = streamServer.AddStreamEntry(state.EntryTypeL2BlockEnd, genesisBlockEnd.Encode())
if err != nil {
log.Fatal(err)
}

err = streamServer.CommitAtomicOp()
if err != nil {
log.Fatal(err)
}
} else {
latestEntry, err := streamServer.GetEntry(header.TotalEntries - 1)
if err != nil {
log.Fatal(err)
}

log.Infof("Latest entry: %+v", latestEntry)

switch latestEntry.Type {
case state.EntryTypeBookMark:
log.Info("Latest entry type is BookMark")
currentL2Block = binary.LittleEndian.Uint64(latestEntry.Data[1:9])
skipBookMark = true
currentL2Block--
case state.EntryTypeL2BlockStart:
log.Info("Latest entry type is L2BlockStart")
currentL2Block = binary.LittleEndian.Uint64(latestEntry.Data[8:16])
skipBookMark = true
skipL2BlockStart = true
currentL2Block--
case state.EntryTypeL2Tx:
log.Info("Latest entry type is L2Tx")
for latestEntry.Type == state.EntryTypeL2Tx {
currentTxIndex++
latestEntry, err = streamServer.GetEntry(header.TotalEntries - currentTxIndex)
if err != nil {
log.Fatal(err)
}
}
if latestEntry.Type != state.EntryTypeL2BlockStart {
log.Fatal("Latest entry is not a L2BlockStart")
}
currentL2Block = binary.LittleEndian.Uint64(latestEntry.Data[8:16])
currentL2Block--
case state.EntryTypeL2BlockEnd:
log.Info("Latest entry type is L2BlockEnd")
currentL2Block = binary.LittleEndian.Uint64(latestEntry.Data[0:8])
}
}

log.Infof("Current transaction index: %d", currentTxIndex)
log.Infof("Current L2 block number: %d", currentL2Block)

var limit uint64 = 1000
var offset uint64 = currentL2Block
var entry uint64 = header.TotalEntries
var l2blocks []*state.DSL2Block

if entry > 0 {
entry--
}

for err == nil {
log.Infof("Current entry number: %d", entry)

l2blocks, err = s.state.GetDSL2Blocks(ctx, limit, offset, nil)
offset += limit
if len(l2blocks) == 0 {
break
}
// Get transactions for all the retrieved l2 blocks
l2Transactions, err := s.state.GetDSL2Transactions(ctx, l2blocks[0].L2BlockNumber, l2blocks[len(l2blocks)-1].L2BlockNumber, nil)
if err != nil {
log.Fatal(err)
}

err = streamServer.StartAtomicOp()
if err != nil {
log.Fatal(err)
}

for x, l2block := range l2blocks {
if currentTxIndex > 0 {
x += int(currentTxIndex)
currentTxIndex = 0
}

bookMark := state.DSBookMark{
Type: state.BookMarkTypeL2Block,
L2BlockNumber: l2block.L2BlockNumber,
}

if !skipBookMark {
_, err = streamServer.AddStreamBookmark(bookMark.Encode())
if err != nil {
log.Fatal(err)
}
} else {
skipBookMark = false
}

blockStart := state.DSL2BlockStart{
BatchNumber: l2block.BatchNumber,
L2BlockNumber: l2block.L2BlockNumber,
Timestamp: l2block.Timestamp,
GlobalExitRoot: l2block.GlobalExitRoot,
Coinbase: l2block.Coinbase,
ForkID: l2block.ForkID,
}

if !skipL2BlockStart {
_, err = streamServer.AddStreamEntry(state.EntryTypeL2BlockStart, blockStart.Encode())
if err != nil {
log.Fatal(err)
}
} else {
skipL2BlockStart = false
}

entry, err = streamServer.AddStreamEntry(state.EntryTypeL2Tx, l2Transactions[x].Encode())
if err != nil {
log.Fatal(err)
}

blockEnd := state.DSL2BlockEnd{
L2BlockNumber: l2block.L2BlockNumber,
BlockHash: l2block.BlockHash,
StateRoot: l2block.StateRoot,
}

_, err = streamServer.AddStreamEntry(state.EntryTypeL2BlockEnd, blockEnd.Encode())
if err != nil {
log.Fatal(err)
}
}
err = streamServer.CommitAtomicOp()
if err != nil {
log.Fatal(err)
}
err := state.GenerateDataStreamerFile(ctx, streamServer, s.state)
if err != nil {
log.Fatalf("failed to generate data streamer file, err: %v", err)
}

log.Info("Data streamer file updated")
}

Expand Down
Loading