Skip to content

Commit

Permalink
fix: remove boost-gfm and go-data-transfer (#1740)
Browse files Browse the repository at this point in the history 
* remove gfm, pull datatransfer

* make gen, docsgen

* fix circular depndencies

* go mod tidy

* fix lint errors

* convert RetrievalAskGetter to interface

* fix interface pointer

* fix dependency injection, move ask db

* fix itests

* clean up mocks

* chore: lots more dt & gs debugging (#1776)

* fix lint errs, add back ProviderDataTransfer

* regenrate cbor

* cleanup go-data-transfer

* fix graphsync retrievals

* use new client for GS tests

* fix selector type

* restore selector type

* fix: clean up and fix retrieval framework & tests (#1848)

* fix: clean up and fix retrieval framework & tests

* Use new go-trustless-utils primitives and traversal tools
* Simplify Framework#Retrieve

* fix: wait for full startup of data transfer when making client

* fix lint errors

* fix go mod

* cleanup itest fixtures, fix gql

* go mod tidy

* fix directDealProv

* handle storage ask

* cleanup config

* remove graphsync unit tests

* add retrieval config headers

* update retrieval configs

* fix storage ask db command

* fix BoostLegacyDealByProposalCid api

* make cbor-gen

* fix missing api example

* migrate config, dump dagstore config

---------

Co-authored-by: Rod Vagg <[email protected]>
  • Loading branch information
@LexLuthr @rvagg
LexLuthr and rvagg authored Feb 19, 2024
1 parent e552e7c commit 3b49cd1
Show file tree
Hide file tree
Showing 214 changed files with 32,333 additions and 11,501 deletions.
27 changes: 6 additions & 21 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,29 +319,14 @@ workflows:
target: "./itests/dummydeal_test.go"

- test:
name: test-itest-markets_v1_deal
suite: itest-markets_v1_deal
target: "./itests/markets_v1_deal_test.go"
name: test-graphsync_identity_cid
suite: itest-graphsync_identity_cid
target: "./itests/graphsync_identity_cid_test.go"

- test:
name: test-itest-markets_v1_identity_cid
suite: itest-markets_v1_identity_cid
target: "./itests/markets_v1_identity_cid_test.go"

- test:
name: test-itest-markets_v1_offline_deal
suite: itest-markets_v1_offline_deal
target: "./itests/markets_v1_offline_deal_test.go"

- test:
name: test-itest-markets_v1_retrieval
suite: itest-markets_v1_retrieval
target: "./itests/markets_v1_retrieval_test.go"

- test:
name: test-itest-disabled_markets_v1_deal
suite: itest-disabled_markets_v1_deal
target: "./itests/disabled_markets_v1_deal_test.go"
name: test-itest-retrieval
suite: itest-retrieval
target: "./itests/graphsync_retrieval_test.go"

#- test:
#name: test-itest-direct_deal
Expand Down
90 changes: 3 additions & 87 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,10 @@ package api
import (
"context"

"github.com/filecoin-project/boost-gfm/retrievalmarket"
"github.com/filecoin-project/boost-gfm/storagemarket"
smtypes "github.com/filecoin-project/boost/storagemarket/types"
"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/boost/storagemarket/types/legacytypes"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multihash"
)

Expand Down Expand Up @@ -42,17 +35,9 @@ type Boost interface {
BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin
BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin
BoostIndexerAnnounceDealRemoved(ctx context.Context, propCid cid.Cid) (cid.Cid, error) //perm:admin
BoostLegacyDealByProposalCid(ctx context.Context, propCid cid.Cid) (legacytypes.MinerDeal, error) //perm:admin
BoostIndexerAnnounceDeal(ctx context.Context, deal *smtypes.ProviderDealState) (cid.Cid, error) //perm:admin
BoostIndexerAnnounceLegacyDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
BoostLegacyDealByProposalCid(ctx context.Context, propCid cid.Cid) (storagemarket.MinerDeal, error) //perm:admin
BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin
BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read
BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin
BoostIndexerAnnounceLegacyDeal(ctx context.Context, proposalCid cid.Cid) (cid.Cid, error) //perm:admin
BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write
BoostDirectDeal(ctx context.Context, params smtypes.DirectDealParams) (*ProviderDealRejectionInfo, error) //perm:admin

Expand All @@ -64,75 +49,6 @@ type Boost interface {
// MethodGroup: PieceDirectory
PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin

// RuntimeSubsystems returns the subsystems that are enabled
// in this instance.
RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read

// MethodGroup: LegacyMarket
MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) //perm:read
MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error //perm:admin
MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error) //perm:read
MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error) //perm:read
MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) //perm:write
MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) //perm:write
MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) //perm:read
MarketPendingDeals(ctx context.Context) (lapi.PendingDealInfo, error) //perm:write
SectorsRefs(context.Context) (map[string][]lapi.SealedRef, error) //perm:read

// MethodGroup: Actor
ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read

// MethodGroup: Deals
DealsConsiderOnlineStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderOnlineStorageDeals(context.Context, bool) error //perm:admin
DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error //perm:admin
DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error) //perm:admin
DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error //perm:admin
DealsConsiderOfflineStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderOfflineStorageDeals(context.Context, bool) error //perm:admin
DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error //perm:admin
DealsConsiderVerifiedStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error //perm:admin
DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error //perm:admin

// MethodGroup: Misc
OnlineBackup(context.Context, string) error //perm:admin
}

// DagstoreShardInfo is the serialized form of dagstore.DagstoreShardInfo that
// we expose through JSON-RPC to avoid clients having to depend on the
// dagstore lib.
type DagstoreShardInfo struct {
Key string
State string
Error string
}

// DagstoreShardResult enumerates results per shard.
type DagstoreShardResult struct {
Key string
Success bool
Error string
}

type DagstoreInitializeAllParams struct {
MaxConcurrency int
IncludeSealed bool
}

// DagstoreInitializeAllEvent represents an initialization event.
type DagstoreInitializeAllEvent struct {
Key string
Event string // "start", "end"
Success bool
Error string
Total int
Current int
}
80 changes: 1 addition & 79 deletions api/api_full.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,11 @@ import (
"fmt"
"time"

"github.com/ipfs/go-cid"
textselector "github.com/ipld/go-ipld-selector-text-lite"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"

"github.com/filecoin-project/boost-gfm/retrievalmarket"
"github.com/filecoin-project/boost-gfm/storagemarket"
"github.com/ipfs/go-cid"

"github.com/filecoin-project/go-state-types/builtin/v8/paych"
"github.com/filecoin-project/lotus/chain/actors/builtin"
Expand Down Expand Up @@ -89,29 +83,6 @@ type Import struct {
CARPath string
}

type DealInfo struct {
ProposalCid cid.Cid
State storagemarket.StorageDealStatus
Message string // more information about deal state, particularly errors
DealStages *storagemarket.DealStages
Provider address.Address

DataRef *storagemarket.DataRef
PieceCID cid.Cid
Size uint64

PricePerEpoch types.BigInt
Duration uint64

DealID abi.DealID

CreationTime time.Time
Verified bool

TransferChannelID *datatransfer.ChannelID
DataTransfer *DataTransferChannel
}

type MsgLookup struct {
Message cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed
Receipt types.MessageReceipt
Expand Down Expand Up @@ -222,37 +193,6 @@ type MinerPower struct {
HasMinPower bool
}

type QueryOffer struct {
Err string

Root cid.Cid
Piece *cid.Cid

Size uint64
MinPrice types.BigInt
UnsealPrice types.BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
Miner address.Address
MinerPeer retrievalmarket.RetrievalPeer
}

func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
return RetrievalOrder{
Root: o.Root,
Piece: o.Piece,
Size: o.Size,
Total: o.MinPrice,
UnsealPrice: o.UnsealPrice,
PaymentInterval: o.PaymentInterval,
PaymentIntervalIncrease: o.PaymentIntervalIncrease,
Client: client,

Miner: o.Miner,
MinerPeer: &o.MinerPeer,
}
}

type MarketBalance struct {
Escrow big.Int
Locked big.Int
Expand All @@ -263,24 +203,6 @@ type MarketDeal struct {
State market.DealState
}

type RetrievalOrder struct {
// TODO: make this less unixfs specific
Root cid.Cid
Piece *cid.Cid
DatamodelPathSelector *textselector.Expression
Size uint64

FromLocalCAR string // if specified, get data from a local CARv2 file.
// TODO: support offset
Total types.BigInt
UnsealPrice types.BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
Client address.Address
Miner address.Address
MinerPeer *retrievalmarket.RetrievalPeer
}

type InvocResult struct {
MsgCid cid.Cid
Msg *types.Message
Expand Down
25 changes: 8 additions & 17 deletions api/docgen/docgen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (
"time"
"unicode"

"github.com/filecoin-project/boost-gfm/filestore"
"github.com/filecoin-project/boost-gfm/retrievalmarket"
"github.com/filecoin-project/boost/datatransfer"
"github.com/filecoin-project/boost/retrievalmarket/types/legacyretrievaltypes"
"github.com/filecoin-project/boost/storagemarket/types/legacytypes/filestore"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
Expand Down Expand Up @@ -94,7 +94,7 @@ func init() {

storeIDExample := imports.ID(50)
textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash")
clientEvent := retrievalmarket.ClientEventDealAccepted
clientEvent := legacyretrievaltypes.ClientEventDealAccepted

addExample(bitfield.NewFromSet([]uint64{5}))
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
Expand Down Expand Up @@ -129,14 +129,15 @@ func init() {
addExample(&storeIDExample)
addExample(clientEvent)
addExample(&clientEvent)
addExample(retrievalmarket.ClientEventDealAccepted)
addExample(retrievalmarket.DealStatusNew)
addExample(legacyretrievaltypes.ClientEventDealAccepted)
addExample(legacyretrievaltypes.DealStatusNew)
addExample(&textSelExample)
addExample(network.ReachabilityPublic)
addExample(build.TestNetworkVersion)
allocationID := verifreg.AllocationId(0)
addExample(allocationID)
addExample(&allocationID)
addExample(filestore.Path(""))
addExample(map[string]int{"name": 42})
addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
addExample(&types.ExecutionTrace{
Expand Down Expand Up @@ -186,10 +187,9 @@ func init() {
ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr

// miner specific
addExample(filestore.Path(".lotusminer/fstmp123"))
si := uint64(12)
addExample(&si)
addExample(retrievalmarket.DealID(5))
addExample(legacyretrievaltypes.DealID(5))
addExample(abi.ActorID(1000))
addExample(storiface.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
addExample(storiface.FTUnsealed)
Expand Down Expand Up @@ -273,15 +273,6 @@ func init() {

addExample(api.CheckStatusCode(0))
addExample(map[string]interface{}{"abc": 123})
addExample(api.DagstoreShardResult{
Key: "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
Error: "<error>",
})
addExample(api.DagstoreShardInfo{
Key: "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
State: "ShardStateAvailable",
Error: "<error>",
})
addExample(storiface.ResourceTable)
addExample(network.ScopeStat{
Memory: 123,
Expand Down
Loading

0 comments on commit 3b49cd1

Please sign in to comment.