integrate direct data onboarding (nv22) (#1847)

* enable ddo, lock lotus deps, fix APIs

* bump up deps - feat/nv22

* update deps, change inmort command

* fix cborgen

* remove unused fmt

* recreate itest

* fix circleCI

* bump go version

* undo machine type change

* shorten test

* change ubuntu to default

* bump up linter version

* check for claim

* bump to lotus-v1.26.0-rc1

---------

Co-authored-by: LexLuthr <[email protected]>

.circleci/config.yml

@@ -6,11 +6,11 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.20.7
+      - image: cimg/go:1.21.7
     resource_class: 2xlarge
   ubuntu:
     docker:
-      - image: ubuntu:22.04
+      - image: ubuntu:default
 
 commands:
   install-deps:
@@ -87,7 +87,7 @@ jobs:
   lid-docker-compose:
     description: 'Run LID integration tests'
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: default
       resource_class: xlarge
     steps:
       - checkout
@@ -132,7 +132,7 @@ jobs:
       # local index directory tests so it may be worth refactoring so that it's
       # only included for local index directory
       - setup_remote_docker:
-          version: 20.10.23
+          version: default
           docker_layer_caching: true
       - run:
           name: go test
@@ -154,7 +154,7 @@ jobs:
           linux: false
           darwin: true
       - golang/install:
-          version: "1.20.7"
+          version: "1.21.7"
       - run:
           name: Install pkg-config
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
@@ -253,7 +253,7 @@ jobs:
         default: golang
       golangci-lint-version:
         type: string
-        default: 1.52.2
+        default: 1.56.2
       concurrency:
         type: string
         default: '2'
@@ -328,10 +328,10 @@ workflows:
           suite: itest-retrieval
           target: "./itests/graphsync_retrieval_test.go"
 
-      #- test:
-          #name: test-itest-direct_deal
-          #suite: itest-direct_deal
-          #target: "./itests/ddo_test.go"
+      - test:
+          name: test-itest-direct_deal
+          suite: itest-direct_deal
+          target: "./itests/ddo_test.go"
 
       - test:
           name: test-all

api/api_full.go

@@ -200,7 +200,45 @@ type MarketBalance struct {
 
 type MarketDeal struct {
 	Proposal market.DealProposal
-	State    market.DealState
+	State    MarketDealState
+}
+
+type MarketDealState struct {
+	SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector
+	LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated
+	SlashEpoch       abi.ChainEpoch // -1 if deal never slashed
+}
+
+func MakeDealState(mds market.DealState) MarketDealState {
+	return MarketDealState{
+		SectorStartEpoch: mds.SectorStartEpoch(),
+		LastUpdatedEpoch: mds.LastUpdatedEpoch(),
+		SlashEpoch:       mds.SlashEpoch(),
+	}
+}
+
+type mstate struct {
+	s MarketDealState
+}
+
+func (m mstate) SectorStartEpoch() abi.ChainEpoch {
+	return m.s.SectorStartEpoch
+}
+
+func (m mstate) LastUpdatedEpoch() abi.ChainEpoch {
+	return m.s.LastUpdatedEpoch
+}
+
+func (m mstate) SlashEpoch() abi.ChainEpoch {
+	return m.s.SlashEpoch
+}
+
+func (m mstate) Equals(o market.DealState) bool {
+	return market.DealStatesEqual(m, o)
+}
+
+func (m MarketDealState) Iface() market.DealState {
+	return mstate{m}
 }
 
 type InvocResult struct {

cmd/boost/main.go

@@ -40,9 +40,8 @@ func main() {
 			offlineDealCmd,
 			providerCmd,
 			walletCmd,
-			//TODO: enable when DDO ends up in a network upgrade
-			//directDealAllocate,
-			//directDealGetAllocations,
+			directDealAllocate,
+			directDealGetAllocations,
 		},
 	}
 	app.Setup()

cmd/boost/util/util.god → cmd/boost/util/util.go

@@ -3,20 +3,21 @@ package util
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/builtin"
-	"github.com/filecoin-project/go-state-types/builtin/v9/datacap"
-	"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
+	"github.com/filecoin-project/go-state-types/builtin/v13/datacap"
+	"github.com/filecoin-project/go-state-types/builtin/v13/verifreg"
 	"github.com/filecoin-project/lotus/api"
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	datacap2 "github.com/filecoin-project/lotus/chain/actors/builtin/datacap"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/ipfs/go-cid"
-	"strconv"
-	"strings"
 )
 
 func CreateAllocationMsg(ctx context.Context, api api.Gateway, pInfos, miners []string, wallet address.Address, tmin, tmax, exp abi.ChainEpoch) (*types.Message, error) {

cmd/boostd/import_direct_data.god → cmd/boostd/import_direct_data.go

@@ -9,7 +9,9 @@ import (
 	"github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin"
 	"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
+	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"github.com/mitchellh/go-homedir"
@@ -48,20 +50,16 @@ var importDirectDataCmd = &cli.Command{
 		},
 		&cli.IntFlag{
 			Name:  "start-epoch",
-			Usage: "start epoch by when the deal should be proved by provider on-chain",
-			Value: 35000, // default is 35000, handy for tests with 2k/devnet build
-		},
-		&cli.IntFlag{
-			Name:  "duration",
-			Usage: "duration of the deal in epochs",
-			Value: 518400, // default is 2880 * 180 == 180 days
+			Usage: "start epoch by when the deal should be proved by provider on-chain (default: 2 days from now)",
 		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() < 2 {
 			return fmt.Errorf("must specify piececid and file path")
 		}
 
+		ctx := cctx.Context
+
 		piececidStr := cctx.Args().Get(0)
 		path := cctx.Args().Get(1)
 
@@ -91,6 +89,17 @@ var importDirectDataCmd = &cli.Command{
 		}
 		defer closer()
 
+		lapi, lcloser, err := lcli.GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return err
+		}
+		defer lcloser()
+
+		head, err := lapi.ChainHead(ctx)
+		if err != nil {
+			return fmt.Errorf("getting chain head: %w", err)
+		}
+
 		clientAddr, err := address.NewFromString(cctx.String("client-addr"))
 		if err != nil {
 			return fmt.Errorf("failed to parse clientaddr param: %w", err)
@@ -99,7 +108,21 @@ var importDirectDataCmd = &cli.Command{
 		allocationId := cctx.Uint64("allocation-id")
 
 		startEpoch := abi.ChainEpoch(cctx.Int("start-epoch"))
-		endEpoch := startEpoch + abi.ChainEpoch(cctx.Int("duration"))
+		// Set Default if not specified by the user
+		if startEpoch == 0 {
+			startEpoch = head.Height() + (builtin.EpochsInDay * 2)
+		}
+		alloc, err := lapi.StateGetAllocation(ctx, clientAddr, verifreg.AllocationId(allocationId), head.Key())
+		if err != nil {
+			return fmt.Errorf("getting claim details from chain: %w", err)
+		}
+
+		if alloc.Expiration < startEpoch {
+			return fmt.Errorf("allocation will expire on %d before start epoch %d", alloc.Expiration, startEpoch)
+		}
+
+		// Since StartEpoch is more than Head+StartEpochSealingBuffer, we can set end epoch as start+TermMin
+		endEpoch := startEpoch + alloc.TermMin
 
 		ddParams := types.DirectDealParams{
 			DealUUID:           uuid.New(),

cmd/boostd/main.go

@@ -42,7 +42,7 @@ func main() {
 			dummydealCmd,
 			indexProvCmd,
 			importDataCmd,
-			//importDirectDataCmd,
+			importDirectDataCmd,
 			logCmd,
 			netCmd,
 			pieceDirCmd,

cmd/lib/api.go

@@ -151,23 +151,23 @@ func (a *MultiMinerAccessor) GetMinerAddresses() []address.Address {
 func (a *MultiMinerAccessor) GetReader(ctx context.Context, minerAddr address.Address, id abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize) (types.SectionReader, error) {
 	pr, ok := a.readers[minerAddr]
 	if !ok {
-		return nil, fmt.Errorf("get reader: no endpoint registered for miner %s", minerAddr)
+		return nil, fmt.Errorf("get reader: no endpoint registered for miner %s, len(readers)=%d", minerAddr, len(a.readers))
 	}
 	return pr.GetReader(ctx, minerAddr, id, offset, length)
 }
 
 func (a *MultiMinerAccessor) UnsealSectorAt(ctx context.Context, minerAddr address.Address, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) {
 	sa, ok := a.sas[minerAddr]
 	if !ok {
-		return nil, fmt.Errorf("read sector: no endpoint registered for miner %s", minerAddr)
+		return nil, fmt.Errorf("read sector: no endpoint registered for miner %s, len(readers)=%d", minerAddr, len(a.readers))
 	}
 	return sa.UnsealSectorAt(ctx, sectorID, pieceOffset, length)
 }
 
 func (a *MultiMinerAccessor) IsUnsealed(ctx context.Context, minerAddr address.Address, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) {
 	sa, ok := a.sas[minerAddr]
 	if !ok {
-		return false, fmt.Errorf("is unsealed: no endpoint registered for miner %s", minerAddr)
+		return false, fmt.Errorf("is unsealed: no endpoint registered for miner %s, len(readers)=%d", minerAddr, len(a.readers))
 	}
 	return sa.IsUnsealed(ctx, sectorID, offset, length)
 }

docker/devnet/lotus-miner/Dockerfile

@@ -21,6 +21,7 @@ ENV SECTOR_SIZE=8388608
 
 VOLUME /var/tmp/filecoin-proof-parameters
 VOLUME /var/lib/genesis
+VOLUME /var/lib/builtin-actors
 
 WORKDIR /app
 RUN mkdir -p /app

docker/devnet/lotus/Dockerfile

@@ -23,6 +23,7 @@ ENV LOTUS_FEVM_ENABLEETHRPC=true
 
 VOLUME /var/tmp/filecoin-proof-parameters
 VOLUME /var/lib/genesis
+VOLUME /var/lib/builtin-actors
 
 WORKDIR /app
 RUN mkdir -p /app