From 3a9f22e9761f0a1bd2992a3d9081ab4ab57fb29b Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 7 Aug 2024 23:24:45 +0400 Subject: [PATCH 01/15] basic mk12 scafolding --- cmd/curio/guidedsetup/shared_cbor_gen.go | 1 + deps/config/doc_gen.go | 1 - go.mod | 2 +- .../sql/20240730-market-migration.sql | 205 ++++++++ lib/indexing/indexstore/create.cql | 9 + lib/indexing/indexstore/indexstore.go | 359 ++++++++++++++ lib/libp2p/libp2p.go | 207 ++++++++ lib/paths/mocks/index.go | 4 +- lib/paths/mocks/pf.go | 3 +- lib/paths/mocks/store.go | 6 +- market/mk12/libp2pimpl/libp2pimpl.go | 449 ++++++++++++++++++ market/mk12/types.go | 2 + market/storageingest/deal_ingest_seal.go | 2 +- tasks/gc/pipeline_meta_gc.go | 3 + tasks/indexing/task_indexing.go | 5 +- tasks/seal/task_movestorage.go | 2 +- tasks/snap/task_movestorage.go | 2 +- 17 files changed, 1247 insertions(+), 15 deletions(-) create mode 100644 harmony/harmonydb/sql/20240730-market-migration.sql create mode 100644 lib/indexing/indexstore/create.cql create mode 100644 lib/indexing/indexstore/indexstore.go create mode 100644 lib/libp2p/libp2p.go create mode 100644 market/mk12/libp2pimpl/libp2pimpl.go diff --git a/cmd/curio/guidedsetup/shared_cbor_gen.go b/cmd/curio/guidedsetup/shared_cbor_gen.go index c88b9e31f..1f2effd07 100644 --- a/cmd/curio/guidedsetup/shared_cbor_gen.go +++ b/cmd/curio/guidedsetup/shared_cbor_gen.go @@ -10,6 +10,7 @@ import ( storiface "github.com/filecoin-project/curio/lib/storiface" sector "github.com/filecoin-project/curio/lib/types/sector" + abi "github.com/filecoin-project/go-state-types/abi" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 904de0000..664bc799e 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -712,7 +712,6 @@ also be bounded by resources available on the machine.`, { Name: "EnableDealMarket", Type: "bool", - Comment: `EnableDealMarket enabled the deal market on the node. This would also enable libp2p on the node, if configured.`, }, { diff --git a/go.mod b/go.mod index 4434ba23e..9b4381f72 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,6 @@ require ( go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.23.0 @@ -320,6 +319,7 @@ require ( go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/harmony/harmonydb/sql/20240730-market-migration.sql b/harmony/harmonydb/sql/20240730-market-migration.sql new file mode 100644 index 000000000..d936f9bde --- /dev/null +++ b/harmony/harmonydb/sql/20240730-market-migration.sql @@ -0,0 +1,205 @@ +CREATE TABLE market_mk12_deals ( + uuid TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', + signed_proposal_cid TEXT NOT NULL, + proposal_signature BYTEA NOT NULL, + proposal jsonb NOT NULL, + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, + offline BOOLEAN NOT NULL, + verified BOOLEAN NOT NULL, + sp_id BIGINT NOT NULL, + start_epoch BIGINT NOT NULL, + end_epoch BIGINT NOT NULL, + client_peer_id TEXT NOT NULL, + chain_deal_id BIGINT DEFAULT NULL, + publish_cid TEXT DEFAULT NULL, + length BIGINT DEFAULT NULL, + fast_retrieval BOOLEAN NOT NULL, + announce_to_ipni BOOLEAN NOT NULL, + url TEXT DEFAULT NULL, + url_headers jsonb NOT NULL DEFAULT '{}', + error TEXT DEFAULT NULL, + + primary key (uuid, sp_id, piece_cid, signed_proposal_cid), + unique (uuid), + unique (signed_proposal_cid) +); + +CREATE TABLE market_legacy_deals ( + signed_proposal_cid TEXT, + proposal_signature BYTEA, + proposal jsonb, + piece_cid TEXT, + piece_size BIGINT, + offline BOOLEAN, + verified BOOLEAN, + sp_id BIGINT, + start_epoch BIGINT, + end_epoch BIGINT, + publish_cid TEXT, + fast_retrieval BOOLEAN, + chain_deal_id BIGINT, + created_at TIMESTAMPTZ, + sector_num BIGINT, + + primary key (sp_id, piece_cid, signed_proposal_cid) +); + +CREATE TABLE market_piece_metadata ( + piece_cid TEXT NOT NULL PRIMARY KEY, + version INT NOT NULL DEFAULT 2, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', + indexed BOOLEAN NOT NULL DEFAULT FALSE, + indexed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', + + constraint market_piece_meta_identity_key + unique (piece_cid) +); + +CREATE TABLE market_piece_deal ( + id TEXT NOT NULL, + piece_cid TEXT NOT NULL, + boost_deal BOOLEAN NOT NULL, + legacy_deal BOOLEAN NOT NULL DEFAULT FALSE, + chain_deal_id BIGINT NOT NULL DEFAULT 0, + sp_id BIGINT NOT NULL, + sector_num BIGINT NOT NULL, + piece_offset BIGINT NOT NULL, + piece_length BIGINT NOT NULL, + raw_size BIGINT NOT NULL, + + primary key (sp_id, piece_cid, deal), + constraint market_piece_deal_identity_key + unique (sp_id, id) +); + +CREATE OR REPLACE FUNCTION process_piece_deal( + _id TEXT, + _piece_cid TEXT, + _boost_deal BOOLEAN, + _legacy_deal BOOLEAN DEFAULT FALSE, + _chain_deal_id BIGINT DEFAULT 0, + _sp_id BIGINT, + _sector_num BIGINT, + _piece_offset BIGINT, + _piece_length BIGINT, + _raw_size BIGINT +) +RETURNS VOID AS $$ +BEGIN + -- Update or insert into market_piece_metadata +INSERT INTO market_piece_metadata (piece_cid, indexed, indexed_at) +VALUES (_piece_cid, TRUE, CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + ON CONFLICT (piece_cid) DO UPDATE + SET indexed = TRUE, + indexed_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + +-- Insert into market_piece_deal +INSERT INTO market_piece_deal ( + id, piece_cid, boost_deal, legacy_deal, chain_deal_id, + sp_id, sector_num, piece_offset, piece_length, raw_size +) +VALUES ( + _id, _piece_cid, _boost_deal, _legacy_deal, _chain_deal_id, + _sp_id, _sector_num, _piece_offset, _piece_length, _raw_size + ) + ON CONFLICT (sp_id, piece_cid, id) DO NOTHING; +END; +$$ LANGUAGE plpgsql; + +CREATE TABLE market_mk12_storage_ask ( + price BIGINT NOT NULL, + verified_price BIGINT NOT NULL, + min_size BIGINT NOT NULL, + max_size BIGINT NOT NULL, + sp_id BIGINT NOT NULL, + created_at BIGINT NOT NULL, + expiry BIGINT NOT NULL, + sequence BIGINT NOT NULL, + unique (sp_id) +); + +CREATE TABLE market_mk12_deal_pipeline ( + uuid TEXT NOT NULL, + sp_id BIGINT NOT NULL, + started BOOLEAN DEFAULT FALSE, + piece_cid TEXT NOT NULL, + piece_size BOOLEAN NOT NULL, + offline BOOLEAN NOT NULL, + downloaded BOOLEAN DEFAULT FALSE, + url TEXT DEFAULT NULL, + headers jsonb NOT NULL DEFAULT '{}', + file_size BIGINT DEFAULT NULL, + commp_task_id BIGINT DEFAULT NULL, + after_commp BOOLEAN DEFAULT FALSE, + find_task_id BIGINT DEFAULT NULL, + after_find BOOLEAN DEFAULT FALSE, + psd_task_id BIGINT DEFAULT NULL, + after_psd BOOLEAN DEFAULT FALSE, + psd_wait_time TIMESTAMPTZ, + find_deal_task_id BIGINT DEFAULT NULL, + after_find_deal BOOLEAN DEFAULT FALSE, + sector BIGINT, + sector_offset BIGINT, + sealed BOOLEAN DEFAULT FALSE, + indexed BOOLEAN DEFAULT FALSE, + + constraint market_mk12_deal_pipeline_identity_key unique (uuid) +); + +CREATE TABLE market_offline_urls ( + piece_cid TEXT NOT NULL, + url TEXT NOT NULL, + headers jsonb NOT NULL DEFAULT '{}', + raw_size BIGINT NOT NULL, + unique (piece_cid) +); + +CREATE TABLE market_indexing_tasks ( + id TEXT NOT NULL, + sp_id BIGINT NOT NULL, + sector_number BIGINT NOT NULL, + reg_seal_proof INT NOT NULL, + piece_offset BIGINT NOT NULL, + piece_size BIGINT NOT NULL, + raw_size BIGINT NOT NULL, + piece_cid TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', + task_id BIGINT DEFAULT NULL, + + constraint market_indexing_tasks_identity_key + unique (id, sp_id, sector_number, piece_offset, piece_size, piece_cid, reg_seal_proof) +) + +CREATE TABLE libp2p_keys ( + sp_id BIGINT NOT NULL, + priv_key BYTEA NOT NULL, + listen_address TEXT NOT NULL, + announce_address TEXT NOT NULL, + no_announce_address TEXT NOT NULL, +); + +CREATE TABLE direct_deals ( + id TEXT, + created_at TIMESTAMPTZ, + piece_cid TEXT, + piece_size BIGINT, + cleanup_data BOOLEAN, + client_address TEXT, + provider_address TEXT, + allocation_id BIGINT, + start_epoch BIGINT, + end_epoch BIGINT, + inbound_file_path TEXT, + inbound_file_size BIGINT, + sector_id BIGINT, + offset BIGINT, + length BIGINT, + announce_to_ipni BOOLEAN, + keep_unsealed_copy BOOLEAN +); + + + + diff --git a/lib/indexing/indexstore/create.cql b/lib/indexing/indexstore/create.cql new file mode 100644 index 000000000..21157bb83 --- /dev/null +++ b/lib/indexing/indexstore/create.cql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS PayloadToPiece ( + PieceCid BLOB, + PayloadMultihash BLOB, + BlockOffset BIGINT, + BlockSize BIGINT, + PRIMARY KEY (PayloadMultihash, PieceCid) +); + +CREATE INDEX piece_to_payload ON PayloadToPiece(PieceCid, PayloadMultihash); \ No newline at end of file diff --git a/lib/indexing/indexstore/indexstore.go b/lib/indexing/indexstore/indexstore.go new file mode 100644 index 000000000..5af4a5ce0 --- /dev/null +++ b/lib/indexing/indexstore/indexstore.go @@ -0,0 +1,359 @@ +package indexstore + +import ( + "context" + _ "embed" + "errors" + "fmt" + "strings" + "time" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multihash" + "github.com/yugabyte/gocql" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" +) + +//go:embed create.cql +var createCQL string + +var log = logging.Logger("cassandra") + +type settings struct { + // Number of records per insert batch + InsertBatchSize int + // Number of concurrent inserts to split AddIndex/DeleteIndex calls to + InsertConcurrency int +} + +type IndexStore struct { + settings settings + cluster *gocql.ClusterConfig + session *gocql.Session + ctx context.Context +} + +type OffsetSize struct { + // Offset is the offset into the CAR file of the section, where a section + // is
+ Offset uint64 `json:"offset"` + // Size is the size of the block data (not the whole section) + Size uint64 `json:"size"` +} + +type Record struct { + Cid cid.Cid `json:"cid"` + OffsetSize `json:"offsetsize"` +} + +// Probability of a collision in two 20 byte hashes (birthday problem): +// 2^(20*8/2) = 1.4 x 10^24 +const multihashLimitBytes = 20 + +// trimMultihash trims the multihash to the last multihashLimitBytes bytes +func trimMultihash(mh multihash.Multihash) []byte { + var idx int + if len(mh) > multihashLimitBytes { + idx = len(mh) - multihashLimitBytes + } + return mh[idx:] +} + +var ErrNotFound = errors.New("not found") + +func normalizeMultihashError(m multihash.Multihash, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("multihash %s: %w", m, ErrNotFound) + } + return err +} + +func isNotFoundErr(err error) bool { + if err == nil { + return false + } + + if errors.Is(err, gocql.ErrNotFound) { + return true + } + + // Unfortunately it seems like the Cassandra driver doesn't always return + // a specific not found error type, so we need to rely on string parsing + return strings.Contains(strings.ToLower(err.Error()), "not found") +} + +func NewIndexStore(hosts []string) (*IndexStore, error) { + if len(hosts) == 0 { + return nil, xerrors.Errorf("no hosts provided for cassandra") + } + + cluster := gocql.NewCluster(hosts...) + cluster.Timeout = time.Minute + cluster.Keyspace = "curio" + + store := &IndexStore{ + cluster: cluster, + } + + return store, store.Start(context.Background()) +} + +func (i *IndexStore) Start(ctx context.Context) error { + log.Info("Starting cassandra DB") + // Create cassandra keyspace + session, err := i.cluster.CreateSession() + if err != nil { + return xerrors.Errorf("creating cassandra session: %w", err) + + } + query := `CREATE KEYSPACE IF NOT EXISTS ` + i.cluster.Keyspace + + ` WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }` + err = session.Query(query).WithContext(ctx).Exec() + if err != nil { + return xerrors.Errorf("creating cassandra keyspace: %w", err) + } + + lines := strings.Split(createCQL, ";") + for _, line := range lines { + line = strings.Trim(line, "\n \t") + if line == "" { + continue + } + log.Debug(line) + err := session.Query(line).WithContext(ctx).Exec() + if err != nil { + return xerrors.Errorf("creating tables: executing\n%s\n%w", line, err) + } + } + + i.session = session + i.ctx = ctx + + log.Infow("Successfully started Cassandra DB") + return nil +} + +// AddIndex adds multihash -> piece cid mappings, along with offset / size information for the piece. +// It takes a context, the piece cid, and a slice of Record structs as arguments. +// It returns an error if any error occurs during the execution. +func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, records []Record) error { + + Qry := `INSERT INTO PayloadToPiece (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES (?, ?, ?, ?)` + pieceCidBytes := pieceCid.Bytes() + + batchSize := len(records) / i.settings.InsertConcurrency // split the slice into go-routine batches + + if batchSize == 0 { + batchSize = len(records) + } + + log.Debugw("addIndex call", "BatchSize", batchSize, "Total Records", len(records)) + + var eg errgroup.Group + + // Batch to allow multiple goroutines concurrency to do batch inserts + // These batches will be further batched based on InsertBatchSize in the + // goroutines for each BatchInsert operation + for start := 0; start < len(records); start += batchSize { + start := start + end := start + batchSize + if end >= len(records) { + end = len(records) + } + + eg.Go(func() error { + var batch *gocql.Batch + recsb := records[start:end] + // Running a loop on all instead of creating batches require less memory + for allIdx, rec := range recsb { + if batch == nil { + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) + } + + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: Qry, + Args: []any{pieceCidBytes, rec.Cid.Hash(), rec.Offset, rec.Size}, + Idempotent: true, + }) + + if allIdx == len(recsb)-1 || len(batch.Entries) == i.settings.InsertBatchSize { + err := func() error { + defer func(start time.Time) { + log.Debugw("addIndex Batch Insert", "took", time.Since(start), "entries", len(batch.Entries)) + }(time.Now()) + + err := i.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("executing batch insert for piece %s: %w", pieceCid, err) + } + return nil + }() + if err != nil { + return err + } + batch = nil + } + } + return nil + }) + } + + err := eg.Wait() + if err != nil { + return err + } + + return nil +} + +// RemoveIndexes removes all multihash -> piece cid mappings, and all +// offset / size information for the piece. +func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { + Qry := `DELETE FROM PayloadToPiece WHERE PayloadMultihash = ? AND PieceCid = ?` + pieceCidBytes := pieceCid.Bytes() + + // Get multihashes for piece + recs, err := i.GetIndex(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing indexes for piece %s: getting recs: %w", pieceCid, err) + } + + // Batch delete with concurrency + var eg errgroup.Group + for j := 0; j < i.settings.InsertConcurrency; j++ { + eg.Go(func() error { + var batch *gocql.Batch + var num int + select { + case <-ctx.Done(): + return ctx.Err() + case rec, ok := <-recs: + if !ok { + // Finished adding all the queued items, exit the thread + err := i.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("executing batch delete for piece %s: %w", pieceCid, err) + } + return nil + } + + if batch == nil { + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) + } + + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: Qry, + Args: []any{pieceCidBytes, rec.Cid.Hash(), rec.Offset, rec.Size}, + Idempotent: true, + }) + + num++ + + if num == i.settings.InsertBatchSize { + err := i.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("executing batch delete for piece %s: %w", pieceCid, err) + } + } + + } + return nil + }) + } + err = eg.Wait() + if err != nil { + return err + } + + return nil +} + +type IndexRecord struct { + Record + Error error `json:"error,omitempty"` +} + +// GetIndex retrieves the multihashes and offset/size information for a given piece CID. +// It returns a channel of `IndexRecord` structs, which include the CID and the offset/size information. +// If no multihashes are found for the piece, it returns an error. +func (i *IndexStore) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan IndexRecord, error) { + qry := `SELECT PayloadMultihash, BlockOffset, BlockSize FROM PayloadToPiece WHERE PieceCid = ?` + iter := i.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + + records := make(chan IndexRecord) + + parseRecord := func(payload []byte, off, s uint64) { + _, pmh, err := multihash.MHFromBytes(payload) + if err != nil { + records <- IndexRecord{Error: err} + return + } + + records <- IndexRecord{ + Record: Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + OffsetSize: OffsetSize{ + Offset: off, + Size: s, + }, + }, + } + } + + var payloadMHBz []byte + var offset, size uint64 + + // Try to find first record. If not found then we don't have index + // for this piece + found := iter.Scan(&payloadMHBz, &offset, &size) + if !found { + return nil, fmt.Errorf("no multihashed found for piece %s", pieceCid.String()) + } + parseRecord(payloadMHBz, offset, size) + + go func() { + defer close(records) + + for iter.Scan(&payloadMHBz, &offset, &size) { + parseRecord(payloadMHBz, offset, size) + } + + if err := iter.Close(); err != nil { + err = fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) + records <- IndexRecord{Error: err} + } + }() + + return records, nil +} + +// PiecesContainingMultihash gets all pieces that contain a multihash (used when retrieving by payload CID) +func (i *IndexStore) PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) { + var pcids []cid.Cid + var bz []byte + qry := `SELECT PieceCid FROM PayloadToPiece WHERE PayloadMultihash = ?` + iter := i.session.Query(qry, trimMultihash(m)).WithContext(ctx).Iter() + for iter.Scan(&bz) { + pcid, err := cid.Parse(bz) + if err != nil { + return nil, fmt.Errorf("parsing piece cid: %w", err) + } + pcids = append(pcids, pcid) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting pieces containing multihash %s: %w", m, err) + } + + // No pieces found for multihash, return a "not found" error + if len(pcids) == 0 { + return nil, normalizeMultihashError(m, ErrNotFound) + } + return pcids, nil +} diff --git a/lib/libp2p/libp2p.go b/lib/libp2p/libp2p.go new file mode 100644 index 000000000..78c5688c1 --- /dev/null +++ b/lib/libp2p/libp2p.go @@ -0,0 +1,207 @@ +package libp2p + +import ( + "context" + "fmt" + "strings" + + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/metrics" + "github.com/libp2p/go-libp2p/core/peer" + basichost "github.com/libp2p/go-libp2p/p2p/host/basic" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" + "github.com/multiformats/go-multiaddr" + mamask "github.com/whyrusleeping/multiaddr-filter" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/curio/build" + "github.com/filecoin-project/curio/harmony/harmonydb" +) + +var log = logging.Logger("curio-libp2p") + +func NewLibp2pHost(ctx context.Context, db *harmonydb.DB, miners []string) ([]host.Host, error) { + cfg, err := getCfg(ctx, db, miners) + if err != nil { + return nil, err + } + + var ret []host.Host + + for miner, c := range cfg { + pstore, err := pstoremem.NewPeerstore() + if err != nil { + return nil, fmt.Errorf("creating peer store: %w", err) + } + + pubK := c.priv.GetPublic() + id, err := peer.IDFromPublicKey(pubK) + if err != nil { + return nil, fmt.Errorf("getting peer ID: %w", err) + } + + err = pstore.AddPrivKey(id, c.priv) + if err != nil { + return nil, fmt.Errorf("adding private key to peerstore: %w", err) + } + err = pstore.AddPubKey(id, pubK) + if err != nil { + return nil, fmt.Errorf("adding public key to peerstore: %w", err) + } + + addrFactory, err := MakeAddrsFactory(c.AnnounceAddr, c.NoAnnounceAddr) + if err != nil { + return nil, fmt.Errorf("creating address factory: %w", err) + } + + opts := []libp2p.Option{ + libp2p.DefaultTransports, + libp2p.ListenAddrs(c.ListenAddr...), + libp2p.AddrsFactory(addrFactory), + libp2p.Peerstore(pstore), + libp2p.UserAgent("curio-" + build.UserVersion()), + libp2p.Ping(true), + libp2p.DisableRelay(), + libp2p.EnableNATService(), + libp2p.BandwidthReporter(metrics.NewBandwidthCounter()), + } + + h, err := libp2p.New(opts...) + if err != nil { + return nil, xerrors.Errorf("creating libp2p host: %w", err) + } + + // Start listening + err = h.Network().Listen(c.ListenAddr...) + if err != nil { + return nil, xerrors.Errorf("failed to listen on addresses: %w", err) + } + + log.Infof("Libp2p started listening for miner %s", miner.String()) + ret = append(ret, h) + } + + return ret, err + +} + +type libp2pCfg struct { + priv crypto.PrivKey + ListenAddr []multiaddr.Multiaddr + AnnounceAddr []multiaddr.Multiaddr + NoAnnounceAddr []multiaddr.Multiaddr +} + +func getCfg(ctx context.Context, db *harmonydb.DB, miners []string) (map[address.Address]*libp2pCfg, error) { + mm := make(map[int64]address.Address) + var ms []int64 + + for _, miner := range miners { + maddr, err := address.NewFromString(miner) + if err != nil { + return nil, err + } + mid, err := address.IDFromAddress(maddr) + if err != nil { + return nil, err + } + mm[int64(mid)] = maddr + ms = append(ms, int64(mid)) + } + + var cfgs []struct { + SpID int64 `db:"sp_id"` + Key []byte `db:"priv_key"` + ListenAddr string `db:"listen_address"` + AnnounceAddr string `db:"announce_address"` + NoAnnounceAddr string `db:"no_announce_address"` + } + err := db.Select(ctx, &cfgs, `SELECT sp_id, priv_key, listen_address, announce_address FROM libp2p_keys WHERE sp_id = ANY($1)`, ms) + if err != nil { + return nil, xerrors.Errorf("getting libp2p details from DB: %w", err) + } + + if len(cfgs) != len(miners) { + return nil, fmt.Errorf("mismatched number of miners and libp2p configurations") + } + + ret := make(map[address.Address]*libp2pCfg) + + for _, cfg := range cfgs { + p, err := crypto.UnmarshalPrivateKey(cfg.Key) + if err != nil { + return nil, xerrors.Errorf("unmarshaling private key: %w", err) + } + + ret[mm[cfg.SpID]] = &libp2pCfg{ + priv: p, + } + + la := strings.Split(cfg.ListenAddr, ",") + for _, l := range la { + listenAddr, err := multiaddr.NewMultiaddr(l) + if err != nil { + return nil, xerrors.Errorf("parsing listen address: %w", err) + } + ret[mm[cfg.SpID]].ListenAddr = append(ret[mm[cfg.SpID]].ListenAddr, listenAddr) + } + + aa := strings.Split(cfg.AnnounceAddr, ",") + for _, a := range aa { + announceAddr, err := multiaddr.NewMultiaddr(a) + if err != nil { + return nil, xerrors.Errorf("parsing announce address: %w", err) + } + ret[mm[cfg.SpID]].AnnounceAddr = append(ret[mm[cfg.SpID]].AnnounceAddr, announceAddr) + } + + naa := strings.Split(cfg.NoAnnounceAddr, ",") + for _, na := range naa { + noAnnounceAddr, err := multiaddr.NewMultiaddr(na) + if err != nil { + return nil, xerrors.Errorf("parsing no announce address: %w", err) + } + ret[mm[cfg.SpID]].NoAnnounceAddr = append(ret[mm[cfg.SpID]].NoAnnounceAddr, noAnnounceAddr) + } + } + + return ret, nil +} + +func MakeAddrsFactory(announceAddrs, noAnnounce []multiaddr.Multiaddr) (basichost.AddrsFactory, error) { + filters := multiaddr.NewFilters() + noAnnAddrs := map[string]bool{} + for _, addr := range noAnnounce { + f, err := mamask.NewMask(addr.String()) + if err == nil { + filters.AddFilter(*f, multiaddr.ActionDeny) + continue + } + noAnnAddrs[string(addr.Bytes())] = true + } + + return func(allAddrs []multiaddr.Multiaddr) []multiaddr.Multiaddr { + var addrs []multiaddr.Multiaddr + if len(announceAddrs) > 0 { + addrs = announceAddrs + } else { + addrs = allAddrs + } + + var out []multiaddr.Multiaddr + for _, maddr := range addrs { + // check for exact matches + ok := noAnnAddrs[string(maddr.Bytes())] + // check for /ipcidr matches + if !ok && !filters.AddrBlocked(maddr) { + out = append(out, maddr) + } + } + return out + }, nil +} diff --git a/lib/paths/mocks/index.go b/lib/paths/mocks/index.go index 082d22908..347051491 100644 --- a/lib/paths/mocks/index.go +++ b/lib/paths/mocks/index.go @@ -8,14 +8,14 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - abi "github.com/filecoin-project/go-state-types/abi" paths "github.com/filecoin-project/curio/lib/paths" storiface "github.com/filecoin-project/curio/lib/storiface" fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" + + gomock "github.com/golang/mock/gomock" ) // MockSectorIndex is a mock of SectorIndex interface. diff --git a/lib/paths/mocks/pf.go b/lib/paths/mocks/pf.go index 072e47b2e..cb0bb126e 100644 --- a/lib/paths/mocks/pf.go +++ b/lib/paths/mocks/pf.go @@ -8,12 +8,11 @@ import ( io "io" reflect "reflect" - gomock "github.com/golang/mock/gomock" - abi "github.com/filecoin-project/go-state-types/abi" partialfile "github.com/filecoin-project/curio/lib/partialfile" storiface "github.com/filecoin-project/curio/lib/storiface" + gomock "github.com/golang/mock/gomock" ) // MockPartialFileHandler is a mock of PartialFileHandler interface. diff --git a/lib/paths/mocks/store.go b/lib/paths/mocks/store.go index e7c3c184c..b44366c8d 100644 --- a/lib/paths/mocks/store.go +++ b/lib/paths/mocks/store.go @@ -8,14 +8,14 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - cid "github.com/ipfs/go-cid" - abi "github.com/filecoin-project/go-state-types/abi" storiface "github.com/filecoin-project/curio/lib/storiface" fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" + + gomock "github.com/golang/mock/gomock" + cid "github.com/ipfs/go-cid" ) // MockStore is a mock of Store interface. diff --git a/market/mk12/libp2pimpl/libp2pimpl.go b/market/mk12/libp2pimpl/libp2pimpl.go new file mode 100644 index 000000000..7a1d56d5c --- /dev/null +++ b/market/mk12/libp2pimpl/libp2pimpl.go @@ -0,0 +1,449 @@ +package libp2pimpl + +import ( + "context" + "encoding/json" + "fmt" + "runtime/debug" + "time" + + "github.com/google/uuid" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v9/market" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/market/mk12" + "github.com/filecoin-project/curio/market/mk12/legacytypes" + + "github.com/filecoin-project/lotus/chain/types" +) + +var log = logging.Logger("mk12-net") +var propLog = logging.Logger("mk12-prop") + +const DealProtocolv120ID = "/fil/storage/mk/1.2.0" +const DealProtocolv121ID = "/fil/storage/mk/1.2.1" +const DealStatusV12ProtocolID = "/fil/storage/status/1.2.0" + +// The time limit to read a message from the client when the client opens a stream +const providerReadDeadline = 10 * time.Second + +// The time limit to write a response to the client +const providerWriteDeadline = 10 * time.Second + +func SafeHandle(h network.StreamHandler) network.StreamHandler { + defer func() { + if r := recover(); r != nil { + log.Error("panic occurred", "stack", debug.Stack()) + } + }() + + return h +} + +// DealProvider listens for incoming deal proposals over libp2p +type DealProvider struct { + ctx context.Context + host host.Host + prov *mk12.MK12 + api mk12libp2pAPI + db *harmonydb.DB +} + +type mk12libp2pAPI interface { + StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) +} + +func NewDealProvider(h host.Host, db *harmonydb.DB, prov *mk12.MK12, api mk12libp2pAPI) *DealProvider { + p := &DealProvider{ + host: h, + prov: prov, + api: api, + db: db, + } + return p +} + +func (p *DealProvider) Start(ctx context.Context) { + p.ctx = ctx + + // Note that the handling for deal protocol v1.2.0 and v1.2.1 is the same. + // Deal protocol v1.2.1 has a couple of new fields: SkipIPNIAnnounce and + // RemoveUnsealedCopy. + // If a client that supports deal protocol v1.2.0 sends a request to a + // boostd server that supports deal protocol v1.2.1, the DealParams struct + // will be missing these new fields. + // When the DealParams struct is unmarshalled the missing fields will be + // set to false, which maintains the previous behaviour: + // - SkipIPNIAnnounce=false: announce deal to IPNI + // - RemoveUnsealedCopy=false: keep unsealed copy of deal data + p.host.SetStreamHandler(DealProtocolv121ID, SafeHandle(p.handleNewDealStream)) + p.host.SetStreamHandler(DealProtocolv120ID, SafeHandle(p.handleNewDealStream)) + p.host.SetStreamHandler(DealStatusV12ProtocolID, SafeHandle(p.handleNewDealStatusStream)) + + // Handle Query Ask + p.host.SetStreamHandler(legacytypes.AskProtocolID, SafeHandle(p.handleNewAskStream)) + + // Wait for context cancellation + + <-p.ctx.Done() + p.host.RemoveStreamHandler(DealProtocolv121ID) + p.host.RemoveStreamHandler(DealProtocolv120ID) + p.host.RemoveStreamHandler(DealStatusV12ProtocolID) + p.host.RemoveStreamHandler(legacytypes.AskProtocolID) +} + +// Called when the client opens a libp2p stream with a new deal proposal +func (p *DealProvider) handleNewDealStream(s network.Stream) { + start := time.Now() + reqLogUuid := uuid.New() + reqLog := log.With("reqlog-uuid", reqLogUuid.String(), "client-peer", s.Conn().RemotePeer()) + reqLog.Debugw("new deal proposal request") + + defer func() { + err := s.Close() + if err != nil { + reqLog.Infow("closing stream", "err", err) + } + reqLog.Debugw("handled deal proposal request", "duration", time.Since(start).String()) + }() + + // Set a deadline on reading from the stream so it doesn't hang + _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) + + // Read the deal proposal from the stream + var proposal mk12.DealParams + err := proposal.UnmarshalCBOR(s) + _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed + if err != nil { + reqLog.Warnw("reading storage deal proposal from stream", "err", err) + return + } + + reqLog = reqLog.With("id", proposal.DealUUID) + reqLog.Infow("received deal proposal") + + // Start executing the deal. + // Note: This method just waits for the deal to be accepted, it doesn't + // wait for deal execution to complete. + startExec := time.Now() + res, err := p.prov.ExecuteDeal(context.Background(), &proposal, s.Conn().RemotePeer()) + reqLog.Debugw("processed deal proposal accept") + if err != nil { + reqLog.Warnw("deal proposal failed", "err", err, "reason", res.Reason) + } + + // Log the response + propLog.Infow("send deal proposal response", + "id", proposal.DealUUID, + "accepted", res.Accepted, + "msg", res.Reason, + "peer id", s.Conn().RemotePeer(), + "client address", proposal.ClientDealProposal.Proposal.Client, + "provider address", proposal.ClientDealProposal.Proposal.Provider, + "piece cid", proposal.ClientDealProposal.Proposal.PieceCID.String(), + "piece size", proposal.ClientDealProposal.Proposal.PieceSize, + "verified", proposal.ClientDealProposal.Proposal.VerifiedDeal, + "label", proposal.ClientDealProposal.Proposal.Label, + "start epoch", proposal.ClientDealProposal.Proposal.StartEpoch, + "end epoch", proposal.ClientDealProposal.Proposal.EndEpoch, + "price per epoch", proposal.ClientDealProposal.Proposal.StoragePricePerEpoch, + "duration", time.Since(startExec).String(), + ) + + // Set a deadline on writing to the stream so it doesn't hang + _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) + defer s.SetWriteDeadline(time.Time{}) // nolint + + // Write the response to the client + err = cborutil.WriteCborRPC(s, &mk12.DealResponse{Accepted: res.Accepted, Message: res.Reason}) + if err != nil { + reqLog.Warnw("writing deal response", "err", err) + } +} + +func (p *DealProvider) handleNewDealStatusStream(s network.Stream) { + start := time.Now() + reqLogUuid := uuid.New() + reqLog := log.With("reqlog-uuid", reqLogUuid.String(), "client-peer", s.Conn().RemotePeer()) + reqLog.Debugw("new deal status request") + + defer func() { + err := s.Close() + if err != nil { + reqLog.Infow("closing stream", "err", err) + } + reqLog.Debugw("handled deal status request", "duration", time.Since(start).String()) + }() + + // Read the deal status request from the stream + _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) + var req mk12.DealStatusRequest + err := req.UnmarshalCBOR(s) + _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed + if err != nil { + reqLog.Warnw("reading deal status request from stream", "err", err) + return + } + reqLog = reqLog.With("id", req.DealUUID) + reqLog.Debugw("received deal status request") + + resp := p.getDealStatus(req, reqLog) + reqLog.Debugw("processed deal status request") + + // Set a deadline on writing to the stream so it doesn't hang + _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) + defer s.SetWriteDeadline(time.Time{}) // nolint + + if err := cborutil.WriteCborRPC(s, &resp); err != nil { + reqLog.Errorw("failed to write deal status response", "err", err) + } +} + +func (p *DealProvider) getDealStatus(req mk12.DealStatusRequest, reqLog *zap.SugaredLogger) mk12.DealStatusResponse { + errResp := func(err string) mk12.DealStatusResponse { + return mk12.DealStatusResponse{DealUUID: req.DealUUID, Error: err} + } + + var pdeals []struct { + AfterPSD bool `db:"after_psd"` + Sealed bool `db:"sealed"` + Indexed bool `db:"indexed"` + } + + err := p.db.Select(p.ctx, &pdeals, `SELECT + after_psd, + sealed, + indexed + FROM + market_mk12_deal_pipeline + WHERE + uuid = $1;`, req.DealUUID) + + if err != nil { + return errResp(fmt.Sprintf("failed to query the db for deal status: %s", err)) + } + + if len(pdeals) > 1 { + return errResp("found multiple entries for the same UUID, inform the storage provider") + } + + // If deal is still in pipeline + if len(pdeals) == 1 { + pdeal := pdeals[0] + // If PSD is done + if pdeal.AfterPSD { + st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), true) + if err != nil { + reqLog.Errorw("failed to get sealed deal status", "err", err) + return errResp("failed to get sealed deal status") + } + ret := mk12.DealStatusResponse{ + DealUUID: req.DealUUID, + DealStatus: &mk12.DealStatus{ + Error: st.Error, + Status: "Sealing", + SealingStatus: "Sealed", + Proposal: st.Proposal, + SignedProposalCid: st.SignedProposalCID, + PublishCid: &st.PublishCID, + ChainDealID: st.ChainDealID, + }, + IsOffline: st.Offline, + TransferSize: 1, + NBytesReceived: 1, + } + if pdeal.Sealed { + ret.DealStatus.Status = "Sealed" + } + if pdeal.Indexed { + ret.DealStatus.Status = "Sealed and Indexed" + } + } + // ANything before PSD is processing + st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), false) + if err != nil { + reqLog.Errorw("failed to get sealed deal status", "err", err) + return errResp("failed to get sealed deal status") + } + return mk12.DealStatusResponse{ + DealUUID: req.DealUUID, + DealStatus: &mk12.DealStatus{ + Error: st.Error, + Status: "Processing", + SealingStatus: "Not assigned to sector", + Proposal: st.Proposal, + SignedProposalCid: st.SignedProposalCID, + PublishCid: &st.PublishCID, + ChainDealID: st.ChainDealID, + }, + IsOffline: st.Offline, + TransferSize: 1, + NBytesReceived: 1, + } + } + + // If deal is not in deal pipeline + st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), true) + if err != nil { + reqLog.Errorw("failed to get sealed deal status", "err", err) + return errResp("failed to get sealed deal status") + } + + return mk12.DealStatusResponse{ + DealUUID: req.DealUUID, + DealStatus: &mk12.DealStatus{ + Error: st.Error, + Status: "Sealed", + SealingStatus: "Sealed and Indexed", + Proposal: st.Proposal, + SignedProposalCid: st.SignedProposalCID, + PublishCid: &st.PublishCID, + ChainDealID: st.ChainDealID, + }, + IsOffline: st.Offline, + TransferSize: 1, + NBytesReceived: 1, + } +} + +type dealInfo struct { + Offline bool + Error string + Proposal market.DealProposal + SignedProposalCID cid.Cid + ChainDealID abi.DealID + PublishCID cid.Cid +} + +func (p *DealProvider) getSealedDealStatus(ctx context.Context, id string, onChain bool) (dealInfo, error) { + var dealInfos []struct { + Offline bool `db:"offline"` + Error string `db:"error"` + Proposal json.RawMessage `db:"proposal"` + SignedProposalCID string `db:"signed_proposal_cid"` + } + err := p.db.Select(ctx, &dealInfos, `SELECT + offline, + error, + proposal, + signed_proposal_cid + FROM + market_mk12_deals + WHERE + uuid = $1;`, id) + + if err != nil { + return dealInfo{}, xerrors.Errorf("failed to get deal details from DB: %w", err) + } + + if len(dealInfos) != 1 { + return dealInfo{}, xerrors.Errorf("expected 1 row but got %d", len(dealInfos)) + } + + di := dealInfos[0] + + var prop market.DealProposal + err = json.Unmarshal(di.Proposal, &prop) + if err != nil { + return dealInfo{}, xerrors.Errorf("failed to unmarshal deal proposal: %w", err) + } + + spc, err := cid.Parse(di.SignedProposalCID) + if err != nil { + return dealInfo{}, xerrors.Errorf("failed to parse signed proposal CID: %w", err) + } + + ret := dealInfo{ + Offline: di.Offline, + Error: di.Error, + Proposal: prop, + SignedProposalCID: spc, + ChainDealID: abi.DealID(0), + PublishCID: cid.Undef, + } + + if !onChain { + return ret, nil + } + + var cInfos []struct { + ChainDealID int64 `db:"chain_deal_id"` + PublishCID string `db:"publish_cid"` + } + err = p.db.Select(ctx, &dealInfos, `SELECT + chain_deal_id, + publish_cid + FROM + market_mk12_deals + WHERE + uuid = $1;`, id) + + if err != nil { + return dealInfo{}, xerrors.Errorf("failed to get deal details from DB: %w", err) + } + + if len(cInfos) != 1 { + return dealInfo{}, xerrors.Errorf("expected 1 row but got %d", len(dealInfos)) + } + + ci := cInfos[0] + + pc, err := cid.Parse(ci.PublishCID) + if err != nil { + return dealInfo{}, xerrors.Errorf("failed to parse publish CID: %w", err) + } + + ret.PublishCID = pc + ret.ChainDealID = abi.DealID(ci.ChainDealID) + + return ret, nil +} + +func (p *DealProvider) handleNewAskStream(s network.Stream) { + start := time.Now() + reqLog := log.With("client-peer", s.Conn().RemotePeer()) + reqLog.Debugw("new queryAsk request") + + defer func() { + err := s.Close() + if err != nil { + reqLog.Infow("closing stream", "err", err) + } + reqLog.Debugw("handled queryAsk request", "duration", time.Since(start).String()) + }() + + // Read the deal status request from the stream + _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) + var req legacytypes.AskRequest + err := req.UnmarshalCBOR(s) + _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed + if err != nil { + reqLog.Warnw("reading queryAsk request from stream", "err", err) + return + } + + var resp legacytypes.AskResponse + + resp.Ask, err = p.prov.GetAsk(p.ctx, req.Miner) + if err != nil { + reqLog.Warnw("failed to get ask from storage provider", "err", err) + } + + // Set a deadline on writing to the stream so it doesn't hang + _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) + defer s.SetWriteDeadline(time.Time{}) // nolint + + if err := cborutil.WriteCborRPC(s, &resp); err != nil { + reqLog.Errorw("failed to write queryAsk response", "err", err) + } +} diff --git a/market/mk12/types.go b/market/mk12/types.go index 3070b75f3..57245f703 100644 --- a/market/mk12/types.go +++ b/market/mk12/types.go @@ -266,6 +266,8 @@ type ProviderDealState struct { SectorID abi.SectorNumber Offset abi.PaddedPieceSize + Length abi.PaddedPieceSize + // set if there's an error Err string diff --git a/market/storageingest/deal_ingest_seal.go b/market/storageingest/deal_ingest_seal.go index 7a54c712e..99bc7e766 100644 --- a/market/storageingest/deal_ingest_seal.go +++ b/market/storageingest/deal_ingest_seal.go @@ -9,7 +9,6 @@ import ( "net/url" "time" - logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -19,6 +18,7 @@ import ( verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps/config" diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index 42fa80540..157b27a7f 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -31,6 +31,9 @@ func (s *PipelineGC) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done if err := s.cleanupUpgrade(); err != nil { return false, xerrors.Errorf("cleanupUpgrade: %w", err) } + if err := s.cleanupMK12DealPipeline(); err != nil { + return false, xerrors.Errorf("cleanupMK12DealPipeline: %w", err) + } if err := s.cleanupMK12DealPipeline(); err != nil { return false, xerrors.Errorf("cleanupMK12DealPipeline: %w", err) diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 872cec8ea..835b4e90f 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -102,8 +102,8 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return false, xerrors.Errorf("checking if piece is already indexed: %w", err) } - // Return early if already indexed or should not be indexed - if indexed || !task.ShouldIndex { + // Return early if already indexed + if indexed { err = i.recordCompletion(ctx, task, taskID, false) if err != nil { return false, err @@ -277,7 +277,6 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T } func (i *IndexingTask) TypeDetails() harmonytask.TaskTypeDetails { - return harmonytask.TaskTypeDetails{ Name: "Indexing", Cost: resources.Resources{ diff --git a/tasks/seal/task_movestorage.go b/tasks/seal/task_movestorage.go index d64da0e04..f8e2f32b7 100644 --- a/tasks/seal/task_movestorage.go +++ b/tasks/seal/task_movestorage.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/curio/harmony/taskhelp" ffi2 "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) type MoveStorageTask struct { diff --git a/tasks/snap/task_movestorage.go b/tasks/snap/task_movestorage.go index a4f941a97..d85171559 100644 --- a/tasks/snap/task_movestorage.go +++ b/tasks/snap/task_movestorage.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/paths" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/tasks/seal" ) From 9ade420828d3a2f6ca95a93ff27f3ae1da028c1c Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 8 Aug 2024 15:32:48 +0400 Subject: [PATCH 02/15] fix gen test --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9b4381f72..4434ba23e 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.23.0 @@ -319,7 +320,6 @@ require ( go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect From a180b8c5215e6d69f341c99c15a4a474d3813dbb Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 13 Aug 2024 19:13:41 +0400 Subject: [PATCH 03/15] poller redesign --- .circleci/config.yml | 7 + deps/config/doc_gen.go | 8 + .../default-curio-configuration.md | 7 + documentation/en/curio-cli/curio.md | 20 ++ go.sum | 1 + harmony/harmonydb/sql/20240228-piece-park.sql | 4 + .../sql/20240730-market-migration.sql | 104 ++++++++- lib/dealdata/urlpiecereader.go | 30 ++- lib/indexing/indexstore/create.cql | 6 +- lib/indexing/indexstore/indexstore.go | 198 ++++++----------- lib/indexing/indexstore/indexstore_test.go | 125 +++++++++++ lib/libp2p/libp2p.go | 207 ------------------ market/mk12/libp2pimpl/libp2pimpl.go | 2 +- market/storageingest/deal_ingest_seal.go | 2 +- market/storageingest/deal_ingest_snap.go | 6 +- tasks/indexing/task_indexing.go | 3 + tasks/piece/task_park_piece.go | 50 ++++- tasks/storage-market/task_psd.go | 2 +- 18 files changed, 419 insertions(+), 363 deletions(-) create mode 100644 lib/indexing/indexstore/indexstore_test.go delete mode 100644 lib/libp2p/libp2p.go diff --git a/.circleci/config.yml b/.circleci/config.yml index b4dadc809..16235ffd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -283,3 +283,10 @@ workflows: suite: test-all get-params: true resource_class: 2xlarge + - test: + name: test-idxStore + requires: + - build + suite: idxStore + target: "./lib/indexing/indexstore/indexstore_test.go" + get-params: true diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 664bc799e..2d374eee3 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -847,6 +847,14 @@ This will be used to fail the deals which cannot be sealed on time.`, Comment: `SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain Warning: If this check is skipped and there is a commP mismatch, all deals in the +sector will need to be sent again`, + }, + { + Name: "SkipCommP", + Type: "bool", + + Comment: `SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain +Warning: If this check is skipped and there is a commP mismatch, all deals in the sector will need to be sent again`, }, }, diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 503b74f37..8d78d5d79 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -493,6 +493,13 @@ description: The default curio configuration # type: []string #NoAnnounceAddresses = [] + # SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain + # Warning: If this check is skipped and there is a commP mismatch, all deals in the + # sector will need to be sent again + # + # type: bool + #SkipCommP = false + [Ingest] # Maximum number of sectors that can be queued waiting for deals to start processing. diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index f46bbf73c..48932e662 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -639,9 +639,15 @@ USAGE: curio market command [command options] [arguments...] COMMANDS: +<<<<<<< HEAD seal start sealing a deal sector early add-url Add URL to fetch data for offline deals help, h Shows a list of commands or help for one command +======= + seal start sealing a deal sector early + import-data Import data for offline deal + help, h Shows a list of commands or help for one command +>>>>>>> 48e953d (poller redesign) OPTIONS: --help, -h show help @@ -661,6 +667,7 @@ OPTIONS: --help, -h show help ``` +<<<<<<< HEAD ### curio market add-url ``` NAME: @@ -674,6 +681,19 @@ OPTIONS: --header HEADER, -H HEADER [ --header HEADER, -H HEADER ] Custom HEADER to include in the HTTP request --url URL, -u URL URL to send the request to --help, -h show help +======= +### curio market import-data +``` +NAME: + curio market import-data - Import data for offline deal + +USAGE: + curio market import-data [command options] + +OPTIONS: + --actor value Specify actor address to start sealing sectors for + --help, -h show help +>>>>>>> 48e953d (poller redesign) ``` ## curio fetch-params diff --git a/go.sum b/go.sum index 96454a26c..231a7dc55 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 h1:ZFUue+PNxmHlu7pYv+IYMtqlaO/0VwaGEqKepZf9JpA= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= diff --git a/harmony/harmonydb/sql/20240228-piece-park.sql b/harmony/harmonydb/sql/20240228-piece-park.sql index efd529da7..053145211 100644 --- a/harmony/harmonydb/sql/20240228-piece-park.sql +++ b/harmony/harmonydb/sql/20240228-piece-park.sql @@ -34,7 +34,11 @@ create table parked_piece_refs ( data_headers jsonb not null default '{}', -- host Added in 202240730-market-migrations.sql +<<<<<<< HEAD -- host text, +======= + host text, +>>>>>>> 48e953d (poller redesign) foreign key (piece_id) references parked_pieces(id) on delete cascade ); diff --git a/harmony/harmonydb/sql/20240730-market-migration.sql b/harmony/harmonydb/sql/20240730-market-migration.sql index d936f9bde..be06f70cf 100644 --- a/harmony/harmonydb/sql/20240730-market-migration.sql +++ b/harmony/harmonydb/sql/20240730-market-migration.sql @@ -1,24 +1,39 @@ +-- Table for Mk12 or Boost deals CREATE TABLE market_mk12_deals ( uuid TEXT NOT NULL, +<<<<<<< HEAD created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', +======= + sp_id BIGINT NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + +>>>>>>> 48e953d (poller redesign) signed_proposal_cid TEXT NOT NULL, proposal_signature BYTEA NOT NULL, proposal jsonb NOT NULL, - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, + offline BOOLEAN NOT NULL, verified BOOLEAN NOT NULL, - sp_id BIGINT NOT NULL, + start_epoch BIGINT NOT NULL, end_epoch BIGINT NOT NULL, + client_peer_id TEXT NOT NULL, + chain_deal_id BIGINT DEFAULT NULL, publish_cid TEXT DEFAULT NULL, + + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, length BIGINT DEFAULT NULL, + fast_retrieval BOOLEAN NOT NULL, announce_to_ipni BOOLEAN NOT NULL, + url TEXT DEFAULT NULL, url_headers jsonb NOT NULL DEFAULT '{}', + error TEXT DEFAULT NULL, primary key (uuid, sp_id, piece_cid, signed_proposal_cid), @@ -26,6 +41,8 @@ CREATE TABLE market_mk12_deals ( unique (signed_proposal_cid) ); +-- Table for old lotus market deals. This is just for deal +-- which are still alive. It should not be used for any processing CREATE TABLE market_legacy_deals ( signed_proposal_cid TEXT, proposal_signature BYTEA, @@ -46,6 +63,7 @@ CREATE TABLE market_legacy_deals ( primary key (sp_id, piece_cid, signed_proposal_cid) ); +-- This table is used for storing piece metadata (piece indexing) CREATE TABLE market_piece_metadata ( piece_cid TEXT NOT NULL PRIMARY KEY, version INT NOT NULL DEFAULT 2, @@ -57,14 +75,19 @@ CREATE TABLE market_piece_metadata ( unique (piece_cid) ); +-- This table binds the piece metadata to specific deals (piece indexing) CREATE TABLE market_piece_deal ( - id TEXT NOT NULL, + id TEXT NOT NULL, -- (UUID for new deals, PropCID for old) piece_cid TEXT NOT NULL, + boost_deal BOOLEAN NOT NULL, legacy_deal BOOLEAN NOT NULL DEFAULT FALSE, + chain_deal_id BIGINT NOT NULL DEFAULT 0, + sp_id BIGINT NOT NULL, sector_num BIGINT NOT NULL, + piece_offset BIGINT NOT NULL, piece_length BIGINT NOT NULL, raw_size BIGINT NOT NULL, @@ -74,6 +97,7 @@ CREATE TABLE market_piece_deal ( unique (sp_id, id) ); +-- This function is used to insert piece metadata and piece deal (piece indexing) CREATE OR REPLACE FUNCTION process_piece_deal( _id TEXT, _piece_cid TEXT, @@ -108,64 +132,95 @@ VALUES ( END; $$ LANGUAGE plpgsql; +-- Storage Ask for ask protocol CREATE TABLE market_mk12_storage_ask ( + sp_id BIGINT NOT NULL, + price BIGINT NOT NULL, verified_price BIGINT NOT NULL, + min_size BIGINT NOT NULL, max_size BIGINT NOT NULL, - sp_id BIGINT NOT NULL, + created_at BIGINT NOT NULL, expiry BIGINT NOT NULL, + sequence BIGINT NOT NULL, unique (sp_id) ); +-- Used for processing Mk12 deals CREATE TABLE market_mk12_deal_pipeline ( uuid TEXT NOT NULL, sp_id BIGINT NOT NULL, + started BOOLEAN DEFAULT FALSE, + piece_cid TEXT NOT NULL, +<<<<<<< HEAD piece_size BOOLEAN NOT NULL, +======= + piece_size BIGINT NOT NULL, + file_size BIGINT DEFAULT NULL, -- raw piece size + +>>>>>>> 48e953d (poller redesign) offline BOOLEAN NOT NULL, - downloaded BOOLEAN DEFAULT FALSE, + url TEXT DEFAULT NULL, headers jsonb NOT NULL DEFAULT '{}', - file_size BIGINT DEFAULT NULL, + commp_task_id BIGINT DEFAULT NULL, after_commp BOOLEAN DEFAULT FALSE, - find_task_id BIGINT DEFAULT NULL, - after_find BOOLEAN DEFAULT FALSE, + psd_task_id BIGINT DEFAULT NULL, after_psd BOOLEAN DEFAULT FALSE, + psd_wait_time TIMESTAMPTZ, + find_deal_task_id BIGINT DEFAULT NULL, after_find_deal BOOLEAN DEFAULT FALSE, + sector BIGINT, sector_offset BIGINT, + sealed BOOLEAN DEFAULT FALSE, indexed BOOLEAN DEFAULT FALSE, constraint market_mk12_deal_pipeline_identity_key unique (uuid) ); +-- This table can be used to track remote piece for offline deals +-- The entries must be created by users CREATE TABLE market_offline_urls ( piece_cid TEXT NOT NULL, + url TEXT NOT NULL, headers jsonb NOT NULL DEFAULT '{}', + raw_size BIGINT NOT NULL, + unique (piece_cid) ); +-- indexing tracker is separate from CREATE TABLE market_indexing_tasks ( - id TEXT NOT NULL, + uuid TEXT NOT NULL, + sp_id BIGINT NOT NULL, sector_number BIGINT NOT NULL, reg_seal_proof INT NOT NULL, + piece_offset BIGINT NOT NULL, piece_size BIGINT NOT NULL, raw_size BIGINT NOT NULL, piece_cid TEXT NOT NULL, +<<<<<<< HEAD created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', +======= + + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + +>>>>>>> 48e953d (poller redesign) task_id BIGINT DEFAULT NULL, constraint market_indexing_tasks_identity_key @@ -200,6 +255,35 @@ CREATE TABLE direct_deals ( keep_unsealed_copy BOOLEAN ); +ALTER TABLE parked_piece_refs + ADD COLUMN host text; + +create table file_parked_pieces ( + id bigserial primary key, + created_at timestamp default current_timestamp, + piece_cid text not null, + piece_padded_size bigint not null, + piece_raw_size bigint not null, + complete boolean not null default false, + task_id bigint default null, + cleanup_task_id bigint default null, + unique (piece_cid) +); + +/* + * This table is used to keep track of the references to the file parked pieces + * so that we can delete them when they are no longer needed. + * + * All references into the file_parked_pieces table should be done through this table. + */ +create table file_parked_piece_refs ( + ref_id bigserial primary key, + piece_id bigint not null, + data_url text not null, + node text not null, + foreign key (piece_id) references file_parked_pieces(id) on delete cascade +); + diff --git a/lib/dealdata/urlpiecereader.go b/lib/dealdata/urlpiecereader.go index a7c5d683d..938c59caa 100644 --- a/lib/dealdata/urlpiecereader.go +++ b/lib/dealdata/urlpiecereader.go @@ -4,6 +4,7 @@ import ( "io" "net/http" "net/url" + "os" "golang.org/x/xerrors" ) @@ -62,8 +63,33 @@ func (u *UrlPieceReader) Read(p []byte) (n int, err error) { return 0, xerrors.Errorf("a non 200 response code: %s", resp.Status) } - // Set 'active' to the response body - u.active = resp.Body + if goUrl.Scheme == "file" { + fileUrl := goUrl.Path + file, err := os.Open(fileUrl) + if err != nil { + return 0, xerrors.Errorf("error opening file: %w", err) + } + u.active = file + } else { + req, err := http.NewRequest(http.MethodGet, u.Url, nil) + if err != nil { + return 0, xerrors.Errorf("error creating request: %w", err) + } + // Add custom headers for security and authentication + req.Header = u.Headers + // Create a client and make the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return 0, xerrors.Errorf("error making GET request: %w", err) + } + if resp.StatusCode != 200 { + return 0, xerrors.Errorf("a non 200 response code: %s", resp.Status) + } + + // Set 'active' to the response body + u.active = resp.Body + } } // Calculate the maximum number of bytes we can read without exceeding RawSize diff --git a/lib/indexing/indexstore/create.cql b/lib/indexing/indexstore/create.cql index 21157bb83..6f29a872e 100644 --- a/lib/indexing/indexstore/create.cql +++ b/lib/indexing/indexstore/create.cql @@ -1,9 +1,7 @@ CREATE TABLE IF NOT EXISTS PayloadToPiece ( - PieceCid BLOB, + PieceCid BLOB, -- 20 bytes trimmed PayloadMultihash BLOB, BlockOffset BIGINT, BlockSize BIGINT, PRIMARY KEY (PayloadMultihash, PieceCid) -); - -CREATE INDEX piece_to_payload ON PayloadToPiece(PieceCid, PayloadMultihash); \ No newline at end of file +); \ No newline at end of file diff --git a/lib/indexing/indexstore/indexstore.go b/lib/indexing/indexstore/indexstore.go index 5af4a5ce0..35807792a 100644 --- a/lib/indexing/indexstore/indexstore.go +++ b/lib/indexing/indexstore/indexstore.go @@ -14,8 +14,12 @@ import ( "github.com/yugabyte/gocql" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/deps/config" ) +const keyspace = "curio" + //go:embed create.cql var createCQL string @@ -87,17 +91,20 @@ func isNotFoundErr(err error) bool { return strings.Contains(strings.ToLower(err.Error()), "not found") } -func NewIndexStore(hosts []string) (*IndexStore, error) { +func NewIndexStore(hosts []string, cfg *config.CurioConfig) (*IndexStore, error) { if len(hosts) == 0 { return nil, xerrors.Errorf("no hosts provided for cassandra") } cluster := gocql.NewCluster(hosts...) cluster.Timeout = time.Minute - cluster.Keyspace = "curio" store := &IndexStore{ cluster: cluster, + settings: settings{ + InsertBatchSize: cfg.Market.StorageMarketConfig.Indexing.InsertBatchSize, + InsertConcurrency: cfg.Market.StorageMarketConfig.Indexing.InsertConcurrency, + }, } return store, store.Start(context.Background()) @@ -111,13 +118,23 @@ func (i *IndexStore) Start(ctx context.Context) error { return xerrors.Errorf("creating cassandra session: %w", err) } - query := `CREATE KEYSPACE IF NOT EXISTS ` + i.cluster.Keyspace + + query := `CREATE KEYSPACE IF NOT EXISTS ` + keyspace + ` WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }` err = session.Query(query).WithContext(ctx).Exec() if err != nil { return xerrors.Errorf("creating cassandra keyspace: %w", err) } + session.Close() + + // Recreate session with the keyspace + i.cluster.Keyspace = keyspace + session, err = i.cluster.CreateSession() + if err != nil { + return xerrors.Errorf("creating cassandra session: %w", err) + + } + lines := strings.Split(createCQL, ";") for _, line := range lines { line = strings.Trim(line, "\n \t") @@ -141,48 +158,44 @@ func (i *IndexStore) Start(ctx context.Context) error { // AddIndex adds multihash -> piece cid mappings, along with offset / size information for the piece. // It takes a context, the piece cid, and a slice of Record structs as arguments. // It returns an error if any error occurs during the execution. -func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, records []Record) error { - +func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, recordsChan chan Record) error { Qry := `INSERT INTO PayloadToPiece (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES (?, ?, ?, ?)` pieceCidBytes := pieceCid.Bytes() - batchSize := len(records) / i.settings.InsertConcurrency // split the slice into go-routine batches - - if batchSize == 0 { - batchSize = len(records) - } - - log.Debugw("addIndex call", "BatchSize", batchSize, "Total Records", len(records)) - var eg errgroup.Group - // Batch to allow multiple goroutines concurrency to do batch inserts - // These batches will be further batched based on InsertBatchSize in the + // Start worker threads based on InsertConcurrency value + // These workers will be further batch based on InsertBatchSize in the // goroutines for each BatchInsert operation - for start := 0; start < len(records); start += batchSize { - start := start - end := start + batchSize - if end >= len(records) { - end = len(records) - } - + for worker := 0; worker < i.settings.InsertConcurrency; worker++ { eg.Go(func() error { var batch *gocql.Batch - recsb := records[start:end] // Running a loop on all instead of creating batches require less memory - for allIdx, rec := range recsb { + for { if batch == nil { batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) } + rec, ok := <-recordsChan + + if !ok { + if len(batch.Entries) > 0 { + err := i.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("executing batch insert for piece %s: %w", pieceCid, err) + } + } + return nil + } + batch.Entries = append(batch.Entries, gocql.BatchEntry{ Stmt: Qry, - Args: []any{pieceCidBytes, rec.Cid.Hash(), rec.Offset, rec.Size}, + Args: []any{pieceCidBytes, trimMultihash(rec.Cid.Hash()), rec.Offset, rec.Size}, Idempotent: true, }) - if allIdx == len(recsb)-1 || len(batch.Entries) == i.settings.InsertBatchSize { + if len(batch.Entries) == i.settings.InsertBatchSize { err := func() error { defer func(start time.Time) { log.Debugw("addIndex Batch Insert", "took", time.Since(start), "entries", len(batch.Entries)) @@ -200,7 +213,6 @@ func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, records []R batch = nil } } - return nil }) } @@ -215,123 +227,51 @@ func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, records []R // RemoveIndexes removes all multihash -> piece cid mappings, and all // offset / size information for the piece. func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { - Qry := `DELETE FROM PayloadToPiece WHERE PayloadMultihash = ? AND PieceCid = ?` + delQry := `DELETE FROM PayloadToPiece WHERE PayloadMultihash = ? AND PieceCid = ?` pieceCidBytes := pieceCid.Bytes() // Get multihashes for piece - recs, err := i.GetIndex(ctx, pieceCid) - if err != nil { - return fmt.Errorf("removing indexes for piece %s: getting recs: %w", pieceCid, err) - } - - // Batch delete with concurrency - var eg errgroup.Group - for j := 0; j < i.settings.InsertConcurrency; j++ { - eg.Go(func() error { - var batch *gocql.Batch - var num int - select { - case <-ctx.Done(): - return ctx.Err() - case rec, ok := <-recs: - if !ok { - // Finished adding all the queued items, exit the thread - err := i.session.ExecuteBatch(batch) - if err != nil { - return fmt.Errorf("executing batch delete for piece %s: %w", pieceCid, err) - } - return nil - } - - if batch == nil { - batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) - batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) - } - - batch.Entries = append(batch.Entries, gocql.BatchEntry{ - Stmt: Qry, - Args: []any{pieceCidBytes, rec.Cid.Hash(), rec.Offset, rec.Size}, - Idempotent: true, - }) + getQry := `SELECT PayloadMultihash FROM PayloadToPiece WHERE PieceCid = ?` + iter := i.session.Query(getQry, pieceCidBytes).WithContext(ctx).Iter() - num++ + // Create batch for deletion + batch := i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) - if num == i.settings.InsertBatchSize { - err := i.session.ExecuteBatch(batch) - if err != nil { - return fmt.Errorf("executing batch delete for piece %s: %w", pieceCid, err) - } - } + var payloadMHBz []byte + for iter.Scan(&payloadMHBz) { + // Add each delete operation to batch + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: delQry, + Args: []any{payloadMHBz, pieceCidBytes}, + Idempotent: true, + }) + // Execute batch + if len(batch.Entries) >= i.settings.InsertBatchSize { + err := i.session.ExecuteBatch(batch) + if err != nil { + return xerrors.Errorf("executing batch delete for piece %s: %w", pieceCid, err) } - return nil - }) - } - err = eg.Wait() - if err != nil { - return err + // Create a new batch after executing + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) + } } - return nil -} - -type IndexRecord struct { - Record - Error error `json:"error,omitempty"` -} - -// GetIndex retrieves the multihashes and offset/size information for a given piece CID. -// It returns a channel of `IndexRecord` structs, which include the CID and the offset/size information. -// If no multihashes are found for the piece, it returns an error. -func (i *IndexStore) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan IndexRecord, error) { - qry := `SELECT PayloadMultihash, BlockOffset, BlockSize FROM PayloadToPiece WHERE PieceCid = ?` - iter := i.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() - - records := make(chan IndexRecord) - - parseRecord := func(payload []byte, off, s uint64) { - _, pmh, err := multihash.MHFromBytes(payload) + // Execute remaining operations in the batch + if len(batch.Entries) > 0 { + err := i.session.ExecuteBatch(batch) if err != nil { - records <- IndexRecord{Error: err} - return - } - - records <- IndexRecord{ - Record: Record{ - Cid: cid.NewCidV1(cid.Raw, pmh), - OffsetSize: OffsetSize{ - Offset: off, - Size: s, - }, - }, + return xerrors.Errorf("executing batch delete for piece %s: %w", pieceCid, err) } } - var payloadMHBz []byte - var offset, size uint64 - - // Try to find first record. If not found then we don't have index - // for this piece - found := iter.Scan(&payloadMHBz, &offset, &size) - if !found { - return nil, fmt.Errorf("no multihashed found for piece %s", pieceCid.String()) + if err := iter.Close(); err != nil { + return xerrors.Errorf("Getting piece index for piece %s: %w", pieceCid, err) } - parseRecord(payloadMHBz, offset, size) - go func() { - defer close(records) - - for iter.Scan(&payloadMHBz, &offset, &size) { - parseRecord(payloadMHBz, offset, size) - } - - if err := iter.Close(); err != nil { - err = fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) - records <- IndexRecord{Error: err} - } - }() - - return records, nil + return nil } // PiecesContainingMultihash gets all pieces that contain a multihash (used when retrieving by payload CID) diff --git a/lib/indexing/indexstore/indexstore_test.go b/lib/indexing/indexstore/indexstore_test.go new file mode 100644 index 000000000..79e174fb8 --- /dev/null +++ b/lib/indexing/indexstore/indexstore_test.go @@ -0,0 +1,125 @@ +package indexstore + +import ( + "context" + "io" + "os" + "testing" + "time" + + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/go-commp-utils/writer" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/lib/testutils" +) + +func envElse(env, els string) string { + if v := os.Getenv(env); v != "" { + return v + } + return els +} + +func TestNewIndexStore(t *testing.T) { + // Set up the indexStore for testing + + ctx := context.Background() + cfg := config.DefaultCurioConfig() + + idxStore, err := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, cfg) + require.NoError(t, err) + + // Create a car file and calculate commP + dir, err := os.MkdirTemp(os.TempDir(), "curio-indexstore") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(dir) + }() + + rf, err := testutils.CreateRandomFile(dir, int(time.Now().Unix()), 8000000) + require.NoError(t, err) + + caropts := []carv2.Option{ + blockstore.WriteAsCarV1(true), + } + + _, cn, err := testutils.CreateDenseCARWith(dir, rf, 1024, 1024, caropts) + require.NoError(t, err) + + f, err := os.Open(cn) + require.NoError(t, err) + + defer func() { + _ = f.Close() + }() + + w := &writer.Writer{} + _, err = io.CopyBuffer(w, f, make([]byte, writer.CommPBuf)) + require.NoError(t, err) + + commp, err := w.Sum() + require.NoError(t, err) + + _, err = f.Seek(0, io.SeekStart) + require.NoError(t, err) + + // Create recods + dealCfg := cfg.Market.StorageMarketConfig + chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize + + recs := make(chan Record, chanSize) + opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} + blockReader, err := carv2.NewBlockReader(f, opts...) + require.NoError(t, err) + + // Add index to the store + var eg errgroup.Group + eg.Go(func() error { + serr := idxStore.AddIndex(ctx, commp.PieceCID, recs) + return serr + }) + + var m multihash.Multihash + i := 0 + + blockMetadata, err := blockReader.SkipNext() + for err == nil { + if i == 0 { + m = blockMetadata.Hash() + } + recs <- Record{ + Cid: blockMetadata.Cid, + OffsetSize: OffsetSize{ + Offset: blockMetadata.SourceOffset, + Size: blockMetadata.Size, + }, + } + i++ + + blockMetadata, err = blockReader.SkipNext() + } + require.Error(t, io.EOF) + close(recs) + err = eg.Wait() + require.NoError(t, err) + + // Try to find a multihash + pcids, err := idxStore.PiecesContainingMultihash(ctx, m) + require.NoError(t, err) + require.Len(t, pcids, 1) + require.Equal(t, pcids[0], commp.PieceCID) + + // Remove all indexes from the store + err = idxStore.RemoveIndexes(ctx, pcids[0]) + require.NoError(t, err) + + // Drop the table + err = idxStore.session.Query("DROP TABLE PayloadToPiece").Exec() + require.NoError(t, err) +} diff --git a/lib/libp2p/libp2p.go b/lib/libp2p/libp2p.go deleted file mode 100644 index 78c5688c1..000000000 --- a/lib/libp2p/libp2p.go +++ /dev/null @@ -1,207 +0,0 @@ -package libp2p - -import ( - "context" - "fmt" - "strings" - - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/metrics" - "github.com/libp2p/go-libp2p/core/peer" - basichost "github.com/libp2p/go-libp2p/p2p/host/basic" - "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" - "github.com/multiformats/go-multiaddr" - mamask "github.com/whyrusleeping/multiaddr-filter" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/curio/build" - "github.com/filecoin-project/curio/harmony/harmonydb" -) - -var log = logging.Logger("curio-libp2p") - -func NewLibp2pHost(ctx context.Context, db *harmonydb.DB, miners []string) ([]host.Host, error) { - cfg, err := getCfg(ctx, db, miners) - if err != nil { - return nil, err - } - - var ret []host.Host - - for miner, c := range cfg { - pstore, err := pstoremem.NewPeerstore() - if err != nil { - return nil, fmt.Errorf("creating peer store: %w", err) - } - - pubK := c.priv.GetPublic() - id, err := peer.IDFromPublicKey(pubK) - if err != nil { - return nil, fmt.Errorf("getting peer ID: %w", err) - } - - err = pstore.AddPrivKey(id, c.priv) - if err != nil { - return nil, fmt.Errorf("adding private key to peerstore: %w", err) - } - err = pstore.AddPubKey(id, pubK) - if err != nil { - return nil, fmt.Errorf("adding public key to peerstore: %w", err) - } - - addrFactory, err := MakeAddrsFactory(c.AnnounceAddr, c.NoAnnounceAddr) - if err != nil { - return nil, fmt.Errorf("creating address factory: %w", err) - } - - opts := []libp2p.Option{ - libp2p.DefaultTransports, - libp2p.ListenAddrs(c.ListenAddr...), - libp2p.AddrsFactory(addrFactory), - libp2p.Peerstore(pstore), - libp2p.UserAgent("curio-" + build.UserVersion()), - libp2p.Ping(true), - libp2p.DisableRelay(), - libp2p.EnableNATService(), - libp2p.BandwidthReporter(metrics.NewBandwidthCounter()), - } - - h, err := libp2p.New(opts...) - if err != nil { - return nil, xerrors.Errorf("creating libp2p host: %w", err) - } - - // Start listening - err = h.Network().Listen(c.ListenAddr...) - if err != nil { - return nil, xerrors.Errorf("failed to listen on addresses: %w", err) - } - - log.Infof("Libp2p started listening for miner %s", miner.String()) - ret = append(ret, h) - } - - return ret, err - -} - -type libp2pCfg struct { - priv crypto.PrivKey - ListenAddr []multiaddr.Multiaddr - AnnounceAddr []multiaddr.Multiaddr - NoAnnounceAddr []multiaddr.Multiaddr -} - -func getCfg(ctx context.Context, db *harmonydb.DB, miners []string) (map[address.Address]*libp2pCfg, error) { - mm := make(map[int64]address.Address) - var ms []int64 - - for _, miner := range miners { - maddr, err := address.NewFromString(miner) - if err != nil { - return nil, err - } - mid, err := address.IDFromAddress(maddr) - if err != nil { - return nil, err - } - mm[int64(mid)] = maddr - ms = append(ms, int64(mid)) - } - - var cfgs []struct { - SpID int64 `db:"sp_id"` - Key []byte `db:"priv_key"` - ListenAddr string `db:"listen_address"` - AnnounceAddr string `db:"announce_address"` - NoAnnounceAddr string `db:"no_announce_address"` - } - err := db.Select(ctx, &cfgs, `SELECT sp_id, priv_key, listen_address, announce_address FROM libp2p_keys WHERE sp_id = ANY($1)`, ms) - if err != nil { - return nil, xerrors.Errorf("getting libp2p details from DB: %w", err) - } - - if len(cfgs) != len(miners) { - return nil, fmt.Errorf("mismatched number of miners and libp2p configurations") - } - - ret := make(map[address.Address]*libp2pCfg) - - for _, cfg := range cfgs { - p, err := crypto.UnmarshalPrivateKey(cfg.Key) - if err != nil { - return nil, xerrors.Errorf("unmarshaling private key: %w", err) - } - - ret[mm[cfg.SpID]] = &libp2pCfg{ - priv: p, - } - - la := strings.Split(cfg.ListenAddr, ",") - for _, l := range la { - listenAddr, err := multiaddr.NewMultiaddr(l) - if err != nil { - return nil, xerrors.Errorf("parsing listen address: %w", err) - } - ret[mm[cfg.SpID]].ListenAddr = append(ret[mm[cfg.SpID]].ListenAddr, listenAddr) - } - - aa := strings.Split(cfg.AnnounceAddr, ",") - for _, a := range aa { - announceAddr, err := multiaddr.NewMultiaddr(a) - if err != nil { - return nil, xerrors.Errorf("parsing announce address: %w", err) - } - ret[mm[cfg.SpID]].AnnounceAddr = append(ret[mm[cfg.SpID]].AnnounceAddr, announceAddr) - } - - naa := strings.Split(cfg.NoAnnounceAddr, ",") - for _, na := range naa { - noAnnounceAddr, err := multiaddr.NewMultiaddr(na) - if err != nil { - return nil, xerrors.Errorf("parsing no announce address: %w", err) - } - ret[mm[cfg.SpID]].NoAnnounceAddr = append(ret[mm[cfg.SpID]].NoAnnounceAddr, noAnnounceAddr) - } - } - - return ret, nil -} - -func MakeAddrsFactory(announceAddrs, noAnnounce []multiaddr.Multiaddr) (basichost.AddrsFactory, error) { - filters := multiaddr.NewFilters() - noAnnAddrs := map[string]bool{} - for _, addr := range noAnnounce { - f, err := mamask.NewMask(addr.String()) - if err == nil { - filters.AddFilter(*f, multiaddr.ActionDeny) - continue - } - noAnnAddrs[string(addr.Bytes())] = true - } - - return func(allAddrs []multiaddr.Multiaddr) []multiaddr.Multiaddr { - var addrs []multiaddr.Multiaddr - if len(announceAddrs) > 0 { - addrs = announceAddrs - } else { - addrs = allAddrs - } - - var out []multiaddr.Multiaddr - for _, maddr := range addrs { - // check for exact matches - ok := noAnnAddrs[string(maddr.Bytes())] - // check for /ipcidr matches - if !ok && !filters.AddrBlocked(maddr) { - out = append(out, maddr) - } - } - return out - }, nil -} diff --git a/market/mk12/libp2pimpl/libp2pimpl.go b/market/mk12/libp2pimpl/libp2pimpl.go index 7a1d56d5c..0d7911333 100644 --- a/market/mk12/libp2pimpl/libp2pimpl.go +++ b/market/mk12/libp2pimpl/libp2pimpl.go @@ -269,7 +269,7 @@ func (p *DealProvider) getDealStatus(req mk12.DealStatusRequest, reqLog *zap.Sug ret.DealStatus.Status = "Sealed and Indexed" } } - // ANything before PSD is processing + // Anything before PSD is processing st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), false) if err != nil { reqLog.Errorw("failed to get sealed deal status", "err", err) diff --git a/market/storageingest/deal_ingest_seal.go b/market/storageingest/deal_ingest_seal.go index 99bc7e766..7a54c712e 100644 --- a/market/storageingest/deal_ingest_seal.go +++ b/market/storageingest/deal_ingest_seal.go @@ -9,6 +9,7 @@ import ( "net/url" "time" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -18,7 +19,6 @@ import ( verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" - logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps/config" diff --git a/market/storageingest/deal_ingest_snap.go b/market/storageingest/deal_ingest_snap.go index 445f6d7aa..09d91a3ef 100644 --- a/market/storageingest/deal_ingest_snap.go +++ b/market/storageingest/deal_ingest_snap.go @@ -499,12 +499,12 @@ func (p *PieceIngesterSnap) allocateToExisting(ctx context.Context, tx *harmonyd var allocated bool var rerr error - head, err := p.api.ChainHead(ctx) + openSectors, err := p.getOpenSectors(tx, p.addToID[maddr]) if err != nil { - return false, api.SectorOffset{}, xerrors.Errorf("getting chain head: %w", err) + return false, api.SectorOffset{}, err } - openSectors, err := p.getOpenSectors(tx, p.addToID[maddr]) + head, err := p.api.ChainHead(ctx) if err != nil { return false, api.SectorOffset{}, err } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 835b4e90f..eb107ef51 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -277,6 +277,9 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T } func (i *IndexingTask) TypeDetails() harmonytask.TaskTypeDetails { + //dealCfg := i.cfg.Market.StorageMarketConfig + //chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize * 56 // (56 = size of each index.Record) + return harmonytask.TaskTypeDetails{ Name: "Indexing", Cost: resources.Resources{ diff --git a/tasks/piece/task_park_piece.go b/tasks/piece/task_park_piece.go index 4fb9caf82..1a89ffcd5 100644 --- a/tasks/piece/task_park_piece.go +++ b/tasks/piece/task_park_piece.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "net/url" "strconv" "time" @@ -46,12 +47,20 @@ func NewParkPieceTask(db *harmonydb.DB, sc *ffi2.SealCalls, max int) (*ParkPiece ctx := context.Background() - // We should delete all incomplete pieces before we start + // We should delete all incomplete pieces that do not have a file URL scheme before we start // as we would have lost reader for these. The RPC caller will get an error // when Curio shuts down before parking a piece. They can always retry. // Leaving these pieces we utilise unnecessary resources in the form of ParkPieceTask - _, err := db.Exec(ctx, `DELETE FROM parked_pieces WHERE complete = FALSE AND task_id IS NULL`) + _, err := db.Exec(ctx, `DELETE FROM parked_pieces pp + WHERE complete = FALSE + AND task_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM parked_piece_refs ppr + WHERE ppr.piece_id = pp.id + AND ppr.data_url LIKE 'file:///%' + );`) if err != nil { return nil, xerrors.Errorf("failed to delete incomplete parked pieces: %w", err) } @@ -141,7 +150,8 @@ func (p *ParkPieceTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d err = p.db.Select(ctx, &refData, ` SELECT data_url, data_headers FROM parked_piece_refs - WHERE piece_id = $1 AND data_url IS NOT NULL`, pieceData.PieceID) + WHERE piece_id = $1 AND data_url IS NOT NULL + ORDER BY CASE WHEN data_url LIKE 'file:///%' THEN 0 ELSE 1 END`, pieceData.PieceID) if err != nil { return false, xerrors.Errorf("fetching reference data: %w", err) } @@ -193,8 +203,38 @@ func (p *ParkPieceTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d } func (p *ParkPieceTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { - id := ids[0] - return &id, nil + var pieces []struct { + Tid int64 `db:"task_id"` + URL string `db:"data_url"` + Host string `db:"host"` + } + err := p.db.Select(context.Background(), &pieces, `SELECT pp.task_id, ppr.host, ppr.data_url + FROM parked_pieces pp + JOIN parked_piece_refs ppr ON pp.id = ppr.piece_id + WHERE pp.task_id = ANY($1) + ORDER BY pp.task_id, + CASE WHEN ppr.data_url LIKE 'file:///%' THEN 0 ELSE 1 END;`, ids) + if err != nil { + return nil, xerrors.Errorf("failed to get pending task details from DB: %w", err) + } + + for _, p := range pieces { + ret := harmonytask.TaskID(int(p.Tid)) + goUrl, err := url.Parse(p.URL) + if err != nil { + return nil, xerrors.Errorf("failed to parse the URL: %w", err) + } + if goUrl.Scheme == "file" { + if p.Host == engine.Host() { + + return &ret, nil + } + continue + } + return &ret, nil + } + + return nil, xerrors.Errorf("Host %s is not sutiable for pending PiecePark tasks", engine.Host()) } func (p *ParkPieceTask) TypeDetails() harmonytask.TaskTypeDetails { diff --git a/tasks/storage-market/task_psd.go b/tasks/storage-market/task_psd.go index 21cfa95fc..0ad56a177 100644 --- a/tasks/storage-market/task_psd.go +++ b/tasks/storage-market/task_psd.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" - "github.com/filecoin-project/curio/harmony/taskhelp" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -20,6 +19,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/multictladdr" "github.com/filecoin-project/curio/lib/promise" "github.com/filecoin-project/curio/tasks/message" From 8f94e1d8f5bb9436dc213a78196f37afc69002d2 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 15 Aug 2024 17:19:57 +0400 Subject: [PATCH 04/15] remove indexing table, add defaults --- .circleci/config.yml | 5 +- .../sql/20240730-market-migration.sql | 188 ++++++++++-------- market/storageingest/deal_ingest_seal.go | 2 + market/storageingest/deal_ingest_snap.go | 2 + tasks/indexing/task_indexing.go | 4 +- tasks/storage-market/storage_market.go | 2 +- 6 files changed, 112 insertions(+), 91 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16235ffd3..83feedda5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -144,7 +144,7 @@ jobs: description: Flags passed to go test. target: type: string - default: "./..." + default: "./... | grep -v itests" description: Import paths of packages to be tested. proofs-log-test: type: string @@ -287,6 +287,5 @@ workflows: name: test-idxStore requires: - build - suite: idxStore - target: "./lib/indexing/indexstore/indexstore_test.go" + suite: test-all get-params: true diff --git a/harmony/harmonydb/sql/20240730-market-migration.sql b/harmony/harmonydb/sql/20240730-market-migration.sql index be06f70cf..569f4884f 100644 --- a/harmony/harmonydb/sql/20240730-market-migration.sql +++ b/harmony/harmonydb/sql/20240730-market-migration.sql @@ -1,14 +1,9 @@ -- Table for Mk12 or Boost deals CREATE TABLE market_mk12_deals ( uuid TEXT NOT NULL, -<<<<<<< HEAD created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', -======= sp_id BIGINT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), - ->>>>>>> 48e953d (poller redesign) signed_proposal_cid TEXT NOT NULL, proposal_signature BYTEA NOT NULL, proposal jsonb NOT NULL, @@ -41,33 +36,13 @@ CREATE TABLE market_mk12_deals ( unique (signed_proposal_cid) ); --- Table for old lotus market deals. This is just for deal --- which are still alive. It should not be used for any processing -CREATE TABLE market_legacy_deals ( - signed_proposal_cid TEXT, - proposal_signature BYTEA, - proposal jsonb, - piece_cid TEXT, - piece_size BIGINT, - offline BOOLEAN, - verified BOOLEAN, - sp_id BIGINT, - start_epoch BIGINT, - end_epoch BIGINT, - publish_cid TEXT, - fast_retrieval BOOLEAN, - chain_deal_id BIGINT, - created_at TIMESTAMPTZ, - sector_num BIGINT, - - primary key (sp_id, piece_cid, signed_proposal_cid) -); - -- This table is used for storing piece metadata (piece indexing) CREATE TABLE market_piece_metadata ( piece_cid TEXT NOT NULL PRIMARY KEY, + version INT NOT NULL DEFAULT 2, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', + indexed BOOLEAN NOT NULL DEFAULT FALSE, indexed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', @@ -157,13 +132,9 @@ CREATE TABLE market_mk12_deal_pipeline ( started BOOLEAN DEFAULT FALSE, piece_cid TEXT NOT NULL, -<<<<<<< HEAD piece_size BOOLEAN NOT NULL, -======= - piece_size BIGINT NOT NULL, - file_size BIGINT DEFAULT NULL, -- raw piece size + raw_size BIGINT DEFAULT NULL, ->>>>>>> 48e953d (poller redesign) offline BOOLEAN NOT NULL, url TEXT DEFAULT NULL, @@ -181,52 +152,94 @@ CREATE TABLE market_mk12_deal_pipeline ( after_find_deal BOOLEAN DEFAULT FALSE, sector BIGINT, + reg_seal_proof INT NOT NULL, sector_offset BIGINT, sealed BOOLEAN DEFAULT FALSE, + + should_index BOOLEAN DEFAULT FALSE, + indexing_created_at TIMESTAMPTZ, + indexing_task_id BIGINT DEFAULT NULL, indexed BOOLEAN DEFAULT FALSE, + complete BOOLEAN NOT NULL DEFAULT FALSE, + constraint market_mk12_deal_pipeline_identity_key unique (uuid) ); +-- This function creates indexing task based from move_storage tasks +CREATE OR REPLACE FUNCTION create_indexing_task(task_id BIGINT, sealing_table TEXT) +RETURNS VOID AS $$ +DECLARE +query TEXT; -- Holds the dynamic SQL query + pms RECORD; -- Holds each row returned by the query in the loop +BEGIN + -- Construct the dynamic SQL query based on the sealing_table + IF sealing_table = 'sectors_sdr_pipeline' THEN + query := format( + 'SELECT + dp.uuid, + ssp.reg_seal_proof + FROM + %I ssp + JOIN + market_mk12_deal_pipeline dp ON ssp.sp_id = dp.sp_id AND ssp.sector_num = dp.sector + WHERE + ssp.task_id_move_storage = $1', sealing_table); + ELSIF sealing_table = 'sectors_snap_pipeline' THEN + query := format( + 'SELECT + dp.uuid, + (SELECT reg_seal_proof FROM sectors_meta WHERE sp_id = ssp.sp_id AND sector_num = ssp.sector_num) AS reg_seal_proof + FROM + %I ssp + JOIN + market_mk12_deal_pipeline dp ON ssp.sp_id = dp.sp_id AND ssp.sector_num = dp.sector + WHERE + ssp.task_id_move_storage = $1', sealing_table); +ELSE + RAISE EXCEPTION 'Invalid sealing_table name: %', sealing_table; +END IF; + + -- Execute the dynamic SQL query with the task_id parameter +FOR pms IN EXECUTE query USING task_id + LOOP + -- Update the market_mk12_deal_pipeline table with the reg_seal_proof and indexing_created_at values +UPDATE market_mk12_deal_pipeline +SET + reg_seal_proof = pms.reg_seal_proof, + indexing_created_at = NOW() AT TIME ZONE 'UTC' +WHERE + uuid = pms.uuid; +END LOOP; + + -- If everything is successful, simply exit + RETURN; + +EXCEPTION + WHEN OTHERS THEN + -- Rollback the transaction and raise the exception for Go to catch + ROLLBACK; + RAISE EXCEPTION 'Failed to create indexing task: %', SQLERRM; +END; +$$ LANGUAGE plpgsql; + -- This table can be used to track remote piece for offline deals -- The entries must be created by users CREATE TABLE market_offline_urls ( - piece_cid TEXT NOT NULL, + uuid TEXT NOT NULL, - url TEXT NOT NULL, - headers jsonb NOT NULL DEFAULT '{}', + url TEXT NOT NULL, + headers jsonb NOT NULL DEFAULT '{}', - raw_size BIGINT NOT NULL, + raw_size BIGINT NOT NULL, - unique (piece_cid) + CONSTRAINT market_offline_urls_uuid_fk FOREIGN KEY (uuid) + REFERENCES market_mk12_deal_pipeline (uuid) + ON DELETE CASCADE, + CONSTRAINT market_offline_urls_uuid_unique UNIQUE (uuid) ); --- indexing tracker is separate from -CREATE TABLE market_indexing_tasks ( - uuid TEXT NOT NULL, - - sp_id BIGINT NOT NULL, - sector_number BIGINT NOT NULL, - reg_seal_proof INT NOT NULL, - - piece_offset BIGINT NOT NULL, - piece_size BIGINT NOT NULL, - raw_size BIGINT NOT NULL, - piece_cid TEXT NOT NULL, -<<<<<<< HEAD - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', -======= - - created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), - ->>>>>>> 48e953d (poller redesign) - task_id BIGINT DEFAULT NULL, - - constraint market_indexing_tasks_identity_key - unique (id, sp_id, sector_number, piece_offset, piece_size, piece_cid, reg_seal_proof) -) - CREATE TABLE libp2p_keys ( sp_id BIGINT NOT NULL, priv_key BYTEA NOT NULL, @@ -255,33 +268,38 @@ CREATE TABLE direct_deals ( keep_unsealed_copy BOOLEAN ); +-- Add host column to allow local file based +-- piece park ALTER TABLE parked_piece_refs ADD COLUMN host text; -create table file_parked_pieces ( - id bigserial primary key, - created_at timestamp default current_timestamp, - piece_cid text not null, - piece_padded_size bigint not null, - piece_raw_size bigint not null, - complete boolean not null default false, - task_id bigint default null, - cleanup_task_id bigint default null, - unique (piece_cid) -); +-- Table for old lotus market deals. This is just for deal +-- which are still alive. It should not be used for any processing +CREATE TABLE market_legacy_deals ( + signed_proposal_cid TEXT, + sp_id BIGINT, -/* - * This table is used to keep track of the references to the file parked pieces - * so that we can delete them when they are no longer needed. - * - * All references into the file_parked_pieces table should be done through this table. - */ -create table file_parked_piece_refs ( - ref_id bigserial primary key, - piece_id bigint not null, - data_url text not null, - node text not null, - foreign key (piece_id) references file_parked_pieces(id) on delete cascade + proposal_signature BYTEA, + proposal jsonb, + + piece_cid TEXT, + piece_size BIGINT, + + offline BOOLEAN, + verified BOOLEAN, + + start_epoch BIGINT, + end_epoch BIGINT, + + publish_cid TEXT, + chain_deal_id BIGINT, + + fast_retrieval BOOLEAN, + + created_at TIMESTAMPTZ, + sector_num BIGINT, + + primary key (sp_id, piece_cid, signed_proposal_cid) ); diff --git a/market/storageingest/deal_ingest_seal.go b/market/storageingest/deal_ingest_seal.go index 7a54c712e..8092902fb 100644 --- a/market/storageingest/deal_ingest_seal.go +++ b/market/storageingest/deal_ingest_seal.go @@ -9,6 +9,8 @@ import ( "net/url" "time" + "github.com/filecoin-project/curio/build" + "github.com/filecoin-project/curio/deps/config" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" diff --git a/market/storageingest/deal_ingest_snap.go b/market/storageingest/deal_ingest_snap.go index 09d91a3ef..ad2dd5efa 100644 --- a/market/storageingest/deal_ingest_snap.go +++ b/market/storageingest/deal_ingest_snap.go @@ -9,6 +9,8 @@ import ( "net/url" "time" + "github.com/filecoin-project/curio/build" + "github.com/filecoin-project/curio/deps/config" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index eb107ef51..a505f98c2 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -102,8 +102,8 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return false, xerrors.Errorf("checking if piece is already indexed: %w", err) } - // Return early if already indexed - if indexed { + // Return early if already indexed or should not be indexed + if indexed || !task.ShouldIndex { err = i.recordCompletion(ctx, task, taskID, false) if err != nil { return false, err diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index cf0e71229..21a137a9a 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -407,7 +407,7 @@ func (d *CurioStorageDealMarket) findURLForOfflineDeals(ctx context.Context, dea if err != nil { return false, xerrors.Errorf("error making GET request: %w", err) } - + // Check the response code for 404 if resp.StatusCode != http.StatusOK { if resp.StatusCode != 404 { From 6a83233b4b57b7b294f792b42a86f8195e5ab5a8 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 15 Aug 2024 18:07:20 +0400 Subject: [PATCH 05/15] fix ingest, market tasks --- Makefile | 11 ++++++----- lib/paths/mocks/index.go | 2 ++ market/storageingest/deal_ingest_seal.go | 2 -- market/storageingest/deal_ingest_snap.go | 2 -- tasks/storage-market/storage_market.go | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index e84088a1f..1b8b4a4c1 100644 --- a/Makefile +++ b/Makefile @@ -66,10 +66,11 @@ build/.update-modules: # end git modules ## CUDA Library Path -CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc))) -CUDA_LIB_PATH := $(CUDA_PATH)/lib64 -LIBRARY_PATH ?= $(CUDA_LIB_PATH) -export LIBRARY_PATH +setup_cuda: + $(eval CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc)))) + $(eval CUDA_LIB_PATH := $(CUDA_PATH)/lib64) + export LIBRARY_PATH=$(CUDA_LIB_PATH) +.PHONY: setup_cuda ## MAIN BINARIES @@ -97,7 +98,7 @@ BINS+=sptool ifeq ($(shell uname),Linux) -batchdep: build/.supraseal-install +batchdep: setup_cuda build/.supraseal-install batchdep: $(BUILD_DEPS) .PHONY: batchdep diff --git a/lib/paths/mocks/index.go b/lib/paths/mocks/index.go index 347051491..219bba06f 100644 --- a/lib/paths/mocks/index.go +++ b/lib/paths/mocks/index.go @@ -8,6 +8,8 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + abi "github.com/filecoin-project/go-state-types/abi" paths "github.com/filecoin-project/curio/lib/paths" diff --git a/market/storageingest/deal_ingest_seal.go b/market/storageingest/deal_ingest_seal.go index 8092902fb..7a54c712e 100644 --- a/market/storageingest/deal_ingest_seal.go +++ b/market/storageingest/deal_ingest_seal.go @@ -9,8 +9,6 @@ import ( "net/url" "time" - "github.com/filecoin-project/curio/build" - "github.com/filecoin-project/curio/deps/config" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" diff --git a/market/storageingest/deal_ingest_snap.go b/market/storageingest/deal_ingest_snap.go index ad2dd5efa..09d91a3ef 100644 --- a/market/storageingest/deal_ingest_snap.go +++ b/market/storageingest/deal_ingest_snap.go @@ -9,8 +9,6 @@ import ( "net/url" "time" - "github.com/filecoin-project/curio/build" - "github.com/filecoin-project/curio/deps/config" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index 21a137a9a..cf0e71229 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -407,7 +407,7 @@ func (d *CurioStorageDealMarket) findURLForOfflineDeals(ctx context.Context, dea if err != nil { return false, xerrors.Errorf("error making GET request: %w", err) } - + // Check the response code for 404 if resp.StatusCode != http.StatusOK { if resp.StatusCode != 404 { From d2b3800cb579206426a5930ce4821256f729cab6 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 19 Aug 2024 16:20:24 +0400 Subject: [PATCH 06/15] incomplete basic UI code --- Makefile | 2 +- cmd/curio/rpc/rpc.go | 3 ++ cmd/curio/run.go | 1 + .../sql/20240730-market-migration.sql | 40 +++++++++++-------- lib/paths/mocks/index.go | 2 - web/static/ux/curio-ux.mjs | 1 - 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 1b8b4a4c1..e5aa1217f 100644 --- a/Makefile +++ b/Makefile @@ -134,7 +134,7 @@ debug: build all: build .PHONY: all -build: curio sptool +build: curio @[[ $$(type -P "curio") ]] && echo "Caution: you have \ an existing curio binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true diff --git a/cmd/curio/rpc/rpc.go b/cmd/curio/rpc/rpc.go index 11d97050a..9058143fe 100644 --- a/cmd/curio/rpc/rpc.go +++ b/cmd/curio/rpc/rpc.go @@ -250,6 +250,7 @@ func (p *CurioAPI) LogSetLevel(ctx context.Context, subsystem, level string) err } func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) error { + log.Errorf("ENTERED RPC SERVER") fh := &paths.FetchHandler{Local: dependencies.LocalStore, PfHandler: &paths.DefaultPartialFileHandler{}} remoteHandler := func(w http.ResponseWriter, r *http.Request) { if !auth.HasPerm(r.Context(), nil, lapi.PermAdmin) { @@ -261,6 +262,8 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c fh.ServeHTTP(w, r) } + log.Errorf("CREATED STORAGE HANDLER") + var authVerify func(context.Context, string) ([]auth.Permission, error) { privateKey, err := base64.StdEncoding.DecodeString(dependencies.Cfg.Apis.StorageRPCSecret) diff --git a/cmd/curio/run.go b/cmd/curio/run.go index dedc78d3c..e8a221734 100644 --- a/cmd/curio/run.go +++ b/cmd/curio/run.go @@ -130,6 +130,7 @@ var runCmd = &cli.Command{ } defer taskEngine.GracefullyTerminate() + log.Infof("WILL START RPC SERVER NOW") err = rpc.ListenAndServe(ctx, dependencies, shutdownChan) // Monitor for shutdown. if err != nil { return err diff --git a/harmony/harmonydb/sql/20240730-market-migration.sql b/harmony/harmonydb/sql/20240730-market-migration.sql index 569f4884f..ce8fb6564 100644 --- a/harmony/harmonydb/sql/20240730-market-migration.sql +++ b/harmony/harmonydb/sql/20240730-market-migration.sql @@ -21,7 +21,6 @@ CREATE TABLE market_mk12_deals ( piece_cid TEXT NOT NULL, piece_size BIGINT NOT NULL, - length BIGINT DEFAULT NULL, fast_retrieval BOOLEAN NOT NULL, announce_to_ipni BOOLEAN NOT NULL, @@ -41,7 +40,12 @@ CREATE TABLE market_piece_metadata ( piece_cid TEXT NOT NULL PRIMARY KEY, version INT NOT NULL DEFAULT 2, +<<<<<<< HEAD created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', +======= + + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), +>>>>>>> d1327dc (incomplete basic UI code) indexed BOOLEAN NOT NULL DEFAULT FALSE, indexed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', @@ -276,28 +280,32 @@ ALTER TABLE parked_piece_refs -- Table for old lotus market deals. This is just for deal -- which are still alive. It should not be used for any processing CREATE TABLE market_legacy_deals ( - signed_proposal_cid TEXT, - sp_id BIGINT, + signed_proposal_cid TEXT NOT NULL, + sp_id BIGINT NOT NULL, + client_peer_id TEXT NOT NULL, - proposal_signature BYTEA, - proposal jsonb, + proposal_signature BYTEA NOT NULL, + proposal jsonb NOT NULL, - piece_cid TEXT, - piece_size BIGINT, + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, - offline BOOLEAN, - verified BOOLEAN, + offline BOOLEAN NOT NULL, + verified BOOLEAN NOT NULL, - start_epoch BIGINT, - end_epoch BIGINT, + start_epoch BIGINT NOT NULL, + end_epoch BIGINT NOT NULL, - publish_cid TEXT, - chain_deal_id BIGINT, + publish_cid TEXT NOT NULL, + chain_deal_id BIGINT NOT NULL, - fast_retrieval BOOLEAN, + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, - created_at TIMESTAMPTZ, - sector_num BIGINT, + fast_retrieval BOOLEAN NOT NULL, + + created_at TIMESTAMPTZ NOT NULL, + sector_num BIGINT NOT NULL, primary key (sp_id, piece_cid, signed_proposal_cid) ); diff --git a/lib/paths/mocks/index.go b/lib/paths/mocks/index.go index 219bba06f..082d22908 100644 --- a/lib/paths/mocks/index.go +++ b/lib/paths/mocks/index.go @@ -16,8 +16,6 @@ import ( storiface "github.com/filecoin-project/curio/lib/storiface" fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" - - gomock "github.com/golang/mock/gomock" ) // MockSectorIndex is a mock of SectorIndex interface. diff --git a/web/static/ux/curio-ux.mjs b/web/static/ux/curio-ux.mjs index 673545428..fb7a4d82f 100644 --- a/web/static/ux/curio-ux.mjs +++ b/web/static/ux/curio-ux.mjs @@ -164,7 +164,6 @@ class CurioUX extends LitElement { Curio -<<<<<<< HEAD
  • From 7d602730f6d606326e19ad3d74b8ffba73c562e5 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 20 Aug 2024 18:16:23 +0400 Subject: [PATCH 07/15] finish libp2p init, fix tests --- Makefile | 2 +- cmd/curio/rpc/rpc.go | 3 - cmd/curio/run.go | 1 - deps/config/types.go | 1 + harmony/harmonydb/sql/20240228-piece-park.sql | 7 - .../sql/20240730-market-migration.sql | 65 ++- itests/alertnow_test.go | 1 - market/mk12/libp2pimpl/libp2pimpl.go | 449 ------------------ 8 files changed, 58 insertions(+), 471 deletions(-) delete mode 100644 market/mk12/libp2pimpl/libp2pimpl.go diff --git a/Makefile b/Makefile index e5aa1217f..1b8b4a4c1 100644 --- a/Makefile +++ b/Makefile @@ -134,7 +134,7 @@ debug: build all: build .PHONY: all -build: curio +build: curio sptool @[[ $$(type -P "curio") ]] && echo "Caution: you have \ an existing curio binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true diff --git a/cmd/curio/rpc/rpc.go b/cmd/curio/rpc/rpc.go index 9058143fe..11d97050a 100644 --- a/cmd/curio/rpc/rpc.go +++ b/cmd/curio/rpc/rpc.go @@ -250,7 +250,6 @@ func (p *CurioAPI) LogSetLevel(ctx context.Context, subsystem, level string) err } func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) error { - log.Errorf("ENTERED RPC SERVER") fh := &paths.FetchHandler{Local: dependencies.LocalStore, PfHandler: &paths.DefaultPartialFileHandler{}} remoteHandler := func(w http.ResponseWriter, r *http.Request) { if !auth.HasPerm(r.Context(), nil, lapi.PermAdmin) { @@ -262,8 +261,6 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c fh.ServeHTTP(w, r) } - log.Errorf("CREATED STORAGE HANDLER") - var authVerify func(context.Context, string) ([]auth.Permission, error) { privateKey, err := base64.StdEncoding.DecodeString(dependencies.Cfg.Apis.StorageRPCSecret) diff --git a/cmd/curio/run.go b/cmd/curio/run.go index e8a221734..dedc78d3c 100644 --- a/cmd/curio/run.go +++ b/cmd/curio/run.go @@ -130,7 +130,6 @@ var runCmd = &cli.Command{ } defer taskEngine.GracefullyTerminate() - log.Infof("WILL START RPC SERVER NOW") err = rpc.ListenAndServe(ctx, dependencies, shutdownChan) // Monitor for shutdown. if err != nil { return err diff --git a/deps/config/types.go b/deps/config/types.go index 80276ae5f..ac12bba1f 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -638,6 +638,7 @@ type IndexingConfig struct { type Libp2pConfig struct { // Miners ID for which MK12 deals (boosts) should be disabled DisabledMiners []string + // Binding address for the libp2p host - 0 means random port. // Format: multiaddress; see https://multiformats.io/multiaddr/ ListenAddresses []string diff --git a/harmony/harmonydb/sql/20240228-piece-park.sql b/harmony/harmonydb/sql/20240228-piece-park.sql index 053145211..add0a4093 100644 --- a/harmony/harmonydb/sql/20240228-piece-park.sql +++ b/harmony/harmonydb/sql/20240228-piece-park.sql @@ -33,12 +33,5 @@ create table parked_piece_refs ( data_url text, data_headers jsonb not null default '{}', - -- host Added in 202240730-market-migrations.sql -<<<<<<< HEAD - -- host text, -======= - host text, ->>>>>>> 48e953d (poller redesign) - foreign key (piece_id) references parked_pieces(id) on delete cascade ); diff --git a/harmony/harmonydb/sql/20240730-market-migration.sql b/harmony/harmonydb/sql/20240730-market-migration.sql index ce8fb6564..39fa0a768 100644 --- a/harmony/harmonydb/sql/20240730-market-migration.sql +++ b/harmony/harmonydb/sql/20240730-market-migration.sql @@ -40,12 +40,7 @@ CREATE TABLE market_piece_metadata ( piece_cid TEXT NOT NULL PRIMARY KEY, version INT NOT NULL DEFAULT 2, -<<<<<<< HEAD created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', -======= - - created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), ->>>>>>> d1327dc (incomplete basic UI code) indexed BOOLEAN NOT NULL DEFAULT FALSE, indexed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', @@ -244,7 +239,8 @@ CREATE TABLE market_offline_urls ( CONSTRAINT market_offline_urls_uuid_unique UNIQUE (uuid) ); -CREATE TABLE libp2p_keys ( +-- This table is used for coordinating libp2p nodes +CREATE TABLE libp2p ( sp_id BIGINT NOT NULL, priv_key BYTEA NOT NULL, listen_address TEXT NOT NULL, @@ -272,6 +268,60 @@ CREATE TABLE direct_deals ( keep_unsealed_copy BOOLEAN ); +-- -- Function used to update the libp2p table +CREATE OR REPLACE FUNCTION insert_or_update_libp2p( + _sp_id BIGINT, + _listen_address TEXT, + _announce_address TEXT, + _no_announce_address TEXT, + _running_on TEXT +) +RETURNS BYTEA AS $$ +DECLARE +_priv_key BYTEA; + _current_running_on TEXT; + _current_updated_at TIMESTAMPTZ; +BEGIN + -- Check if the sp_id exists and retrieve the current values + SELECT priv_key, running_on, updated_at INTO _priv_key, _current_running_on, _current_updated_at + FROM libp2p + WHERE sp_id = _sp_id; + + -- Raise an exception if no row was found + IF NOT FOUND THEN + RAISE EXCEPTION 'libp2p key for sp_id "%" does not exist', _sp_id; + END IF; + + -- If the sp_id exists and running_on is NULL or matches _running_on + IF _current_running_on IS NULL OR _current_running_on = _running_on THEN + -- Update the record with the provided values and set updated_at to NOW + UPDATE libp2p + SET + listen_address = _listen_address, + announce_address = _announce_address, + no_announce_address = _no_announce_address, + running_on = _running_on, + updated_at = NOW() AT TIME ZONE 'UTC' + WHERE sp_id = _sp_id; + ELSIF _current_updated_at > NOW() - INTERVAL '10 seconds' THEN + -- Raise an exception if running_on is different and updated_at is recent + RAISE EXCEPTION 'Libp2p node already running on "%"', _current_running_on; + ELSE + -- Update running_on and other columns if updated_at is older than 10 seconds + UPDATE libp2p + SET + listen_address = _listen_address, + announce_address = _announce_address, + no_announce_address = _no_announce_address, + running_on = _running_on, + updated_at = NOW() AT TIME ZONE 'UTC' + WHERE sp_id = _sp_id; + END IF; + +RETURN _priv_key; +END; +$$ LANGUAGE plpgsql; + -- Add host column to allow local file based -- piece park ALTER TABLE parked_piece_refs @@ -299,9 +349,6 @@ CREATE TABLE market_legacy_deals ( publish_cid TEXT NOT NULL, chain_deal_id BIGINT NOT NULL, - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, - fast_retrieval BOOLEAN NOT NULL, created_at TIMESTAMPTZ NOT NULL, diff --git a/itests/alertnow_test.go b/itests/alertnow_test.go index 53c947a25..0eef39730 100644 --- a/itests/alertnow_test.go +++ b/itests/alertnow_test.go @@ -22,7 +22,6 @@ func TestAlertNow(t *testing.T) { // Create dependencies sharedITestID := harmonydb.ITestNewID() db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) - require.NoError(t, err) an := alertmanager.NewAlertNow(db, "alertNowMachine") diff --git a/market/mk12/libp2pimpl/libp2pimpl.go b/market/mk12/libp2pimpl/libp2pimpl.go deleted file mode 100644 index 0d7911333..000000000 --- a/market/mk12/libp2pimpl/libp2pimpl.go +++ /dev/null @@ -1,449 +0,0 @@ -package libp2pimpl - -import ( - "context" - "encoding/json" - "fmt" - "runtime/debug" - "time" - - "github.com/google/uuid" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/network" - "go.uber.org/zap" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/builtin/v9/market" - - "github.com/filecoin-project/curio/harmony/harmonydb" - "github.com/filecoin-project/curio/market/mk12" - "github.com/filecoin-project/curio/market/mk12/legacytypes" - - "github.com/filecoin-project/lotus/chain/types" -) - -var log = logging.Logger("mk12-net") -var propLog = logging.Logger("mk12-prop") - -const DealProtocolv120ID = "/fil/storage/mk/1.2.0" -const DealProtocolv121ID = "/fil/storage/mk/1.2.1" -const DealStatusV12ProtocolID = "/fil/storage/status/1.2.0" - -// The time limit to read a message from the client when the client opens a stream -const providerReadDeadline = 10 * time.Second - -// The time limit to write a response to the client -const providerWriteDeadline = 10 * time.Second - -func SafeHandle(h network.StreamHandler) network.StreamHandler { - defer func() { - if r := recover(); r != nil { - log.Error("panic occurred", "stack", debug.Stack()) - } - }() - - return h -} - -// DealProvider listens for incoming deal proposals over libp2p -type DealProvider struct { - ctx context.Context - host host.Host - prov *mk12.MK12 - api mk12libp2pAPI - db *harmonydb.DB -} - -type mk12libp2pAPI interface { - StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) -} - -func NewDealProvider(h host.Host, db *harmonydb.DB, prov *mk12.MK12, api mk12libp2pAPI) *DealProvider { - p := &DealProvider{ - host: h, - prov: prov, - api: api, - db: db, - } - return p -} - -func (p *DealProvider) Start(ctx context.Context) { - p.ctx = ctx - - // Note that the handling for deal protocol v1.2.0 and v1.2.1 is the same. - // Deal protocol v1.2.1 has a couple of new fields: SkipIPNIAnnounce and - // RemoveUnsealedCopy. - // If a client that supports deal protocol v1.2.0 sends a request to a - // boostd server that supports deal protocol v1.2.1, the DealParams struct - // will be missing these new fields. - // When the DealParams struct is unmarshalled the missing fields will be - // set to false, which maintains the previous behaviour: - // - SkipIPNIAnnounce=false: announce deal to IPNI - // - RemoveUnsealedCopy=false: keep unsealed copy of deal data - p.host.SetStreamHandler(DealProtocolv121ID, SafeHandle(p.handleNewDealStream)) - p.host.SetStreamHandler(DealProtocolv120ID, SafeHandle(p.handleNewDealStream)) - p.host.SetStreamHandler(DealStatusV12ProtocolID, SafeHandle(p.handleNewDealStatusStream)) - - // Handle Query Ask - p.host.SetStreamHandler(legacytypes.AskProtocolID, SafeHandle(p.handleNewAskStream)) - - // Wait for context cancellation - - <-p.ctx.Done() - p.host.RemoveStreamHandler(DealProtocolv121ID) - p.host.RemoveStreamHandler(DealProtocolv120ID) - p.host.RemoveStreamHandler(DealStatusV12ProtocolID) - p.host.RemoveStreamHandler(legacytypes.AskProtocolID) -} - -// Called when the client opens a libp2p stream with a new deal proposal -func (p *DealProvider) handleNewDealStream(s network.Stream) { - start := time.Now() - reqLogUuid := uuid.New() - reqLog := log.With("reqlog-uuid", reqLogUuid.String(), "client-peer", s.Conn().RemotePeer()) - reqLog.Debugw("new deal proposal request") - - defer func() { - err := s.Close() - if err != nil { - reqLog.Infow("closing stream", "err", err) - } - reqLog.Debugw("handled deal proposal request", "duration", time.Since(start).String()) - }() - - // Set a deadline on reading from the stream so it doesn't hang - _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) - - // Read the deal proposal from the stream - var proposal mk12.DealParams - err := proposal.UnmarshalCBOR(s) - _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed - if err != nil { - reqLog.Warnw("reading storage deal proposal from stream", "err", err) - return - } - - reqLog = reqLog.With("id", proposal.DealUUID) - reqLog.Infow("received deal proposal") - - // Start executing the deal. - // Note: This method just waits for the deal to be accepted, it doesn't - // wait for deal execution to complete. - startExec := time.Now() - res, err := p.prov.ExecuteDeal(context.Background(), &proposal, s.Conn().RemotePeer()) - reqLog.Debugw("processed deal proposal accept") - if err != nil { - reqLog.Warnw("deal proposal failed", "err", err, "reason", res.Reason) - } - - // Log the response - propLog.Infow("send deal proposal response", - "id", proposal.DealUUID, - "accepted", res.Accepted, - "msg", res.Reason, - "peer id", s.Conn().RemotePeer(), - "client address", proposal.ClientDealProposal.Proposal.Client, - "provider address", proposal.ClientDealProposal.Proposal.Provider, - "piece cid", proposal.ClientDealProposal.Proposal.PieceCID.String(), - "piece size", proposal.ClientDealProposal.Proposal.PieceSize, - "verified", proposal.ClientDealProposal.Proposal.VerifiedDeal, - "label", proposal.ClientDealProposal.Proposal.Label, - "start epoch", proposal.ClientDealProposal.Proposal.StartEpoch, - "end epoch", proposal.ClientDealProposal.Proposal.EndEpoch, - "price per epoch", proposal.ClientDealProposal.Proposal.StoragePricePerEpoch, - "duration", time.Since(startExec).String(), - ) - - // Set a deadline on writing to the stream so it doesn't hang - _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) - defer s.SetWriteDeadline(time.Time{}) // nolint - - // Write the response to the client - err = cborutil.WriteCborRPC(s, &mk12.DealResponse{Accepted: res.Accepted, Message: res.Reason}) - if err != nil { - reqLog.Warnw("writing deal response", "err", err) - } -} - -func (p *DealProvider) handleNewDealStatusStream(s network.Stream) { - start := time.Now() - reqLogUuid := uuid.New() - reqLog := log.With("reqlog-uuid", reqLogUuid.String(), "client-peer", s.Conn().RemotePeer()) - reqLog.Debugw("new deal status request") - - defer func() { - err := s.Close() - if err != nil { - reqLog.Infow("closing stream", "err", err) - } - reqLog.Debugw("handled deal status request", "duration", time.Since(start).String()) - }() - - // Read the deal status request from the stream - _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) - var req mk12.DealStatusRequest - err := req.UnmarshalCBOR(s) - _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed - if err != nil { - reqLog.Warnw("reading deal status request from stream", "err", err) - return - } - reqLog = reqLog.With("id", req.DealUUID) - reqLog.Debugw("received deal status request") - - resp := p.getDealStatus(req, reqLog) - reqLog.Debugw("processed deal status request") - - // Set a deadline on writing to the stream so it doesn't hang - _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) - defer s.SetWriteDeadline(time.Time{}) // nolint - - if err := cborutil.WriteCborRPC(s, &resp); err != nil { - reqLog.Errorw("failed to write deal status response", "err", err) - } -} - -func (p *DealProvider) getDealStatus(req mk12.DealStatusRequest, reqLog *zap.SugaredLogger) mk12.DealStatusResponse { - errResp := func(err string) mk12.DealStatusResponse { - return mk12.DealStatusResponse{DealUUID: req.DealUUID, Error: err} - } - - var pdeals []struct { - AfterPSD bool `db:"after_psd"` - Sealed bool `db:"sealed"` - Indexed bool `db:"indexed"` - } - - err := p.db.Select(p.ctx, &pdeals, `SELECT - after_psd, - sealed, - indexed - FROM - market_mk12_deal_pipeline - WHERE - uuid = $1;`, req.DealUUID) - - if err != nil { - return errResp(fmt.Sprintf("failed to query the db for deal status: %s", err)) - } - - if len(pdeals) > 1 { - return errResp("found multiple entries for the same UUID, inform the storage provider") - } - - // If deal is still in pipeline - if len(pdeals) == 1 { - pdeal := pdeals[0] - // If PSD is done - if pdeal.AfterPSD { - st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), true) - if err != nil { - reqLog.Errorw("failed to get sealed deal status", "err", err) - return errResp("failed to get sealed deal status") - } - ret := mk12.DealStatusResponse{ - DealUUID: req.DealUUID, - DealStatus: &mk12.DealStatus{ - Error: st.Error, - Status: "Sealing", - SealingStatus: "Sealed", - Proposal: st.Proposal, - SignedProposalCid: st.SignedProposalCID, - PublishCid: &st.PublishCID, - ChainDealID: st.ChainDealID, - }, - IsOffline: st.Offline, - TransferSize: 1, - NBytesReceived: 1, - } - if pdeal.Sealed { - ret.DealStatus.Status = "Sealed" - } - if pdeal.Indexed { - ret.DealStatus.Status = "Sealed and Indexed" - } - } - // Anything before PSD is processing - st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), false) - if err != nil { - reqLog.Errorw("failed to get sealed deal status", "err", err) - return errResp("failed to get sealed deal status") - } - return mk12.DealStatusResponse{ - DealUUID: req.DealUUID, - DealStatus: &mk12.DealStatus{ - Error: st.Error, - Status: "Processing", - SealingStatus: "Not assigned to sector", - Proposal: st.Proposal, - SignedProposalCid: st.SignedProposalCID, - PublishCid: &st.PublishCID, - ChainDealID: st.ChainDealID, - }, - IsOffline: st.Offline, - TransferSize: 1, - NBytesReceived: 1, - } - } - - // If deal is not in deal pipeline - st, err := p.getSealedDealStatus(p.ctx, req.DealUUID.String(), true) - if err != nil { - reqLog.Errorw("failed to get sealed deal status", "err", err) - return errResp("failed to get sealed deal status") - } - - return mk12.DealStatusResponse{ - DealUUID: req.DealUUID, - DealStatus: &mk12.DealStatus{ - Error: st.Error, - Status: "Sealed", - SealingStatus: "Sealed and Indexed", - Proposal: st.Proposal, - SignedProposalCid: st.SignedProposalCID, - PublishCid: &st.PublishCID, - ChainDealID: st.ChainDealID, - }, - IsOffline: st.Offline, - TransferSize: 1, - NBytesReceived: 1, - } -} - -type dealInfo struct { - Offline bool - Error string - Proposal market.DealProposal - SignedProposalCID cid.Cid - ChainDealID abi.DealID - PublishCID cid.Cid -} - -func (p *DealProvider) getSealedDealStatus(ctx context.Context, id string, onChain bool) (dealInfo, error) { - var dealInfos []struct { - Offline bool `db:"offline"` - Error string `db:"error"` - Proposal json.RawMessage `db:"proposal"` - SignedProposalCID string `db:"signed_proposal_cid"` - } - err := p.db.Select(ctx, &dealInfos, `SELECT - offline, - error, - proposal, - signed_proposal_cid - FROM - market_mk12_deals - WHERE - uuid = $1;`, id) - - if err != nil { - return dealInfo{}, xerrors.Errorf("failed to get deal details from DB: %w", err) - } - - if len(dealInfos) != 1 { - return dealInfo{}, xerrors.Errorf("expected 1 row but got %d", len(dealInfos)) - } - - di := dealInfos[0] - - var prop market.DealProposal - err = json.Unmarshal(di.Proposal, &prop) - if err != nil { - return dealInfo{}, xerrors.Errorf("failed to unmarshal deal proposal: %w", err) - } - - spc, err := cid.Parse(di.SignedProposalCID) - if err != nil { - return dealInfo{}, xerrors.Errorf("failed to parse signed proposal CID: %w", err) - } - - ret := dealInfo{ - Offline: di.Offline, - Error: di.Error, - Proposal: prop, - SignedProposalCID: spc, - ChainDealID: abi.DealID(0), - PublishCID: cid.Undef, - } - - if !onChain { - return ret, nil - } - - var cInfos []struct { - ChainDealID int64 `db:"chain_deal_id"` - PublishCID string `db:"publish_cid"` - } - err = p.db.Select(ctx, &dealInfos, `SELECT - chain_deal_id, - publish_cid - FROM - market_mk12_deals - WHERE - uuid = $1;`, id) - - if err != nil { - return dealInfo{}, xerrors.Errorf("failed to get deal details from DB: %w", err) - } - - if len(cInfos) != 1 { - return dealInfo{}, xerrors.Errorf("expected 1 row but got %d", len(dealInfos)) - } - - ci := cInfos[0] - - pc, err := cid.Parse(ci.PublishCID) - if err != nil { - return dealInfo{}, xerrors.Errorf("failed to parse publish CID: %w", err) - } - - ret.PublishCID = pc - ret.ChainDealID = abi.DealID(ci.ChainDealID) - - return ret, nil -} - -func (p *DealProvider) handleNewAskStream(s network.Stream) { - start := time.Now() - reqLog := log.With("client-peer", s.Conn().RemotePeer()) - reqLog.Debugw("new queryAsk request") - - defer func() { - err := s.Close() - if err != nil { - reqLog.Infow("closing stream", "err", err) - } - reqLog.Debugw("handled queryAsk request", "duration", time.Since(start).String()) - }() - - // Read the deal status request from the stream - _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) - var req legacytypes.AskRequest - err := req.UnmarshalCBOR(s) - _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed - if err != nil { - reqLog.Warnw("reading queryAsk request from stream", "err", err) - return - } - - var resp legacytypes.AskResponse - - resp.Ask, err = p.prov.GetAsk(p.ctx, req.Miner) - if err != nil { - reqLog.Warnw("failed to get ask from storage provider", "err", err) - } - - // Set a deadline on writing to the stream so it doesn't hang - _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) - defer s.SetWriteDeadline(time.Time{}) // nolint - - if err := cborutil.WriteCborRPC(s, &resp); err != nil { - reqLog.Errorw("failed to write queryAsk response", "err", err) - } -} From bcd0c20cb42c219253ca5a7737972a552ae12ede Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 20 Aug 2024 18:31:06 +0400 Subject: [PATCH 08/15] fix test-all --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83feedda5..4bb43029c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -144,7 +144,7 @@ jobs: description: Flags passed to go test. target: type: string - default: "./... | grep -v itests" + default: "./..." description: Import paths of packages to be tested. proofs-log-test: type: string From 36a9abb5e08fb1db937466a552bda47295e13ac1 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 29 Aug 2024 21:31:36 +0530 Subject: [PATCH 09/15] ipni provider --- cmd/curio/tasks/tasks.go | 3 +- deps/config/doc_gen.go | 58 ++ deps/config/types.go | 44 ++ .../default-curio-configuration.md | 29 + go.mod | 63 +- go.sum | 151 +++-- .../sql/20240731-market-migration.sql | 20 +- harmony/harmonydb/sql/20240823-ipni.sql | 165 +++++ lib/indexing/indexstore/create.cql | 7 - lib/indexing/indexstore/indexstore.go | 299 --------- lib/indexing/indexstore/indexstore_test.go | 125 ---- lib/ipni/chunker/chunker.go | 159 +++++ lib/ipni/ipni-provider/ipni-provider.go | 577 ++++++++++++++++++ lib/ipni/ipniculib/ipniculib.go | 18 + market/mk12/mk12.go | 12 +- tasks/indexing/task_indexing.go | 4 +- tasks/indexing/task_ipni.go | 451 ++++++++++++++ 17 files changed, 1628 insertions(+), 557 deletions(-) create mode 100644 harmony/harmonydb/sql/20240823-ipni.sql delete mode 100644 lib/indexing/indexstore/create.cql delete mode 100644 lib/indexing/indexstore/indexstore.go delete mode 100644 lib/indexing/indexstore/indexstore_test.go create mode 100644 lib/ipni/chunker/chunker.go create mode 100644 lib/ipni/ipni-provider/ipni-provider.go create mode 100644 lib/ipni/ipniculib/ipniculib.go create mode 100644 tasks/indexing/task_ipni.go diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index 2157f26ad..25a8e9f6f 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -241,7 +241,8 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task } indexingTask := indexing.NewIndexingTask(db, sc, iStore, pp, cfg) - activeTasks = append(activeTasks, indexingTask) + ipniTask := indexing.NewIPNITask(db, sc, iStore, pp, cfg) + activeTasks = append(activeTasks, indexingTask, ipniTask) } diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 2d374eee3..413438246 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -756,6 +756,52 @@ on a sinle node. Enabling on multiple nodes will cause issues with libp2p deals. Comment: ``, }, }, + "HTTPConfig": { + { + Name: "ListenAddress", + Type: "string", + + Comment: `ListenAddress is where HTTP server will be listening on`, + }, + { + Name: "AnnounceAddresses", + Type: "[]string", + + Comment: `AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node. +Curio allows running more than one node for HTTP server and thus all addressed can be announced +simultaneously to the client`, + }, + }, + "IPNIConfig": { + { + Name: "Enable", + Type: "bool", + + Comment: `Enable set whether to enable indexing announcement to the network and expose endpoints that +allow indexer nodes to process announcements. Enabled by default.`, + }, + { + Name: "TopicName", + Type: "string", + + Comment: `TopicName sets the topic name on which the changes to the advertised content are announced. +If not explicitly specified, the topic name is automatically inferred from the network name +in following format: '/indexer/ingest/' +Defaults to empty, which implies the topic name is inferred from network name.`, + }, + { + Name: "WebHost", + Type: "string", + + Comment: `The network indexer host that the web UI should link to for published announcements`, + }, + { + Name: "DirectAnnounceURLs", + Type: "[]string", + + Comment: `The list of URLs of indexing nodes to announce to.`, + }, + }, "IndexingConfig": { { Name: "InsertBatchSize", @@ -865,6 +911,12 @@ sector will need to be sent again`, Comment: `StorageMarketConfig houses all the deal related market configuration`, }, + { + Name: "HTTP", + Type: "HTTPConfig", + + Comment: `HTTP configuration for market HTTP server`, + }, }, "PagerDutyConfig": { { @@ -949,6 +1001,12 @@ The server must have 2 endpoints Comment: `Indexing configuration for deal indexing`, }, + { + Name: "IPNI", + Type: "IPNIConfig", + + Comment: `IPNI configuration for ipni-provider`, + }, { Name: "MK12", Type: "MK12Config", diff --git a/deps/config/types.go b/deps/config/types.go index ac12bba1f..24a7e2040 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -71,6 +71,10 @@ func DefaultCurioConfig() *CurioConfig { }, }, Market: MarketConfig{ + HTTP: HTTPConfig{ + ListenAddress: "0.0.0.0:8888", + AnnounceAddresses: []string{}, + }, StorageMarketConfig: StorageMarketConfig{ PieceLocator: []PieceLocatorConfig{}, Indexing: IndexingConfig{ @@ -90,6 +94,12 @@ func DefaultCurioConfig() *CurioConfig { ExpectedPoRepSealDuration: Duration(8 * time.Hour), ExpectedSnapSealDuration: Duration(2 * time.Hour), }, + IPNI: IPNIConfig{ + Enable: true, + TopicName: "", + WebHost: "https://cid.contact", + DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"}, + }, }, }, } @@ -576,6 +586,9 @@ type ApisConfig struct { type MarketConfig struct { // StorageMarketConfig houses all the deal related market configuration StorageMarketConfig StorageMarketConfig + + // HTTP configuration for market HTTP server + HTTP HTTPConfig } type StorageMarketConfig struct { @@ -589,6 +602,9 @@ type StorageMarketConfig struct { // Indexing configuration for deal indexing Indexing IndexingConfig + // IPNI configuration for ipni-provider + IPNI IPNIConfig + // MK12 encompasses all configuration related to deal protocol mk1.2.0 and mk1.2.1 (i.e. Boost deals) MK12 MK12Config } @@ -650,3 +666,31 @@ type Libp2pConfig struct { // Format: multiaddress NoAnnounceAddresses []string } + +type IPNIConfig struct { + // Enable set whether to enable indexing announcement to the network and expose endpoints that + // allow indexer nodes to process announcements. Enabled by default. + Enable bool + + // TopicName sets the topic name on which the changes to the advertised content are announced. + // If not explicitly specified, the topic name is automatically inferred from the network name + // in following format: '/indexer/ingest/' + // Defaults to empty, which implies the topic name is inferred from network name. + TopicName string + + // The network indexer host that the web UI should link to for published announcements + WebHost string + + // The list of URLs of indexing nodes to announce to. + DirectAnnounceURLs []string +} + +type HTTPConfig struct { + // ListenAddress is where HTTP server will be listening on + ListenAddress string + + // AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node. + // Curio allows running more than one node for HTTP server and thus all addressed can be announced + // simultaneously to the client + AnnounceAddresses []string +} diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 8d78d5d79..53d43c90d 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -229,6 +229,7 @@ description: The default curio configuration # type: bool #EnableBatchSeal = false +<<<<<<< HEAD # EnableDealMarket enabled the deal market on the node. This would also enable libp2p on the node, if configured. # # type: bool @@ -236,6 +237,9 @@ description: The default curio configuration # EnableCommP enables the commP task on te node. CommP is calculated before sending PublishDealMessage for a Mk12 deal # Must have EnableDealMarket = True +======= + # EnableCommP enabled the commP task on te node. CommP is calculated before sending PublishDealMessage for a Mk12 deal +>>>>>>> 563f95c (ipni provider) # # type: bool #EnableCommP = false @@ -431,6 +435,31 @@ description: The default curio configuration # type: int #InsertConcurrency = 8 + [Market.StorageMarketConfig.IPNI] + # Enable set whether to enable indexing announcement to the network and expose endpoints that + # allow indexer nodes to process announcements. Enabled by default. + # + # type: bool + #Enable = true + + # TopicName sets the topic name on which the changes to the advertised content are announced. + # If not explicitly specified, the topic name is automatically inferred from the network name + # in following format: '/indexer/ingest/' + # Defaults to empty, which implies the topic name is inferred from network name. + # + # type: string + #TopicName = "" + + # The network indexer host that the web UI should link to for published announcements + # + # type: string + #WebHost = "https://cid.contact" + + # The list of URLs of indexing nodes to announce to. + # + # type: []string + #DirectAnnounceURLs = ["https://cid.contact/ingest/announce"] + [Market.StorageMarketConfig.MK12] # When a deal is ready to publish, the amount of time to wait for more # deals to be ready to publish before publishing them all as a batch diff --git a/go.mod b/go.mod index 4434ba23e..1dfc00659 100644 --- a/go.mod +++ b/go.mod @@ -56,20 +56,21 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car/v2 v2.13.1 - github.com/ipni/go-libipni v0.0.8 + github.com/ipld/go-ipld-prime v0.21.0 + github.com/ipni/go-libipni v0.6.11 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/kelseyhightower/envconfig v1.4.0 github.com/libp2p/go-buffer-pool v0.1.0 - github.com/libp2p/go-libp2p v0.35.4 + github.com/libp2p/go-libp2p v0.36.2 github.com/manifoldco/promptui v0.9.0 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/multiformats/go-multiaddr v0.12.4 + github.com/multiformats/go-multiaddr v0.13.0 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_golang v1.20.0 github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 github.com/samber/lo v1.39.0 @@ -85,12 +86,12 @@ require ( go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.26.0 - golang.org/x/sync v0.7.0 - golang.org/x/sys v0.23.0 - golang.org/x/text v0.16.0 - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa + golang.org/x/net v0.28.0 + golang.org/x/sync v0.8.0 + golang.org/x/sys v0.24.0 + golang.org/x/text v0.17.0 + golang.org/x/tools v0.24.0 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 ) @@ -134,7 +135,7 @@ require ( github.com/drand/kyber-bls12381 v0.3.1 // indirect github.com/elastic/go-elasticsearch/v7 v7.14.0 // indirect github.com/elastic/go-windows v1.0.0 // indirect - github.com/elastic/gosigar v0.14.2 // indirect + github.com/elastic/gosigar v0.14.3 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect @@ -174,7 +175,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 // indirect + github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect @@ -202,7 +203,6 @@ require ( github.com/ipfs/go-verifcid v0.0.3 // indirect github.com/ipld/go-car v0.6.2 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect - github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect @@ -222,7 +222,7 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kad-dht v0.25.2 // indirect github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect - github.com/libp2p/go-libp2p-pubsub v0.11.0 // indirect + github.com/libp2p/go-libp2p-pubsub v0.12.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect github.com/libp2p/go-maddr-filter v0.1.0 // indirect @@ -239,7 +239,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.16 // indirect - github.com/miekg/dns v1.1.59 // indirect + github.com/miekg/dns v1.1.62 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -256,27 +256,27 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nikkolasg/hexjson v0.1.0 // indirect github.com/nkovacs/streamquote v1.0.0 // indirect - github.com/onsi/ginkgo/v2 v2.17.3 // indirect + github.com/onsi/ginkgo/v2 v2.20.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect - github.com/pion/datachannel v1.5.6 // indirect - github.com/pion/dtls/v2 v2.2.11 // indirect - github.com/pion/ice/v2 v2.3.25 // indirect - github.com/pion/interceptor v0.1.29 // indirect + github.com/pion/datachannel v1.5.8 // indirect + github.com/pion/dtls/v2 v2.2.12 // indirect + github.com/pion/ice/v2 v2.3.34 // indirect + github.com/pion/interceptor v0.1.30 // indirect github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.12 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.14 // indirect - github.com/pion/rtp v1.8.6 // indirect - github.com/pion/sctp v1.8.16 // indirect + github.com/pion/rtp v1.8.9 // indirect + github.com/pion/sctp v1.8.33 // indirect github.com/pion/sdp/v3 v3.0.9 // indirect - github.com/pion/srtp/v2 v2.0.18 // indirect + github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/transport/v2 v2.2.5 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/webrtc/v3 v3.2.40 // indirect + github.com/pion/webrtc/v3 v3.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -284,7 +284,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/quic-go v0.44.0 // indirect + github.com/quic-go/quic-go v0.46.0 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -299,6 +299,7 @@ require ( github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/wlynxg/anet v0.0.4 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-filecoin-go v0.11.1 // indirect @@ -315,13 +316,13 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect - go.uber.org/dig v1.17.1 // indirect - go.uber.org/fx v1.22.1 // indirect + go.uber.org/dig v1.18.0 // indirect + go.uber.org/fx v1.22.2 // indirect go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.25.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/term v0.22.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/mod v0.20.0 // indirect + golang.org/x/term v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect diff --git a/go.sum b/go.sum index 231a7dc55..15df3dfc0 100644 --- a/go.sum +++ b/go.sum @@ -107,10 +107,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -191,7 +191,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 h1:ZFUue+PNxmHlu7pYv+IYMtqlaO/0VwaGEqKepZf9JpA= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= @@ -242,8 +241,8 @@ github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6 github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= +github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -497,8 +496,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 h1:velgFPYr1X9TDwLIfkV7fWqsFlf7TeP11M/7kPd/dVI= -github.com/google/pprof v0.0.0-20240509144519-723abb6459b7/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -689,6 +688,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-test v0.0.4 h1:DKT66T6GBB6PsDFLoO56QZPrOmzJkqU1FZH5C9ySkew= +github.com/ipfs/go-test v0.0.4/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI= github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-unixfsnode v1.9.0 h1:ubEhQhr22sPAKO2DNsyVBW7YB/zA8Zkif25aBvz8rc8= github.com/ipfs/go-unixfsnode v1.9.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8= @@ -708,8 +709,8 @@ github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOan github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipni/go-libipni v0.0.8 h1:0wLfZRSBG84swmZwmaLKul/iB/FlBkkl9ZcR1ub+Z+w= -github.com/ipni/go-libipni v0.0.8/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk= +github.com/ipni/go-libipni v0.6.11 h1:i+a+OCVgtKd0FMg8L9PrNpJgq//MYTxsl7YlyCHKAJY= +github.com/ipni/go-libipni v0.6.11/go.mod h1:hHkfaG5zP8M8RQX8C84gTUre5KODHGPxEbI8E2SgCNw= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= @@ -794,6 +795,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -811,8 +814,8 @@ github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFG github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= -github.com/libp2p/go-libp2p v0.35.4 h1:FDiBUYLkueFwsuNJUZaxKRdpKvBOWU64qQPL768bSeg= -github.com/libp2p/go-libp2p v0.35.4/go.mod h1:RKCDNt30IkFipGL0tl8wQW/3zVWEGFUZo8g2gAKxwjU= +github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U= +github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -834,8 +837,8 @@ github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCv github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-pubsub v0.11.0 h1:+JvS8Kty0OiyUiN0i8H5JbaCgjnJTRnTHe4rU88dLFc= -github.com/libp2p/go-libp2p-pubsub v0.11.0/go.mod h1:QEb+hEV9WL9wCiUAnpY29FZR6W3zK8qYlaml8R4q6gQ= +github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI= +github.com/libp2p/go-libp2p-pubsub v0.12.0/go.mod h1:Oi0zw9aw8/Y5GC99zt+Ef2gYAl+0nZlwdJonDyOz/sE= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= @@ -923,8 +926,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= -github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -970,8 +973,8 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.12.4 h1:rrKqpY9h+n80EwhhC/kkcunCZZ7URIF8yN1WEUt2Hvc= -github.com/multiformats/go-multiaddr v0.12.4/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= +github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= +github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= @@ -1023,14 +1026,14 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU= -github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= +github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.33.0 h1:snPCflnZrpMsy94p4lXVEkHo12lmPnc3vY5XBbreexE= -github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1050,15 +1053,15 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= -github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= -github.com/pion/datachannel v1.5.6/go.mod h1:1eKT6Q85pRnr2mHiWHxJwO50SfZRtWHTsNIVb/NfGW4= +github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= +github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/dtls/v2 v2.2.11 h1:9U/dpCYl1ySttROPWJgqWKEylUdT0fXp/xst6JwY5Ks= -github.com/pion/dtls/v2 v2.2.11/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.25 h1:M5rJA07dqhi3nobJIg+uPtcVjFECTrhcR3n0ns8kDZs= -github.com/pion/ice/v2 v2.3.25/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= -github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= -github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= +github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= +github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM= +github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= +github.com/pion/interceptor v0.1.30 h1:au5rlVHsgmxNi+v/mjOPazbW1SHzfx7/hYOEYQnUcxA= +github.com/pion/interceptor v0.1.30/go.mod h1:RQuKT5HTdkP2Fi0cuOS5G5WNymTjzXaGF75J4k7z2nc= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= @@ -1069,31 +1072,29 @@ github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9 github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.6 h1:MTmn/b0aWWsAzux2AmP8WGllusBVw4NPYPVFFd7jUPw= -github.com/pion/rtp v1.8.6/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/sctp v1.8.13/go.mod h1:YKSgO/bO/6aOMP9LCie1DuD7m+GamiK2yIiPM6vH+GA= -github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY= -github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= +github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk= +github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw= +github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM= github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= -github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= -github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= +github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk= +github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.5 h1:iyi25i/21gQck4hfRhomF6SktmUQjRsRW4WJdhfc3Kc= -github.com/pion/transport/v2 v2.2.5/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= -github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4= -github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0= +github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= +github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.40 h1:Wtfi6AZMQg+624cvCXUuSmrKWepSB7zfgYDOYqsSOVU= -github.com/pion/webrtc/v3 v3.2.40/go.mod h1:M1RAe3TNTD1tzyvqHrbVODfwdPGSXOUo/OgpoGGJqFY= +github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= +github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1114,8 +1115,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1147,8 +1148,8 @@ github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Ez github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0= -github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQlpnPcWmek= +github.com/quic-go/quic-go v0.46.0 h1:uuwLClEEyk1DNvchH8uCByQVjo3yKL9opKulExNDs7Y= +github.com/quic-go/quic-go v0.46.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg= github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= @@ -1318,6 +1319,9 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1: github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/wlynxg/anet v0.0.4 h1:0de1OFQxnNqAu+x2FAKKCVIrnfGKQbs7FQz++tB0+Uw= +github.com/wlynxg/anet v0.0.4/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= @@ -1383,10 +1387,10 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= -go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.22.1 h1:nvvln7mwyT5s1q201YE29V/BFrGor6vMiDNpU/78Mys= -go.uber.org/fx v1.22.1/go.mod h1:HT2M7d7RHo+ebKGh9NRcrsrHHfpZ60nW3QRubMRfv48= +go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= +go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.22.2 h1:iPW+OPxv0G8w75OemJ1RAnTUrF55zOJlXlo1TbJ0Buw= +go.uber.org/fx v1.22.2/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -1434,10 +1438,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1448,8 +1450,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1474,8 +1476,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1529,13 +1531,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1560,8 +1559,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1646,10 +1645,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1659,10 +1656,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1675,8 +1670,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1737,8 +1732,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/harmony/harmonydb/sql/20240731-market-migration.sql b/harmony/harmonydb/sql/20240731-market-migration.sql index 7eaef853b..64c283689 100644 --- a/harmony/harmonydb/sql/20240731-market-migration.sql +++ b/harmony/harmonydb/sql/20240731-market-migration.sql @@ -178,6 +178,8 @@ CREATE TABLE market_mk12_deal_pipeline ( indexing_task_id BIGINT DEFAULT NULL, indexed BOOLEAN DEFAULT FALSE, + announce BOOLEAN DEFAULT FALSE, + complete BOOLEAN NOT NULL DEFAULT FALSE, constraint market_mk12_deal_pipeline_identity_key unique (uuid) @@ -213,21 +215,21 @@ BEGIN market_mk12_deal_pipeline dp ON ssp.sp_id = dp.sp_id AND ssp.sector_num = dp.sector WHERE ssp.task_id_move_storage = $1', sealing_table); -ELSE + ELSE RAISE EXCEPTION 'Invalid sealing_table name: %', sealing_table; -END IF; + END IF; -- Execute the dynamic SQL query with the task_id parameter FOR pms IN EXECUTE query USING task_id LOOP -- Update the market_mk12_deal_pipeline table with the reg_seal_proof and indexing_created_at values -UPDATE market_mk12_deal_pipeline -SET - reg_seal_proof = pms.reg_seal_proof, - indexing_created_at = NOW() AT TIME ZONE 'UTC' -WHERE - uuid = pms.uuid; -END LOOP; + UPDATE market_mk12_deal_pipeline + SET + reg_seal_proof = pms.reg_seal_proof, + indexing_created_at = NOW() AT TIME ZONE 'UTC' + WHERE + uuid = pms.uuid; + END LOOP; -- If everything is successful, simply exit RETURN; diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql new file mode 100644 index 000000000..6d4761c7d --- /dev/null +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -0,0 +1,165 @@ +-- Table for storing IPNI ads +CREATE TABLE ipni ( + order_number BIGSERIAL PRIMARY KEY, -- Unique increasing order number + ad_cid TEXT NOT NULL, + context_id TEXT NOT NULL, -- abi.PieceInfo in Curio + -- metadata column in not required as Curio only supports one type of metadata(HTTP) + is_rm BOOLEAN NOT NULL, + + previous TEXT, -- previous ad will only be null for first ad in chain + + provider TEXT NOT NULL, -- peerID from libp2p, this is main identifier on IPNI side + addresses TEXT NOT NULL, -- HTTP retrieval server addresses + + signature BYTEA NOT NULL, + entries TEXT NOT NULL, -- CID of first link in entry chain + + unique (ad_cid) +); + +-- This index will help speed up the lookup of all ads for a specific provider and ensure fast ordering by order_number +CREATE INDEX ipni_provider_order_number ON ipni(provider, order_number); + +-- This index will speed up lookups based on the ad_cid, which is frequently used to identify specific ads +CREATE UNIQUE INDEX ipni_ad_cid ON ipni(ad_cid); + +-- This index will speed up lookups based on the ad_cid, which is frequently used to identify specific ads +CREATE UNIQUE INDEX ipni_context_id ON ipni(context_id, ad_cid, is_rm); + +-- Since the get_ad_chain function relies on both provider and ad_cid to find the order_number, this index will optimize that query: +CREATE INDEX ipni_provider_ad_cid ON ipni(provider, ad_cid); + + +CREATE TABLE ipni_head ( + provider TEXT NOT NULL PRIMARY KEY, -- PeerID from libp2p, this is the main identifier + head TEXT NOT NULL, -- ad_cid from the ipni table, representing the head of the ad chain + + FOREIGN KEY (head) REFERENCES ipni(ad_cid) ON DELETE RESTRICT -- Prevents deletion if it's referenced +); + +CREATE OR REPLACE FUNCTION insert_ad_and_update_head( + _ad_cid TEXT, + _context_id TEXT, + _is_rm BOOLEAN, + _provider TEXT, + _addresses TEXT, + _signature BYTEA, + _entries TEXT +) RETURNS VOID AS $$ +DECLARE +_previous TEXT; +BEGIN + -- Determine the previous ad_cid in the chain for this provider + SELECT head INTO _previous + FROM ipni_head + WHERE provider = _provider; + + -- Insert the new ad into the ipni table with an automatically assigned order_number + INSERT INTO ipni (ad_cid, context_id, is_rm, previous, provider, addresses, signature, entries) + VALUES (_ad_cid, _context_id, _is_rm, _previous, _provider, _addresses, _signature, _entries); + + -- Update the ipni_head table to set the new ad as the head of the chain + INSERT INTO ipni_head (provider, head) + VALUES (_provider, _ad_cid) + ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION get_ad_chain( + _provider TEXT, + _ad_cid TEXT +) RETURNS TABLE ( + ad_cid TEXT, + context_id TEXT, + is_rm BOOLEAN, + previous TEXT, + provider TEXT, + addresses TEXT, + signature BYTEA, + entries TEXT, + order_number BIGINT +) AS $$ +DECLARE +_order_number BIGINT; +BEGIN + -- Get the order_number for the specified ad_cid and provider + SELECT order_number INTO _order_number + FROM ipni + WHERE ad_cid = _ad_cid AND provider = _provider; + + -- Return all ads from the head to the specified order_number + RETURN QUERY + SELECT ad_cid, context_id, is_rm, previous, provider, addresses, signature, entries, order_number + FROM ipni + WHERE provider = _provider AND order_number <= _order_number + ORDER BY order_number ASC; +END; +$$ LANGUAGE plpgsql; + +-- IPNI pipeline is kept separate from rest for robustness +-- and reuse. This allows for removing, recreating as using CLI. +CREATE TABLE ipni_task ( + sp_id BIGINT NOT NULL, + sector BIGINT NOT NULL, + reg_seal_proof INT NOT NULL, + sector_offset BIGINT, + + context_id BYTEA NOT NULL PRIMARY KEY, + is_rm BOOLEAN NOT NULL, + + provider TEXT NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + task_id BIGINT DEFAULT NULL, + complete BOOLEAN DEFAULT FALSE, + + PRIMARY KEY (provider, context_id, is_rm) +); + +-- Function to create ipni tasks +CREATE OR REPLACE FUNCTION insert_ipni_task( + _sp_id BIGINT, + _sector BIGINT, + _reg_seal_proof INT, + _sector_offset BIGINT, + _context_id BYTEA, + _is_rm BOOLEAN, + _provider TEXT, + _task_id BIGINT DEFAULT NULL +) RETURNS VOID AS $$ +DECLARE + _existing_is_rm BOOLEAN; + _latest_is_rm BOOLEAN; +BEGIN + -- Check if ipni_task has the same context_id and provider with a different is_rm value + SELECT is_rm INTO _existing_is_rm + FROM ipni_task + WHERE provider = _provider AND context_id = _context_id AND is_rm != _is_rm + LIMIT 1; + + -- If a different is_rm exists for the same context_id and provider, insert the new task + IF FOUND THEN + INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) + VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); + RETURN; + END IF; + + -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table + SELECT is_rm INTO _latest_is_rm + FROM ipni + WHERE provider = _provider AND context_id = _context_id + ORDER BY order_number DESC + LIMIT 1; + + -- If the latest ad has the same is_rm value, raise an exception + IF FOUND AND _latest_is_rm = _is_rm THEN + RAISE EXCEPTION 'already published'; + END IF; + + -- If all conditions are met, insert the new task into ipni_task + INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) + VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); +END; +$$ LANGUAGE plpgsql; + diff --git a/lib/indexing/indexstore/create.cql b/lib/indexing/indexstore/create.cql deleted file mode 100644 index 6f29a872e..000000000 --- a/lib/indexing/indexstore/create.cql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE IF NOT EXISTS PayloadToPiece ( - PieceCid BLOB, -- 20 bytes trimmed - PayloadMultihash BLOB, - BlockOffset BIGINT, - BlockSize BIGINT, - PRIMARY KEY (PayloadMultihash, PieceCid) -); \ No newline at end of file diff --git a/lib/indexing/indexstore/indexstore.go b/lib/indexing/indexstore/indexstore.go deleted file mode 100644 index 35807792a..000000000 --- a/lib/indexing/indexstore/indexstore.go +++ /dev/null @@ -1,299 +0,0 @@ -package indexstore - -import ( - "context" - _ "embed" - "errors" - "fmt" - "strings" - "time" - - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multihash" - "github.com/yugabyte/gocql" - "golang.org/x/sync/errgroup" - "golang.org/x/xerrors" - - "github.com/filecoin-project/curio/deps/config" -) - -const keyspace = "curio" - -//go:embed create.cql -var createCQL string - -var log = logging.Logger("cassandra") - -type settings struct { - // Number of records per insert batch - InsertBatchSize int - // Number of concurrent inserts to split AddIndex/DeleteIndex calls to - InsertConcurrency int -} - -type IndexStore struct { - settings settings - cluster *gocql.ClusterConfig - session *gocql.Session - ctx context.Context -} - -type OffsetSize struct { - // Offset is the offset into the CAR file of the section, where a section - // is
    - Offset uint64 `json:"offset"` - // Size is the size of the block data (not the whole section) - Size uint64 `json:"size"` -} - -type Record struct { - Cid cid.Cid `json:"cid"` - OffsetSize `json:"offsetsize"` -} - -// Probability of a collision in two 20 byte hashes (birthday problem): -// 2^(20*8/2) = 1.4 x 10^24 -const multihashLimitBytes = 20 - -// trimMultihash trims the multihash to the last multihashLimitBytes bytes -func trimMultihash(mh multihash.Multihash) []byte { - var idx int - if len(mh) > multihashLimitBytes { - idx = len(mh) - multihashLimitBytes - } - return mh[idx:] -} - -var ErrNotFound = errors.New("not found") - -func normalizeMultihashError(m multihash.Multihash, err error) error { - if err == nil { - return nil - } - if isNotFoundErr(err) { - return fmt.Errorf("multihash %s: %w", m, ErrNotFound) - } - return err -} - -func isNotFoundErr(err error) bool { - if err == nil { - return false - } - - if errors.Is(err, gocql.ErrNotFound) { - return true - } - - // Unfortunately it seems like the Cassandra driver doesn't always return - // a specific not found error type, so we need to rely on string parsing - return strings.Contains(strings.ToLower(err.Error()), "not found") -} - -func NewIndexStore(hosts []string, cfg *config.CurioConfig) (*IndexStore, error) { - if len(hosts) == 0 { - return nil, xerrors.Errorf("no hosts provided for cassandra") - } - - cluster := gocql.NewCluster(hosts...) - cluster.Timeout = time.Minute - - store := &IndexStore{ - cluster: cluster, - settings: settings{ - InsertBatchSize: cfg.Market.StorageMarketConfig.Indexing.InsertBatchSize, - InsertConcurrency: cfg.Market.StorageMarketConfig.Indexing.InsertConcurrency, - }, - } - - return store, store.Start(context.Background()) -} - -func (i *IndexStore) Start(ctx context.Context) error { - log.Info("Starting cassandra DB") - // Create cassandra keyspace - session, err := i.cluster.CreateSession() - if err != nil { - return xerrors.Errorf("creating cassandra session: %w", err) - - } - query := `CREATE KEYSPACE IF NOT EXISTS ` + keyspace + - ` WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }` - err = session.Query(query).WithContext(ctx).Exec() - if err != nil { - return xerrors.Errorf("creating cassandra keyspace: %w", err) - } - - session.Close() - - // Recreate session with the keyspace - i.cluster.Keyspace = keyspace - session, err = i.cluster.CreateSession() - if err != nil { - return xerrors.Errorf("creating cassandra session: %w", err) - - } - - lines := strings.Split(createCQL, ";") - for _, line := range lines { - line = strings.Trim(line, "\n \t") - if line == "" { - continue - } - log.Debug(line) - err := session.Query(line).WithContext(ctx).Exec() - if err != nil { - return xerrors.Errorf("creating tables: executing\n%s\n%w", line, err) - } - } - - i.session = session - i.ctx = ctx - - log.Infow("Successfully started Cassandra DB") - return nil -} - -// AddIndex adds multihash -> piece cid mappings, along with offset / size information for the piece. -// It takes a context, the piece cid, and a slice of Record structs as arguments. -// It returns an error if any error occurs during the execution. -func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, recordsChan chan Record) error { - Qry := `INSERT INTO PayloadToPiece (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES (?, ?, ?, ?)` - pieceCidBytes := pieceCid.Bytes() - - var eg errgroup.Group - - // Start worker threads based on InsertConcurrency value - // These workers will be further batch based on InsertBatchSize in the - // goroutines for each BatchInsert operation - for worker := 0; worker < i.settings.InsertConcurrency; worker++ { - eg.Go(func() error { - var batch *gocql.Batch - // Running a loop on all instead of creating batches require less memory - for { - if batch == nil { - batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) - batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) - } - - rec, ok := <-recordsChan - - if !ok { - if len(batch.Entries) > 0 { - err := i.session.ExecuteBatch(batch) - if err != nil { - return fmt.Errorf("executing batch insert for piece %s: %w", pieceCid, err) - } - } - return nil - } - - batch.Entries = append(batch.Entries, gocql.BatchEntry{ - Stmt: Qry, - Args: []any{pieceCidBytes, trimMultihash(rec.Cid.Hash()), rec.Offset, rec.Size}, - Idempotent: true, - }) - - if len(batch.Entries) == i.settings.InsertBatchSize { - err := func() error { - defer func(start time.Time) { - log.Debugw("addIndex Batch Insert", "took", time.Since(start), "entries", len(batch.Entries)) - }(time.Now()) - - err := i.session.ExecuteBatch(batch) - if err != nil { - return fmt.Errorf("executing batch insert for piece %s: %w", pieceCid, err) - } - return nil - }() - if err != nil { - return err - } - batch = nil - } - } - }) - } - - err := eg.Wait() - if err != nil { - return err - } - - return nil -} - -// RemoveIndexes removes all multihash -> piece cid mappings, and all -// offset / size information for the piece. -func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { - delQry := `DELETE FROM PayloadToPiece WHERE PayloadMultihash = ? AND PieceCid = ?` - pieceCidBytes := pieceCid.Bytes() - - // Get multihashes for piece - getQry := `SELECT PayloadMultihash FROM PayloadToPiece WHERE PieceCid = ?` - iter := i.session.Query(getQry, pieceCidBytes).WithContext(ctx).Iter() - - // Create batch for deletion - batch := i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) - batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) - - var payloadMHBz []byte - for iter.Scan(&payloadMHBz) { - // Add each delete operation to batch - batch.Entries = append(batch.Entries, gocql.BatchEntry{ - Stmt: delQry, - Args: []any{payloadMHBz, pieceCidBytes}, - Idempotent: true, - }) - - // Execute batch - if len(batch.Entries) >= i.settings.InsertBatchSize { - err := i.session.ExecuteBatch(batch) - if err != nil { - return xerrors.Errorf("executing batch delete for piece %s: %w", pieceCid, err) - } - // Create a new batch after executing - batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) - batch.Entries = make([]gocql.BatchEntry, 0, i.settings.InsertBatchSize) - } - } - - // Execute remaining operations in the batch - if len(batch.Entries) > 0 { - err := i.session.ExecuteBatch(batch) - if err != nil { - return xerrors.Errorf("executing batch delete for piece %s: %w", pieceCid, err) - } - } - - if err := iter.Close(); err != nil { - return xerrors.Errorf("Getting piece index for piece %s: %w", pieceCid, err) - } - - return nil -} - -// PiecesContainingMultihash gets all pieces that contain a multihash (used when retrieving by payload CID) -func (i *IndexStore) PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) { - var pcids []cid.Cid - var bz []byte - qry := `SELECT PieceCid FROM PayloadToPiece WHERE PayloadMultihash = ?` - iter := i.session.Query(qry, trimMultihash(m)).WithContext(ctx).Iter() - for iter.Scan(&bz) { - pcid, err := cid.Parse(bz) - if err != nil { - return nil, fmt.Errorf("parsing piece cid: %w", err) - } - pcids = append(pcids, pcid) - } - if err := iter.Close(); err != nil { - return nil, fmt.Errorf("getting pieces containing multihash %s: %w", m, err) - } - - // No pieces found for multihash, return a "not found" error - if len(pcids) == 0 { - return nil, normalizeMultihashError(m, ErrNotFound) - } - return pcids, nil -} diff --git a/lib/indexing/indexstore/indexstore_test.go b/lib/indexing/indexstore/indexstore_test.go deleted file mode 100644 index 79e174fb8..000000000 --- a/lib/indexing/indexstore/indexstore_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package indexstore - -import ( - "context" - "io" - "os" - "testing" - "time" - - carv2 "github.com/ipld/go-car/v2" - "github.com/ipld/go-car/v2/blockstore" - "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" - - "github.com/filecoin-project/go-commp-utils/writer" - - "github.com/filecoin-project/curio/deps/config" - "github.com/filecoin-project/curio/lib/testutils" -) - -func envElse(env, els string) string { - if v := os.Getenv(env); v != "" { - return v - } - return els -} - -func TestNewIndexStore(t *testing.T) { - // Set up the indexStore for testing - - ctx := context.Background() - cfg := config.DefaultCurioConfig() - - idxStore, err := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, cfg) - require.NoError(t, err) - - // Create a car file and calculate commP - dir, err := os.MkdirTemp(os.TempDir(), "curio-indexstore") - require.NoError(t, err) - defer func() { - _ = os.RemoveAll(dir) - }() - - rf, err := testutils.CreateRandomFile(dir, int(time.Now().Unix()), 8000000) - require.NoError(t, err) - - caropts := []carv2.Option{ - blockstore.WriteAsCarV1(true), - } - - _, cn, err := testutils.CreateDenseCARWith(dir, rf, 1024, 1024, caropts) - require.NoError(t, err) - - f, err := os.Open(cn) - require.NoError(t, err) - - defer func() { - _ = f.Close() - }() - - w := &writer.Writer{} - _, err = io.CopyBuffer(w, f, make([]byte, writer.CommPBuf)) - require.NoError(t, err) - - commp, err := w.Sum() - require.NoError(t, err) - - _, err = f.Seek(0, io.SeekStart) - require.NoError(t, err) - - // Create recods - dealCfg := cfg.Market.StorageMarketConfig - chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize - - recs := make(chan Record, chanSize) - opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} - blockReader, err := carv2.NewBlockReader(f, opts...) - require.NoError(t, err) - - // Add index to the store - var eg errgroup.Group - eg.Go(func() error { - serr := idxStore.AddIndex(ctx, commp.PieceCID, recs) - return serr - }) - - var m multihash.Multihash - i := 0 - - blockMetadata, err := blockReader.SkipNext() - for err == nil { - if i == 0 { - m = blockMetadata.Hash() - } - recs <- Record{ - Cid: blockMetadata.Cid, - OffsetSize: OffsetSize{ - Offset: blockMetadata.SourceOffset, - Size: blockMetadata.Size, - }, - } - i++ - - blockMetadata, err = blockReader.SkipNext() - } - require.Error(t, io.EOF) - close(recs) - err = eg.Wait() - require.NoError(t, err) - - // Try to find a multihash - pcids, err := idxStore.PiecesContainingMultihash(ctx, m) - require.NoError(t, err) - require.Len(t, pcids, 1) - require.Equal(t, pcids[0], commp.PieceCID) - - // Remove all indexes from the store - err = idxStore.RemoveIndexes(ctx, pcids[0]) - require.NoError(t, err) - - // Drop the table - err = idxStore.session.Query("DROP TABLE PayloadToPiece").Exec() - require.NoError(t, err) -} diff --git a/lib/ipni/chunker/chunker.go b/lib/ipni/chunker/chunker.go new file mode 100644 index 000000000..9971e5cef --- /dev/null +++ b/lib/ipni/chunker/chunker.go @@ -0,0 +1,159 @@ +package chunker + +import ( + "cmp" + "errors" + "fmt" + "io" + "slices" + + lru "github.com/hashicorp/golang-lru/v2" + logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-car/v2/index" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipni/go-libipni/ingest/schema" + "github.com/multiformats/go-multihash" + + "github.com/filecoin-project/curio/lib/ipni/ipniculib" +) + +var log = logging.Logger("chunker") + +const EntriesChunkSize = 16384 +const EntriesCacheCapacity = 4096 + +// Chunker chunks advertisement entries as a chained series of schema.EntryChunk nodes. +// See: NewChunker +type Chunker struct { + chunkSize int + cache *lru.Cache[ipld.Link, datamodel.Node] +} + +// NewChunker instantiates a new chain chunker that given a provider.MultihashIterator it drains +// all its mulithashes and stores them in the given link system represented as a chain of +// schema.EntryChunk nodes where each chunk contains no more than chunkSize number of multihashes. +// +// See: schema.EntryChunk. +func NewChunker(cache *lru.Cache[ipld.Link, datamodel.Node]) *Chunker { + chunker := &Chunker{ + chunkSize: EntriesChunkSize, + } + if cache != nil { + chunker.cache = cache + } + return chunker +} + +// Chunk chunks all the mulithashes returned by the given iterator into a chain of schema.EntryChunk +// nodes where each chunk contains no more than chunkSize number of multihashes and returns the link +// the root chunk node. +// +// See: schema.EntryChunk. +func (ls *Chunker) Chunk(mhi SliceMhIterator) (ipld.Link, error) { + mhs := make([]multihash.Multihash, 0, ls.chunkSize) + var next ipld.Link + var mhCount, chunkCount int + for { + mh, err := mhi.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + mhs = append(mhs, mh) + mhCount++ + if len(mhs) >= ls.chunkSize { + cNode, err := newEntriesChunkNode(mhs, next) + if err != nil { + return nil, err + } + next, err = ipniculib.NodeToLink(cNode, schema.Linkproto) + if err != nil { + return nil, err + } + if ls.cache != nil { + ls.cache.Add(next, cNode) + } + chunkCount++ + // NewLinkedListOfMhs makes it own copy, so safe to reuse mhs + mhs = mhs[:0] + } + } + if len(mhs) != 0 { + cNode, err := newEntriesChunkNode(mhs, next) + if err != nil { + return nil, err + } + next, err = ipniculib.NodeToLink(cNode, schema.Linkproto) + if err != nil { + return nil, err + } + chunkCount++ + } + + log.Infow("Generated linked chunks of multihashes", "totalMhCount", mhCount, "chunkCount", chunkCount) + return next, nil +} + +func newEntriesChunkNode(mhs []multihash.Multihash, next ipld.Link) (datamodel.Node, error) { + chunk := schema.EntryChunk{ + Entries: mhs, + } + if next != nil { + chunk.Next = next + } + return chunk.ToNode() +} + +// SliceMhIterator is a simple MultihashIterator implementation that +// iterates a slice of multihash.Multihash. +type SliceMhIterator struct { + mhs []multihash.Multihash + pos int +} + +type iteratorStep struct { + mh multihash.Multihash + offset uint64 +} + +// CarMultihashIterator constructs a new MultihashIterator from a CAR index. +// +// This iterator supplies multihashes in deterministic order of their +// corresponding CAR offset. The order is maintained consistently regardless of +// the underlying IterableIndex implementation. Returns error if duplicate +// offsets detected. +func CarMultihashIterator(idx index.IterableIndex) (*SliceMhIterator, error) { + var steps []iteratorStep + if err := idx.ForEach(func(mh multihash.Multihash, offset uint64) error { + steps = append(steps, iteratorStep{mh, offset}) + return nil + }); err != nil { + return nil, err + } + slices.SortFunc(steps, func(a, b iteratorStep) int { + return cmp.Compare(a.offset, b.offset) + }) + + var lastOffset uint64 + mhs := make([]multihash.Multihash, len(steps)) + for i := range steps { + if steps[i].offset == lastOffset { + return nil, fmt.Errorf("car multihash iterator has duplicate offset %d", steps[i].offset) + } + mhs[i] = steps[i].mh + } + return &SliceMhIterator{mhs: mhs}, nil +} + +// Next implements the MultihashIterator interface. +func (it *SliceMhIterator) Next() (multihash.Multihash, error) { + if it.pos >= len(it.mhs) { + return nil, io.EOF + } + mh := it.mhs[it.pos] + it.pos++ + return mh, nil +} diff --git a/lib/ipni/ipni-provider/ipni-provider.go b/lib/ipni/ipni-provider/ipni-provider.go new file mode 100644 index 000000000..06895e76c --- /dev/null +++ b/lib/ipni/ipni-provider/ipni-provider.go @@ -0,0 +1,577 @@ +package ipni_provider + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/filecoin-project/curio/lib/storiface" + "github.com/gorilla/mux" + lru "github.com/hashicorp/golang-lru/v2" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/index" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagjson" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipni/go-libipni/announce" + "github.com/ipni/go-libipni/announce/httpsender" + "github.com/ipni/go-libipni/dagsync/ipnisync/head" + "github.com/ipni/go-libipni/ingest/schema" + "github.com/ipni/go-libipni/metadata" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/ipni/chunker" + "github.com/filecoin-project/curio/lib/ipni/ipniculib" + "github.com/filecoin-project/curio/lib/pieceprovider" + + "github.com/filecoin-project/lotus/node/modules/dtypes" +) + +const IPNIPath = "/ipni/v1/ad/" +const ProviderPath = "/ipni-provider" + +var ( + log = logging.Logger("ipni-provider") + ErrNotFound = errors.New("not found") +) + +type ipniAPI interface { + StateNetworkName(context.Context) (dtypes.NetworkName, error) +} + +type peerInfo struct { + ID peer.ID + Key crypto.PrivKey +} + +type Provider struct { + api ipniAPI + db *harmonydb.DB + pieceProvider *pieceprovider.PieceProvider + entriesChunker *chunker.Chunker + cache *lru.Cache[ipld.Link, datamodel.Node] + topic string + httpPrefix string + keys map[string]*peerInfo // map[peerID String]Private_Key + // announceURLs enables sending direct announcements via HTTP. This is + // the list of indexer URLs to send direct HTTP announce messages to. + announceURLs []*url.URL + httpServerAddresses []multiaddr.Multiaddr +} + +func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { + c, err := lru.New[ipld.Link, datamodel.Node](chunker.EntriesCacheCapacity) + if err != nil { + return nil, xerrors.Errorf("creating new cache: %w", err) + } + + ctx := context.Background() + + keyMap := make(map[string]*peerInfo) + + rows, err := deps.DB.Query(ctx, `SELECT priv_key FROM libp2p`) + if err != nil { + return nil, xerrors.Errorf("failed to get private libp2p keys from DB: %w", err) + } + + for rows.Next() && rows.Err() == nil { + var priv []byte + err := rows.Scan(&priv) + if err != nil { + return nil, xerrors.Errorf("failed to scan the row: %w", err) + } + + pkey, err := crypto.UnmarshalPrivateKey(priv) + if err != nil { + return nil, xerrors.Errorf("unmarshaling private key: %w", err) + } + + id, err := peer.IDFromPublicKey(pkey.GetPublic()) + if err != nil { + return nil, xerrors.Errorf("generating peer ID from private key: %w", err) + } + + keyMap[id.String()] = &peerInfo{ + Key: pkey, + ID: id, + } + } + + if rows.Err() != nil { + return nil, err + } + + nn, err := api.StateNetworkName(ctx) + if err != nil { + return nil, err + } + + topic := "/indexer/ingest/" + string(nn) + + if deps.Cfg.Market.StorageMarketConfig.IPNI.TopicName != "" { + topic = deps.Cfg.Market.StorageMarketConfig.IPNI.TopicName + } + + var announceURLs []*url.URL + + for _, us := range deps.Cfg.Market.StorageMarketConfig.IPNI.DirectAnnounceURLs { + u, err := url.Parse(us) + if err != nil { + return nil, err + } + announceURLs = append(announceURLs, u) + } + + var httpServerAddresses []multiaddr.Multiaddr + + for _, a := range deps.Cfg.Market.HTTP.AnnounceAddresses { + addr, err := multiaddr.NewMultiaddr(a) + if err != nil { + return nil, err + } + httpServerAddresses = append(httpServerAddresses, addr) + } + + return &Provider{ + api: api, + db: deps.DB, + pieceProvider: deps.PieceProvider, + cache: c, + keys: keyMap, + topic: topic, + announceURLs: announceURLs, + httpServerAddresses: httpServerAddresses, + }, nil +} + +func (p *Provider) getAd(ctx context.Context, ad cid.Cid, provider string) (schema.Advertisement, error) { + var ads []struct { + PreviousID string + Provider string + Addresses string + Signature []byte + Entries string + ContextID []byte + IsRm bool + } + + err := p.db.Select(ctx, &ads, `SELECT context_id, + is_rm, previous, provider, addresses, + signature, entries FROM ipni WHERE ad_cid = $1 AND provider = $2`, ad.String(), provider) + + if err != nil { + return schema.Advertisement{}, xerrors.Errorf("getting ad from DB: %w", err) + } + + if len(ads) == 0 { + return schema.Advertisement{}, ErrNotFound + } + + if len(ads) > 1 { + return schema.Advertisement{}, xerrors.Errorf("expected 1 ad but got %d", len(ads)) + } + + a := ads[0] + + prev, err := cid.Parse(a.PreviousID) + if err != nil { + return schema.Advertisement{}, xerrors.Errorf("parsing previous CID: %w", err) + } + + e, err := cid.Parse(a.Entries) + if err != nil { + return schema.Advertisement{}, xerrors.Errorf("parsing entry CID: %w", err) + } + + mds := metadata.IpfsGatewayHttp{} + md, err := mds.MarshalBinary() + if err != nil { + return schema.Advertisement{}, xerrors.Errorf("marshalling metadata: %w", err) + } + + return schema.Advertisement{ + PreviousID: cidlink.Link{Cid: prev}, + Provider: a.Provider, + Addresses: strings.Split(a.Addresses, ","), + Signature: a.Signature, + Entries: cidlink.Link{Cid: e}, + ContextID: a.ContextID, + IsRm: a.IsRm, + Metadata: md, + }, nil +} + +func (p *Provider) getHead(ctx context.Context, provider string) ([]byte, error) { + var headStr string + err := p.db.QueryRow(ctx, `SELECT head FROM ipni_head WHERE provider = $1`, provider).Scan(&headStr) + if err != nil { + return nil, xerrors.Errorf("querying previous head: %w", err) + } + + if headStr == "" { + return nil, ErrNotFound + } + + ad, err := cid.Parse(headStr) + if err != nil { + return nil, err + } + + h, err := p.getAd(ctx, ad, provider) + if err != nil { + return nil, err + } + + hn, err := h.ToNode() + if err != nil { + return nil, err + } + + lnk, err := ipniculib.NodeToLink(hn, schema.Linkproto) + if err != nil { + return nil, err + } + + signedHead, err := head.NewSignedHead(lnk.(cidlink.Link).Cid, p.topic, p.keys[provider].Key) + if err != nil { + return nil, xerrors.Errorf("failed to generate signed head for peer %s: %w", provider, err) + } + + return signedHead.Encode() +} + +func (p *Provider) GetEntry(block cid.Cid, provider string) ([]byte, error) { + // We should use background context to avoid early exit + // while chunking as first attempt will always fail + ctx := context.Background() + + // Check if cache has it + cacheKey := cidlink.Link{Cid: block} + if value, ok := p.cache.Get(cacheKey); ok { + b := new(bytes.Buffer) + err := dagjson.Encode(value, b) + if err != nil { + return nil, err + } + return b.Bytes(), nil + } + + // If cache does not have it then this must be Entry Head + // We should find the relevant ContextID(abi.PieceInfo) and chunk the piece again + // to generate all the required links + + var pis []abi.PieceInfo + + row, err := p.db.Query(ctx, `SELECT context_id FROM ipni WHERE entries = $1 AND provider = $2`, block.String(), provider) + if err != nil { + return nil, xerrors.Errorf("querying ads with entry link %s: %w", block, err) + } + + for row.Next() && row.Err() == nil { + var contextID []byte + err := row.Scan(&contextID) + if err != nil { + return nil, xerrors.Errorf("failed to scan the row: %w", err) + } + + var pi abi.PieceInfo + err = pi.UnmarshalCBOR(bytes.NewReader(contextID)) + if err != nil { + return nil, xerrors.Errorf("unmarshaling piece info: %w", err) + } + + pis = append(pis, pi) + } + + if row.Err() != nil { + return nil, err + } + + type info struct { + SPID int64 `db:"sp_id"` + Sector abi.SectorNumber `db:"sector_num"` + Offset int64 `db:"piece_offset"` + Length int64 `db:"piece_length"` + RawSize int64 `db:"raw_size"` + Proof abi.RegisteredSealProof `db:"reg_seal_proof"` + } + + pieceMap := make(map[abi.PieceInfo][]info) + + for _, pi := range pis { + if _, ok := pieceMap[pi]; ok { + continue + } + + var infos []info + + err := p.db.Select(ctx, &infos, `SELECT + mpd.sp_id, + mpd.sector_num, + mpd.piece_offset, + mpd.piece_length, + mpd.raw_size, + sm.reg_seal_proof + FROM + market_piece_deal mpd + INNER JOIN + sectors_meta sm + ON + mpd.sp_id = sm.sp_id + AND mpd.sector_num = sm.sector_num + WHERE piece_cid = $1`, pi.PieceCID.String()) + + if err != nil { + return nil, xerrors.Errorf("getting deal info from database: %w", err) + } + pieceMap[pi] = infos + } + + // Chunk the piece and generate links + for pi, infos := range pieceMap { + for _, i := range infos { + unsealed, err := p.pieceProvider.IsUnsealed(ctx, storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(i.SPID), + Number: i.Sector, + }, + ProofType: i.Proof, + }, storiface.UnpaddedByteIndex(i.Offset), pi.Size.Unpadded()) + if err != nil { + return nil, xerrors.Errorf("checking if sector is unsealed :%w", err) + } + + if !unsealed { + continue + } + + reader, err := p.pieceProvider.ReadPiece(ctx, storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(i.SPID), + Number: i.Sector, + }, + ProofType: i.Proof, + }, storiface.UnpaddedByteIndex(i.Offset), abi.UnpaddedPieceSize(pi.Size), pi.PieceCID) + if err != nil { + return nil, xerrors.Errorf("getting piece reader: %w", err) + } + + var recs []index.Record + opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} + blockReader, err := carv2.NewBlockReader(reader, opts...) + if err != nil { + return nil, fmt.Errorf("getting block reader over piece: %w", err) + } + + blockMetadata, err := blockReader.SkipNext() + for err == nil { + recs = append(recs, index.Record{ + Cid: blockMetadata.Cid, + Offset: blockMetadata.Offset, + }) + blockMetadata, err = blockReader.SkipNext() + } + if !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("generating index for piece: %w", err) + } + + mis := make(index.MultihashIndexSorted) + err = mis.Load(recs) + + // To avoid - Cannot assert pinter to interface + idxF := func(sorted *index.MultihashIndexSorted) index.Index { + return sorted + } + + idx := idxF(&mis) + iterableIndex := idx.(index.IterableIndex) + + mhi, err := chunker.CarMultihashIterator(iterableIndex) + if err != nil { + return nil, xerrors.Errorf("getting CAR multihash iterator: %w", err) + } + + _, err = p.entriesChunker.Chunk(*mhi) + if err != nil { + return nil, xerrors.Errorf("chunking CAR multihash iterator: %w", err) + } + + if value, ok := p.cache.Get(cacheKey); ok { + b := new(bytes.Buffer) + err := dagjson.Encode(value, b) + if err != nil { + return nil, err + } + return b.Bytes(), nil + } + } + } + + return nil, ErrNotFound +} + +func (p *Provider) handleGet(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + pp := strings.TrimPrefix(r.URL.RawPath, p.httpPrefix+ProviderPath) + pps := strings.Split(pp, "/") + providerID := pps[0] + + req := strings.TrimPrefix(pp, IPNIPath) + switch req { + case "head": + sh, err := p.getHead(r.Context(), providerID) + if err != nil { + if errors.Is(err, ErrNotFound) { + http.Error(w, "", http.StatusNoContent) + return + } + log.Errorf("failed to get signed head for peer %s: %w", providerID, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + w.WriteHeader(http.StatusOK) + _, err = w.Write(sh) + if err != nil { + log.Errorw("failed to write HTTP response", "err", err) + } + default: + b, err := cid.Parse(req) + if err != nil { + log.Debugw("invalid CID as path parameter while getting content", "request", req, "err", err) + http.Error(w, "invalid CID: "+req, http.StatusBadRequest) + return + } + ad, err := p.getAd(r.Context(), b, providerID) + if err != nil { + if errors.Is(err, ErrNotFound) { + // Check if this is an entry CID + entry, err := p.GetEntry(b, providerID) + if err != nil { + if errors.Is(err, ErrNotFound) { + log.Debugw("No Content Found", "CID", b.String()) + http.Error(w, "", http.StatusNoContent) + return + } + log.Errorf("failed to get entry %s for peer %s: %w", b.String(), providerID, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + _, err = w.Write(entry) + if err != nil { + log.Errorw("failed to write HTTP response", "err", err) + } + return + } + log.Errorf("failed to get ad %s for peer %s: %w", b.String(), providerID, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + adn, err := ad.ToNode() + if err != nil { + log.Errorf("failed to convert ad %s for peer %s to IPLD node: %w", b.String(), providerID, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // Use local buffer for better error handing + resp := new(bytes.Buffer) + err = dagjson.Encode(adn, resp) + if err != nil { + log.Errorf("failed to encode ad %s for peer %s: %w", b.String(), providerID, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + _, err = w.Write(resp.Bytes()) + if err != nil { + log.Errorw("failed to write HTTP response", "err", err) + } + return + } +} + +func contentRouter(r *mux.Router, p *Provider) { + r.Methods("GET").Path("/*").HandlerFunc(p.handleGet) +} + +func Routes(r *mux.Router, p *Provider) { + contentRouter(r.PathPrefix("/ipni-provider").Subrouter(), p) +} + +func (p *Provider) StartPublishing(ctx context.Context) { + // A poller which publishes head for each provider + // every 10 minutes + ticker := time.NewTicker(10 * time.Minute) + go func() { + for { + select { + case <-ticker.C: + // Call the function to publish head for each provider + p.publishHead(ctx) + case <-ctx.Done(): + ticker.Stop() + return + } + } + }() +} + +func (p *Provider) getHeadCID(ctx context.Context, provider string) (cid.Cid, error) { + var headStr string + err := p.db.QueryRow(ctx, `SELECT head FROM ipni_head WHERE provider = $1`, provider).Scan(&headStr) + if err != nil { + return cid.Undef, xerrors.Errorf("querying previous head: %w", err) + } + + if headStr == "" { + return cid.Undef, ErrNotFound + } + + return cid.Parse(headStr) +} + +func (p *Provider) publishHead(ctx context.Context) { + for provider := range p.keys { + c, err := p.getHeadCID(ctx, provider) + if err != nil { + log.Errorw("failed to get head CID", "provider", provider, "error", err) + continue + } + err = p.publishhttp(ctx, c, provider) + if err != nil { + log.Errorw("failed to publish head for provide", "provider", provider, "error", err) + } + } +} + +func (p *Provider) publishhttp(ctx context.Context, adCid cid.Cid, peer string) error { + // Create the http announce sender. + httpSender, err := httpsender.New(p.announceURLs, p.keys[peer].ID) + if err != nil { + return fmt.Errorf("cannot create http announce sender: %w", err) + } + + log.Infow("Announcing advertisements over HTTP", "urls", p.announceURLs) + return announce.Send(ctx, adCid, p.httpServerAddresses, httpSender) +} diff --git a/lib/ipni/ipniculib/ipniculib.go b/lib/ipni/ipniculib/ipniculib.go new file mode 100644 index 000000000..e5971ef52 --- /dev/null +++ b/lib/ipni/ipniculib/ipniculib.go @@ -0,0 +1,18 @@ +package ipniculib + +import ( + "crypto/sha256" + + "github.com/ipld/go-ipld-prime/codec/dagjson" + "github.com/ipld/go-ipld-prime/datamodel" + "golang.org/x/xerrors" +) + +func NodeToLink(node datamodel.Node, lp datamodel.LinkPrototype) (datamodel.Link, error) { + hasher := sha256.New() + err := dagjson.Encode(node, hasher) + if err != nil { + return nil, xerrors.Errorf("failed to encode: %w", err) + } + return lp.BuildLink(hasher.Sum(nil)), nil +} diff --git a/market/mk12/mk12.go b/market/mk12/mk12.go index 65cd50b71..b21b56edd 100644 --- a/market/mk12/mk12.go +++ b/market/mk12/mk12.go @@ -425,19 +425,19 @@ func (m *MK12) processDeal(ctx context.Context, deal *ProviderDealState) (*Provi Opaque: fmt.Sprintf("%d", refID), } - _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, url, raw_size, should_index) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (uuid) DO NOTHING`, + _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, url, raw_size, should_index, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (uuid) DO NOTHING`, deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.IsOffline, pieceIDUrl, deal.Transfer.Size, - deal.FastRetrieval) + deal.FastRetrieval, deal.AnnounceToIPNI) if err != nil { return false, xerrors.Errorf("inserting deal into deal pipeline: %w", err) } } else { // Insert the offline deal into the deal pipeline - _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, should_index) - VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (uuid) DO NOTHING`, - deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.IsOffline, deal.FastRetrieval) + _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, should_index, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (uuid) DO NOTHING`, + deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.IsOffline, deal.FastRetrieval, deal.AnnounceToIPNI) if err != nil { return false, xerrors.Errorf("inserting deal into deal pipeline: %w", err) } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index a505f98c2..9aa796c57 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -62,6 +62,7 @@ type itask struct { ChainID abi.DealID `db:"chain_deal_id"` RawSize int64 `db:"raw_size"` ShouldIndex bool `db:"should_index"` + Announce bool `db:"announce"` } func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { @@ -80,7 +81,8 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do reg_seal_proof, chain_deal_id, raw_size, - should_index + should_index, + announce FROM market_mk12_deal_pipeline WHERE diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go new file mode 100644 index 000000000..3ab7e54f1 --- /dev/null +++ b/tasks/indexing/task_ipni.go @@ -0,0 +1,451 @@ +package indexing + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/filecoin-project/curio/market/indexstore" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/index" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipni/go-libipni/ingest/schema" + "github.com/ipni/go-libipni/metadata" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/ipni/chunker" + "github.com/filecoin-project/curio/lib/ipni/ipniculib" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/pieceprovider" + "github.com/filecoin-project/curio/lib/storiface" +) + +var ilog = logging.Logger("ipni") + +type IPNITask struct { + db *harmonydb.DB + indexStore *indexstore.IndexStore + pieceProvider *pieceprovider.PieceProvider + sc *ffi.SealCalls + cfg *config.CurioConfig +} + +func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.PieceProvider, cfg *config.CurioConfig) *IPNITask { + + return &IPNITask{ + db: db, + indexStore: indexStore, + pieceProvider: pieceProvider, + sc: sc, + cfg: cfg, + } +} + +func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var tasks []struct { + SPID int64 `db:"sp_id"` + Sector abi.SectorNumber `db:"sector"` + Proof abi.RegisteredSealProof `db:"reg_seal_proof"` + Offset int64 `db:"sector_offset"` + CTXID []byte `db:"context_id"` + RM bool `db:"is_rm"` + Prov string `db:"provider"` + } + + err = I.db.Select(ctx, &tasks, `SELECT + sp_id, + sector, + reg_seal_proof, + sector_offset, + context_id, + is_rm, + provider + FROM + ipni_task + WHERE + task_id = $1;`, taskID) + if err != nil { + return false, xerrors.Errorf("getting ipni task params: %w", err) + } + + if len(tasks) != 1 { + return false, xerrors.Errorf("expected 1 ipni task params, got %d", len(tasks)) + } + + task := tasks[0] + + var pi abi.PieceInfo + err = pi.UnmarshalCBOR(bytes.NewReader(task.CTXID)) + if err != nil { + return false, xerrors.Errorf("unmarshaling piece info: %w", err) + } + + unsealed, err := I.pieceProvider.IsUnsealed(ctx, storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(task.SPID), + Number: task.Sector, + }, + ProofType: task.Proof, + }, storiface.UnpaddedByteIndex(task.Offset), pi.Size.Unpadded()) + if err != nil { + return false, xerrors.Errorf("checking if sector is unsealed :%w", err) + } + + if !unsealed { + return false, xerrors.Errorf("sector %d for miner %d is not unsealed", task.Sector, task.SPID) + } + + reader, err := I.pieceProvider.ReadPiece(ctx, storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(task.SPID), + Number: task.Sector, + }, + ProofType: task.Proof, + }, storiface.UnpaddedByteIndex(task.Offset), abi.UnpaddedPieceSize(pi.Size), pi.PieceCID) + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } + + var recs []index.Record + opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} + blockReader, err := carv2.NewBlockReader(reader, opts...) + if err != nil { + return false, fmt.Errorf("getting block reader over piece: %w", err) + } + + blockMetadata, err := blockReader.SkipNext() + for err == nil { + recs = append(recs, index.Record{ + Cid: blockMetadata.Cid, + Offset: blockMetadata.Offset, + }) + blockMetadata, err = blockReader.SkipNext() + } + if !errors.Is(err, io.EOF) { + return false, fmt.Errorf("generating index for piece: %w", err) + } + + mis := make(index.MultihashIndexSorted) + err = mis.Load(recs) + + // To avoid - Cannot assert pinter to interface + idxF := func(sorted *index.MultihashIndexSorted) index.Index { + return sorted + } + + idx := idxF(&mis) + iterableIndex := idx.(index.IterableIndex) + + mhi, err := chunker.CarMultihashIterator(iterableIndex) + if err != nil { + return false, xerrors.Errorf("getting CAR multihash iterator: %w", err) + } + + lnk, err := chunker.NewChunker(nil).Chunk(*mhi) + if err != nil { + return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) + } + + _, err = I.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var prev string + err = tx.QueryRow(`SELECT head FROM ipni_head WHERE provider = $1`, task.Prov).Scan(&prev) + if err != nil { + return false, xerrors.Errorf("querying previous head: %w", err) + } + + prevCID, err := cid.Parse(prev) + if err != nil { + return false, xerrors.Errorf("parsing previous CID: %w", err) + } + + mds := metadata.IpfsGatewayHttp{} + md, err := mds.MarshalBinary() + if err != nil { + return false, xerrors.Errorf("marshaling metadata: %w", err) + } + + var privKey []byte + err = tx.QueryRow(`SELECT priv_key FROM libp2p WHERE sp_id = $1`, task.SPID).Scan(&privKey) + if err != nil { + return false, xerrors.Errorf("failed to get private libp2p key for miner %d: %w", task.SPID, err) + } + + pkey, err := crypto.UnmarshalPrivateKey(privKey) + if err != nil { + return false, xerrors.Errorf("unmarshaling private key: %w", err) + } + + adv := schema.Advertisement{ + PreviousID: cidlink.Link{Cid: prevCID}, + Provider: task.Prov, + Addresses: make([]string, 0), + Entries: lnk, + ContextID: task.CTXID, + Metadata: md, + IsRm: task.RM, + } + + err = adv.Sign(pkey) + if err != nil { + return false, xerrors.Errorf("signing the advertisement: %w", err) + } + + err = adv.Validate() + if err != nil { + return false, xerrors.Errorf("validating the advertisement: %w", err) + } + + adNode, err := adv.ToNode() + if err != nil { + return false, xerrors.Errorf("converting advertisement to node: %w", err) + } + + ad, err := ipniculib.NodeToLink(adNode, schema.Linkproto) + if err != nil { + return false, xerrors.Errorf("converting advertisement to link: %w", err) + } + + n, err := I.db.Exec(ctx, `SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7)`, + ad.(cidlink.Link).Cid.String(), adv.ContextID, adv.IsRm, adv.Provider, adv.Addresses, + adv.Signature, adv.Entries.String()) + + if err != nil { + return false, xerrors.Errorf("adding advertisement to the database: %w", err) + } + + if n != 1 { + return false, xerrors.Errorf("updated %d rows", n) + } + + return true, nil + + }) + if err != nil { + return false, xerrors.Errorf("store IPNI success: %w", err) + } + + return true, nil +} + +func (I *IPNITask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + var tasks []struct { + TaskID harmonytask.TaskID `db:"task_id"` + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + StorageID string `db:"storage_id"` + } + + if storiface.FTUnsealed != 1 { + panic("storiface.FTUnsealed != 1") + } + + ctx := context.Background() + + indIDs := make([]int64, len(ids)) + for i, id := range ids { + indIDs[i] = int64(id) + } + + err := I.db.Select(ctx, &tasks, ` + SELECT dp.indexing_task_id, dp.sp_id, dp.sector_number, l.storage_id FROM market_mk12_deal_pipeline dp + INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector_number = l.sector_num + WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1 +`, indIDs) + if err != nil { + return nil, xerrors.Errorf("getting tasks: %w", err) + } + + ls, err := I.sc.LocalStorage(ctx) + if err != nil { + return nil, xerrors.Errorf("getting local storage: %w", err) + } + + acceptables := map[harmonytask.TaskID]bool{} + + for _, t := range ids { + acceptables[t] = true + } + + for _, t := range tasks { + if _, ok := acceptables[t.TaskID]; !ok { + continue + } + + for _, l := range ls { + if string(l.ID) == t.StorageID { + return &t.TaskID, nil + } + } + } + + return nil, nil +} + +func (I *IPNITask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Name: "IPNI", + Cost: resources.Resources{ + Cpu: 1, + Ram: 8 << 30, // 8 GiB for the most dense cars + }, + MaxFailures: 3, + IAmBored: passcall.Every(5*time.Minute, func(taskFunc harmonytask.AddTaskFunc) error { + return I.schedule(context.Background(), taskFunc) + }), + } +} + +func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + // schedule submits + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var pendings []itask + + err := I.db.Select(ctx, &pendings, `SELECT + uuid, + sp_id, + sector_number, + piece_cid, + piece_size, + piece_offset, + reg_seal_proof, + chain_deal_id, + raw_size, + should_index, + announce + FROM market_mk12_deal_pipeline + WHERE sealed = TRUE + AND indexed = TRUE + AND complete = FALSE + ORDER BY indexing_created_at ASC;`) + if err != nil { + return false, xerrors.Errorf("getting pending IPNI announcing tasks: %w", err) + } + + if len(pendings) == 0 { + return false, nil + } + + p := pendings[0] + + if !p.Announce { + n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID) + if err != nil { + return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("store IPNI success: updated %d rows", n) + } + stop = false // we found a task to schedule, keep going + return true, nil + } + + var privKey []byte + err = tx.QueryRow(`SELECT priv_key FROM libp2p WHERE sp_id = $1`, p.SpID).Scan(&privKey) + if err != nil { + return false, xerrors.Errorf("failed to get private libp2p key for miner %d: %w", p.SpID, err) + } + + pkey, err := crypto.UnmarshalPrivateKey(privKey) + if err != nil { + return false, xerrors.Errorf("unmarshaling private key: %w", err) + } + + pubK := pkey.GetPublic() + pid, err := peer.IDFromPublicKey(pubK) + if err != nil { + return false, fmt.Errorf("getting peer ID: %w", err) + } + + pcid, err := cid.Parse(p.PieceCid) + if err != nil { + return false, xerrors.Errorf("parsing piece CID: %w", err) + } + + pi := abi.PieceInfo{ + PieceCID: pcid, + Size: p.Size, + } + + b := new(bytes.Buffer) + err = pi.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8)`, p.SpID, + p.Sector, p.Proof, p.Offset, b.Bytes(), false, pid.String(), id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + ilog.Infof("Another IPNI announce task already present for piece %s in deal %s", p.PieceCid, p.UUID) + // SET "complete" status to true for this deal, so it is not considered next time + n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID) + if err != nil { + return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("store IPNI success: updated %d rows", n) + } + // We should commit this transaction to set the value + stop = false // we found a task to schedule, keep going + return true, nil + } + if strings.Contains(err.Error(), "already published") { + ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.UUID) + // SET "complete" status to true for this deal, so it is not considered next time + n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID) + if err != nil { + return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("store IPNI success: updated %d rows", n) + } + // We should commit this transaction to set the value + stop = false // we found a task to schedule, keep going + return true, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + } + + return nil +} + +func (I *IPNITask) Adder(taskFunc harmonytask.AddTaskFunc) {} + +func (I *IPNITask) GetSpid(db *harmonydb.DB, taskID int64) string { + var spid string + err := db.QueryRow(context.Background(), `SELECT sp_id FROM ipni_task WHERE task_id = $1`, taskID).Scan(&spid) + if err != nil { + log.Errorf("getting spid: %s", err) + return "" + } + return spid +} + +var _ = harmonytask.Reg(&IPNITask{}) +var _ harmonytask.TaskInterface = &IPNITask{} From 30296cafdc58f8b2c6d94db609ca450dc62216ce Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 30 Aug 2024 19:53:43 +0530 Subject: [PATCH 10/15] fix storeiface, publisher multiaddrs --- deps/config/types.go | 6 +-- lib/ipni/ipni-provider/ipni-provider.go | 35 ++++++++++++++-- lib/ipni/ipniculib/ipniculib.go | 13 ++---- lib/urltomultiaddr/urltomultiaddr.go | 53 +++++++++++++++++++++++++ tasks/indexing/task_ipni.go | 5 ++- 5 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 lib/urltomultiaddr/urltomultiaddr.go diff --git a/deps/config/types.go b/deps/config/types.go index 24a7e2040..8838949d8 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -72,7 +72,7 @@ func DefaultCurioConfig() *CurioConfig { }, Market: MarketConfig{ HTTP: HTTPConfig{ - ListenAddress: "0.0.0.0:8888", + ListenAddress: "0.0.0.0:12400", AnnounceAddresses: []string{}, }, StorageMarketConfig: StorageMarketConfig{ @@ -686,11 +686,11 @@ type IPNIConfig struct { } type HTTPConfig struct { - // ListenAddress is where HTTP server will be listening on + // ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400" ListenAddress string // AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node. // Curio allows running more than one node for HTTP server and thus all addressed can be announced - // simultaneously to the client + // simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"] AnnounceAddresses []string } diff --git a/lib/ipni/ipni-provider/ipni-provider.go b/lib/ipni/ipni-provider/ipni-provider.go index 06895e76c..66cb84410 100644 --- a/lib/ipni/ipni-provider/ipni-provider.go +++ b/lib/ipni/ipni-provider/ipni-provider.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/filecoin-project/curio/lib/storiface" "github.com/gorilla/mux" lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" @@ -39,10 +38,13 @@ import ( "github.com/filecoin-project/curio/lib/ipni/chunker" "github.com/filecoin-project/curio/lib/ipni/ipniculib" "github.com/filecoin-project/curio/lib/pieceprovider" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/urltomultiaddr" "github.com/filecoin-project/lotus/node/modules/dtypes" ) +const IPNIRoutePath = "/ipni-provider" const IPNIPath = "/ipni/v1/ad/" const ProviderPath = "/ipni-provider" @@ -141,7 +143,11 @@ func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { var httpServerAddresses []multiaddr.Multiaddr for _, a := range deps.Cfg.Market.HTTP.AnnounceAddresses { - addr, err := multiaddr.NewMultiaddr(a) + addr, err := urltomultiaddr.UrlToMultiaddr(a) + if err != nil { + return nil, err + } + addr, err = multiaddr.NewMultiaddr(addr.String() + IPNIRoutePath) if err != nil { return nil, err } @@ -393,6 +399,9 @@ func (p *Provider) GetEntry(block cid.Cid, provider string) ([]byte, error) { mis := make(index.MultihashIndexSorted) err = mis.Load(recs) + if err != nil { + return nil, xerrors.Errorf("failed to load indexed in multihash sorter: %w", err) + } // To avoid - Cannot assert pinter to interface idxF := func(sorted *index.MultihashIndexSorted) index.Index { @@ -516,7 +525,7 @@ func contentRouter(r *mux.Router, p *Provider) { } func Routes(r *mux.Router, p *Provider) { - contentRouter(r.PathPrefix("/ipni-provider").Subrouter(), p) + contentRouter(r.PathPrefix(IPNIRoutePath).Subrouter(), p) } func (p *Provider) StartPublishing(ctx context.Context) { @@ -572,6 +581,24 @@ func (p *Provider) publishhttp(ctx context.Context, adCid cid.Cid, peer string) return fmt.Errorf("cannot create http announce sender: %w", err) } + addrs, err := p.getHTTPAddressForPeer(peer) + if err != nil { + return fmt.Errorf("cannot create provider http addresses: %w", err) + } + log.Infow("Announcing advertisements over HTTP", "urls", p.announceURLs) - return announce.Send(ctx, adCid, p.httpServerAddresses, httpSender) + return announce.Send(ctx, adCid, addrs, httpSender) +} + +func (p *Provider) getHTTPAddressForPeer(peer string) ([]multiaddr.Multiaddr, error) { + var ret []multiaddr.Multiaddr + for _, addr := range p.httpServerAddresses { + a, err := multiaddr.NewMultiaddr(addr.String() + "/" + peer) + if err != nil { + return nil, err + } + ret = append(ret, a) + } + + return ret, nil } diff --git a/lib/ipni/ipniculib/ipniculib.go b/lib/ipni/ipniculib/ipniculib.go index e5971ef52..714416238 100644 --- a/lib/ipni/ipniculib/ipniculib.go +++ b/lib/ipni/ipniculib/ipniculib.go @@ -1,18 +1,11 @@ package ipniculib import ( - "crypto/sha256" - - "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" - "golang.org/x/xerrors" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) func NodeToLink(node datamodel.Node, lp datamodel.LinkPrototype) (datamodel.Link, error) { - hasher := sha256.New() - err := dagjson.Encode(node, hasher) - if err != nil { - return nil, xerrors.Errorf("failed to encode: %w", err) - } - return lp.BuildLink(hasher.Sum(nil)), nil + linkSystem := cidlink.DefaultLinkSystem() + return linkSystem.ComputeLink(lp, node) } diff --git a/lib/urltomultiaddr/urltomultiaddr.go b/lib/urltomultiaddr/urltomultiaddr.go new file mode 100644 index 000000000..dfed65ea0 --- /dev/null +++ b/lib/urltomultiaddr/urltomultiaddr.go @@ -0,0 +1,53 @@ +package urltomultiaddr + +import ( + "errors" + "fmt" + "net" + "net/url" + + "github.com/multiformats/go-multiaddr" +) + +const protocol = "tcp" + +func UrlToMultiaddr(urlStr string) (multiaddr.Multiaddr, error) { + // Parse the URL + parsedURL, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + + // Extract the host and port + host := parsedURL.Hostname() + port := parsedURL.Port() + if port == "" { + // Default port based on the scheme + switch parsedURL.Scheme { + case "https": + port = "443" + case "http": + port = "80" + default: + return nil, errors.New("invalid URL scheme") + } + } + + // Determine if the host is an IP address or a DNS name + var addrStr string + if ip := net.ParseIP(host); ip != nil { + // It's an IP address, check if it's IPv4 or IPv6 + if ip.To4() != nil { + // IPv4 + addrStr = fmt.Sprintf("/ip4/%s/%s/%s/%s", ip.String(), protocol, port, parsedURL.Scheme) + } else { + // IPv6 + addrStr = fmt.Sprintf("/ip6/%s/%s/%s/%s", ip.String(), protocol, port, parsedURL.Scheme) + } + } else { + // It's a DNS name + addrStr = fmt.Sprintf("/dns/%s/%s/%s/%s", host, protocol, port, parsedURL.Scheme) + } + + return multiaddr.NewMultiaddr(addrStr) +} diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 3ab7e54f1..afd424b45 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/filecoin-project/curio/market/indexstore" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" carv2 "github.com/ipld/go-car/v2" @@ -33,6 +32,7 @@ import ( "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/indexstore" ) var ilog = logging.Logger("ipni") @@ -144,6 +144,9 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b mis := make(index.MultihashIndexSorted) err = mis.Load(recs) + if err != nil { + return false, xerrors.Errorf("failed to load indexed in multihash sorter: %w", err) + } // To avoid - Cannot assert pinter to interface idxF := func(sorted *index.MultihashIndexSorted) index.Index { From 22ef709e27fa97ca9a0d7bb682a666064d4a8160 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 30 Aug 2024 20:08:58 +0530 Subject: [PATCH 11/15] close rows --- lib/ipni/ipni-provider/ipni-provider.go | 12 ++++++++---- web/api/webrpc/sync_state.go | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/ipni/ipni-provider/ipni-provider.go b/lib/ipni/ipni-provider/ipni-provider.go index 66cb84410..f66281cf0 100644 --- a/lib/ipni/ipni-provider/ipni-provider.go +++ b/lib/ipni/ipni-provider/ipni-provider.go @@ -92,6 +92,8 @@ func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { return nil, xerrors.Errorf("failed to get private libp2p keys from DB: %w", err) } + defer rows.Close() + for rows.Next() && rows.Err() == nil { var priv []byte err := rows.Scan(&priv) @@ -284,14 +286,16 @@ func (p *Provider) GetEntry(block cid.Cid, provider string) ([]byte, error) { var pis []abi.PieceInfo - row, err := p.db.Query(ctx, `SELECT context_id FROM ipni WHERE entries = $1 AND provider = $2`, block.String(), provider) + rows, err := p.db.Query(ctx, `SELECT context_id FROM ipni WHERE entries = $1 AND provider = $2`, block.String(), provider) if err != nil { return nil, xerrors.Errorf("querying ads with entry link %s: %w", block, err) } - for row.Next() && row.Err() == nil { + defer rows.Close() + + for rows.Next() && rows.Err() == nil { var contextID []byte - err := row.Scan(&contextID) + err := rows.Scan(&contextID) if err != nil { return nil, xerrors.Errorf("failed to scan the row: %w", err) } @@ -305,7 +309,7 @@ func (p *Provider) GetEntry(block cid.Cid, provider string) ([]byte, error) { pis = append(pis, pi) } - if row.Err() != nil { + if rows.Err() != nil { return nil, err } diff --git a/web/api/webrpc/sync_state.go b/web/api/webrpc/sync_state.go index 81a381173..7bfc1cfa1 100644 --- a/web/api/webrpc/sync_state.go +++ b/web/api/webrpc/sync_state.go @@ -43,6 +43,8 @@ func (a *WebRPC) loadConfigs(ctx context.Context) (map[string]string, error) { return nil, xerrors.Errorf("getting db configs: %w", err) } + defer rows.Close() + configs := make(map[string]string) for rows.Next() { var title, config string From 70ed4bf45b51072850a621bdf5431b631f5e17ad Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 30 Aug 2024 21:45:12 +0530 Subject: [PATCH 12/15] apply suggestions --- deps/config/doc_gen.go | 27 +++++---- deps/config/types.go | 28 +++++----- .../default-curio-configuration.md | 27 +++++---- lib/ipni/chunker/chunker.go | 15 ++--- lib/ipni/ipni-provider/ipni-provider.go | 56 +++++++++---------- tasks/indexing/task_ipni.go | 15 +++-- 6 files changed, 85 insertions(+), 83 deletions(-) diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 413438246..7b15ad7b3 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -761,7 +761,7 @@ on a sinle node. Enabling on multiple nodes will cause issues with libp2p deals. Name: "ListenAddress", Type: "string", - Comment: `ListenAddress is where HTTP server will be listening on`, + Comment: `ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400"`, }, { Name: "AnnounceAddresses", @@ -769,31 +769,34 @@ on a sinle node. Enabling on multiple nodes will cause issues with libp2p deals. Comment: `AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node. Curio allows running more than one node for HTTP server and thus all addressed can be announced -simultaneously to the client`, +simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"]`, }, }, "IPNIConfig": { { - Name: "Enable", + Name: "Disable", Type: "bool", - Comment: `Enable set whether to enable indexing announcement to the network and expose endpoints that -allow indexer nodes to process announcements. Enabled by default.`, + Comment: `Disable set whether to disable indexing announcement to the network and expose endpoints that +allow indexer nodes to process announcements. Default: False`, }, { - Name: "TopicName", - Type: "string", + Name: "EntriesCacheCapacity", + Type: "int", - Comment: `TopicName sets the topic name on which the changes to the advertised content are announced. -If not explicitly specified, the topic name is automatically inferred from the network name -in following format: '/indexer/ingest/' -Defaults to empty, which implies the topic name is inferred from network name.`, + Comment: `EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement +entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The +maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and +the length of multihashes being advertised. For example, advertising 128-bit long multihashes +with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to +1GiB when full.`, }, { Name: "WebHost", Type: "string", - Comment: `The network indexer host that the web UI should link to for published announcements`, + Comment: `The network indexer host that the web UI should link to for published announcements +TODO: should we use this for checking published heas before publishing? Later commit`, }, { Name: "DirectAnnounceURLs", diff --git a/deps/config/types.go b/deps/config/types.go index 8838949d8..b3a686d4e 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -95,10 +95,9 @@ func DefaultCurioConfig() *CurioConfig { ExpectedSnapSealDuration: Duration(2 * time.Hour), }, IPNI: IPNIConfig{ - Enable: true, - TopicName: "", - WebHost: "https://cid.contact", - DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"}, + EntriesCacheCapacity: 4096, + WebHost: "https://cid.contact", + DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"}, }, }, }, @@ -668,17 +667,20 @@ type Libp2pConfig struct { } type IPNIConfig struct { - // Enable set whether to enable indexing announcement to the network and expose endpoints that - // allow indexer nodes to process announcements. Enabled by default. - Enable bool - - // TopicName sets the topic name on which the changes to the advertised content are announced. - // If not explicitly specified, the topic name is automatically inferred from the network name - // in following format: '/indexer/ingest/' - // Defaults to empty, which implies the topic name is inferred from network name. - TopicName string + // Disable set whether to disable indexing announcement to the network and expose endpoints that + // allow indexer nodes to process announcements. Default: False + Disable bool + + // EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement + // entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The + // maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and + // the length of multihashes being advertised. For example, advertising 128-bit long multihashes + // with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to + // 1GiB when full. + EntriesCacheCapacity int // The network indexer host that the web UI should link to for published announcements + // TODO: should we use this for checking published heas before publishing? Later commit WebHost string // The list of URLs of indexing nodes to announce to. diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 53d43c90d..c35b1a14c 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -229,7 +229,6 @@ description: The default curio configuration # type: bool #EnableBatchSeal = false -<<<<<<< HEAD # EnableDealMarket enabled the deal market on the node. This would also enable libp2p on the node, if configured. # # type: bool @@ -237,9 +236,6 @@ description: The default curio configuration # EnableCommP enables the commP task on te node. CommP is calculated before sending PublishDealMessage for a Mk12 deal # Must have EnableDealMarket = True -======= - # EnableCommP enabled the commP task on te node. CommP is calculated before sending PublishDealMessage for a Mk12 deal ->>>>>>> 563f95c (ipni provider) # # type: bool #EnableCommP = false @@ -436,21 +432,24 @@ description: The default curio configuration #InsertConcurrency = 8 [Market.StorageMarketConfig.IPNI] - # Enable set whether to enable indexing announcement to the network and expose endpoints that - # allow indexer nodes to process announcements. Enabled by default. + # Disable set whether to disable indexing announcement to the network and expose endpoints that + # allow indexer nodes to process announcements. Default: False # # type: bool - #Enable = true - - # TopicName sets the topic name on which the changes to the advertised content are announced. - # If not explicitly specified, the topic name is automatically inferred from the network name - # in following format: '/indexer/ingest/' - # Defaults to empty, which implies the topic name is inferred from network name. + #Disable = false + + # EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement + # entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The + # maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and + # the length of multihashes being advertised. For example, advertising 128-bit long multihashes + # with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to + # 1GiB when full. # - # type: string - #TopicName = "" + # type: int + #EntriesCacheCapacity = 4096 # The network indexer host that the web UI should link to for published announcements + # TODO: should we use this for checking published heas before publishing? Later commit # # type: string #WebHost = "https://cid.contact" diff --git a/lib/ipni/chunker/chunker.go b/lib/ipni/chunker/chunker.go index 9971e5cef..4731169f2 100644 --- a/lib/ipni/chunker/chunker.go +++ b/lib/ipni/chunker/chunker.go @@ -20,8 +20,7 @@ import ( var log = logging.Logger("chunker") -const EntriesChunkSize = 16384 -const EntriesCacheCapacity = 4096 +const entriesChunkSize = 16384 // Chunker chunks advertisement entries as a chained series of schema.EntryChunk nodes. // See: NewChunker @@ -36,13 +35,10 @@ type Chunker struct { // // See: schema.EntryChunk. func NewChunker(cache *lru.Cache[ipld.Link, datamodel.Node]) *Chunker { - chunker := &Chunker{ - chunkSize: EntriesChunkSize, + return &Chunker{ + chunkSize: entriesChunkSize, + cache: cache, } - if cache != nil { - chunker.cache = cache - } - return chunker } // Chunk chunks all the mulithashes returned by the given iterator into a chain of schema.EntryChunk @@ -63,7 +59,6 @@ func (ls *Chunker) Chunk(mhi SliceMhIterator) (ipld.Link, error) { return nil, err } mhs = append(mhs, mh) - mhCount++ if len(mhs) >= ls.chunkSize { cNode, err := newEntriesChunkNode(mhs, next) if err != nil { @@ -77,6 +72,7 @@ func (ls *Chunker) Chunk(mhi SliceMhIterator) (ipld.Link, error) { ls.cache.Add(next, cNode) } chunkCount++ + mhCount += len(mhs) // NewLinkedListOfMhs makes it own copy, so safe to reuse mhs mhs = mhs[:0] } @@ -91,6 +87,7 @@ func (ls *Chunker) Chunk(mhi SliceMhIterator) (ipld.Link, error) { return nil, err } chunkCount++ + mhCount += len(mhs) } log.Infow("Generated linked chunks of multihashes", "totalMhCount", mhCount, "chunkCount", chunkCount) diff --git a/lib/ipni/ipni-provider/ipni-provider.go b/lib/ipni/ipni-provider/ipni-provider.go index f66281cf0..bfd685632 100644 --- a/lib/ipni/ipni-provider/ipni-provider.go +++ b/lib/ipni/ipni-provider/ipni-provider.go @@ -47,6 +47,7 @@ import ( const IPNIRoutePath = "/ipni-provider" const IPNIPath = "/ipni/v1/ad/" const ProviderPath = "/ipni-provider" +const publishInterval = 10 * time.Minute var ( log = logging.Logger("ipni-provider") @@ -68,7 +69,6 @@ type Provider struct { pieceProvider *pieceprovider.PieceProvider entriesChunker *chunker.Chunker cache *lru.Cache[ipld.Link, datamodel.Node] - topic string httpPrefix string keys map[string]*peerInfo // map[peerID String]Private_Key // announceURLs enables sending direct announcements via HTTP. This is @@ -78,7 +78,7 @@ type Provider struct { } func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { - c, err := lru.New[ipld.Link, datamodel.Node](chunker.EntriesCacheCapacity) + c, err := lru.New[ipld.Link, datamodel.Node](deps.Cfg.Market.StorageMarketConfig.IPNI.EntriesCacheCapacity) if err != nil { return nil, xerrors.Errorf("creating new cache: %w", err) } @@ -121,30 +121,19 @@ func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { return nil, err } - nn, err := api.StateNetworkName(ctx) - if err != nil { - return nil, err - } - - topic := "/indexer/ingest/" + string(nn) - - if deps.Cfg.Market.StorageMarketConfig.IPNI.TopicName != "" { - topic = deps.Cfg.Market.StorageMarketConfig.IPNI.TopicName - } - - var announceURLs []*url.URL + announceURLs := make([]*url.URL, 0, len(deps.Cfg.Market.StorageMarketConfig.IPNI.DirectAnnounceURLs)) - for _, us := range deps.Cfg.Market.StorageMarketConfig.IPNI.DirectAnnounceURLs { + for i, us := range deps.Cfg.Market.StorageMarketConfig.IPNI.DirectAnnounceURLs { u, err := url.Parse(us) if err != nil { return nil, err } - announceURLs = append(announceURLs, u) + announceURLs[i] = u } - var httpServerAddresses []multiaddr.Multiaddr + httpServerAddresses := make([]multiaddr.Multiaddr, 0, len(deps.Cfg.Market.HTTP.AnnounceAddresses)) - for _, a := range deps.Cfg.Market.HTTP.AnnounceAddresses { + for i, a := range deps.Cfg.Market.HTTP.AnnounceAddresses { addr, err := urltomultiaddr.UrlToMultiaddr(a) if err != nil { return nil, err @@ -153,7 +142,7 @@ func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { if err != nil { return nil, err } - httpServerAddresses = append(httpServerAddresses, addr) + httpServerAddresses[i] = addr } return &Provider{ @@ -162,7 +151,6 @@ func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) { pieceProvider: deps.PieceProvider, cache: c, keys: keyMap, - topic: topic, announceURLs: announceURLs, httpServerAddresses: httpServerAddresses, }, nil @@ -179,9 +167,17 @@ func (p *Provider) getAd(ctx context.Context, ad cid.Cid, provider string) (sche IsRm bool } - err := p.db.Select(ctx, &ads, `SELECT context_id, - is_rm, previous, provider, addresses, - signature, entries FROM ipni WHERE ad_cid = $1 AND provider = $2`, ad.String(), provider) + err := p.db.Select(ctx, &ads, `SELECT + context_id, + is_rm, + previous, + provider, + addresses, + signature, + entries + FROM ipni + WHERE ad_cid = $1 + AND provider = $2`, ad.String(), provider) if err != nil { return schema.Advertisement{}, xerrors.Errorf("getting ad from DB: %w", err) @@ -256,7 +252,7 @@ func (p *Provider) getHead(ctx context.Context, provider string) ([]byte, error) return nil, err } - signedHead, err := head.NewSignedHead(lnk.(cidlink.Link).Cid, p.topic, p.keys[provider].Key) + signedHead, err := head.NewSignedHead(lnk.(cidlink.Link).Cid, "", p.keys[provider].Key) if err != nil { return nil, xerrors.Errorf("failed to generate signed head for peer %s: %w", provider, err) } @@ -461,7 +457,7 @@ func (p *Provider) handleGet(w http.ResponseWriter, r *http.Request) { return } log.Errorf("failed to get signed head for peer %s: %w", providerID, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "", http.StatusInternalServerError) } w.WriteHeader(http.StatusOK) _, err = w.Write(sh) @@ -487,7 +483,7 @@ func (p *Provider) handleGet(w http.ResponseWriter, r *http.Request) { return } log.Errorf("failed to get entry %s for peer %s: %w", b.String(), providerID, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "", http.StatusInternalServerError) return } w.WriteHeader(http.StatusOK) @@ -498,13 +494,13 @@ func (p *Provider) handleGet(w http.ResponseWriter, r *http.Request) { return } log.Errorf("failed to get ad %s for peer %s: %w", b.String(), providerID, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "", http.StatusInternalServerError) return } adn, err := ad.ToNode() if err != nil { log.Errorf("failed to convert ad %s for peer %s to IPLD node: %w", b.String(), providerID, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "", http.StatusInternalServerError) return } // Use local buffer for better error handing @@ -512,7 +508,7 @@ func (p *Provider) handleGet(w http.ResponseWriter, r *http.Request) { err = dagjson.Encode(adn, resp) if err != nil { log.Errorf("failed to encode ad %s for peer %s: %w", b.String(), providerID, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "", http.StatusInternalServerError) return } w.WriteHeader(http.StatusOK) @@ -535,7 +531,7 @@ func Routes(r *mux.Router, p *Provider) { func (p *Provider) StartPublishing(ctx context.Context) { // A poller which publishes head for each provider // every 10 minutes - ticker := time.NewTicker(10 * time.Minute) + ticker := time.NewTicker(publishInterval) go func() { for { select { diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index afd424b45..f6a09ee6e 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -64,8 +64,8 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b Sector abi.SectorNumber `db:"sector"` Proof abi.RegisteredSealProof `db:"reg_seal_proof"` Offset int64 `db:"sector_offset"` - CTXID []byte `db:"context_id"` - RM bool `db:"is_rm"` + CtxID []byte `db:"context_id"` + Rm bool `db:"is_rm"` Prov string `db:"provider"` } @@ -92,7 +92,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b task := tasks[0] var pi abi.PieceInfo - err = pi.UnmarshalCBOR(bytes.NewReader(task.CTXID)) + err = pi.UnmarshalCBOR(bytes.NewReader(task.CtxID)) if err != nil { return false, xerrors.Errorf("unmarshaling piece info: %w", err) } @@ -200,9 +200,9 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b Provider: task.Prov, Addresses: make([]string, 0), Entries: lnk, - ContextID: task.CTXID, + ContextID: task.CtxID, Metadata: md, - IsRm: task.RM, + IsRm: task.Rm, } err = adv.Sign(pkey) @@ -316,6 +316,11 @@ func (I *IPNITask) TypeDetails() harmonytask.TaskTypeDetails { } func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + // If IPNI is disabled then don't schedule any tasks + if I.cfg.Market.StorageMarketConfig.IPNI.Disable { + return nil + } + // schedule submits var stop bool for !stop { From e2d9c9a856c235192ae42c73fc815e6cd7f32c66 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 30 Aug 2024 22:28:54 +0530 Subject: [PATCH 13/15] fix error --- harmony/harmonydb/sql/20240823-ipni.sql | 2 +- lib/paths/local_test.go | 6 ++++-- lib/paths/remote_test.go | 6 ++++-- tasks/indexing/task_indexing.go | 24 ++++++++++++++++++------ tasks/indexing/task_ipni.go | 2 ++ 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql index 6d4761c7d..56835d57d 100644 --- a/harmony/harmonydb/sql/20240823-ipni.sql +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -105,7 +105,7 @@ CREATE TABLE ipni_task ( reg_seal_proof INT NOT NULL, sector_offset BIGINT, - context_id BYTEA NOT NULL PRIMARY KEY, + context_id BYTEA NOT NULL, is_rm BOOLEAN NOT NULL, provider TEXT NOT NULL, diff --git a/lib/paths/local_test.go b/lib/paths/local_test.go index 92c9b5074..4edcef6a9 100644 --- a/lib/paths/local_test.go +++ b/lib/paths/local_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/curio/harmony/harmonydb" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/storage/sealer/fsutil" ) @@ -82,7 +82,9 @@ func TestLocalStorage(t *testing.T) { root: root, } - db, err := harmonydb.NewFromConfigWithITestID(t, "testlocalstorage") + sharedITestID := harmonydb.ITestNewID() + + db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) require.NoError(t, err) index := NewDBIndex(nil, db) diff --git a/lib/paths/remote_test.go b/lib/paths/remote_test.go index 7602eb116..af397d00f 100644 --- a/lib/paths/remote_test.go +++ b/lib/paths/remote_test.go @@ -25,7 +25,7 @@ import ( "github.com/filecoin-project/curio/lib/partialfile" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/paths/mocks" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) const metaFile = "sectorstore.json" @@ -59,7 +59,9 @@ func createTestStorage(t *testing.T, p string, seal bool, att ...*paths.Local) s func TestMoveShared(t *testing.T) { logging.SetAllLoggers(logging.LevelDebug) - db, err := harmonydb.NewFromConfigWithITestID(t, "testlocalstorage") + sharedITestID := harmonydb.ITestNewID() + + db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) require.NoError(t, err) index := paths.NewDBIndex(nil, db) diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 9aa796c57..a53d9ff51 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -214,13 +214,25 @@ func (i *IndexingTask) recordCompletion(ctx context.Context, task itask, taskID return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err) } - n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, complete = TRUE WHERE uuid = $1`, task.UUID) - if err != nil { - return xerrors.Errorf("store indexing success: updating pipeline: %w", err) - } - if n != 1 { - return xerrors.Errorf("store indexing success: updated %d rows", n) + // If IPNI is disabled then mark deal as complete otherwise just mark as indexed + if i.cfg.Market.StorageMarketConfig.IPNI.Disable { + n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, complete = TRUE WHERE uuid = $1`, task.UUID) + if err != nil { + return xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("store indexing success: updated %d rows", n) + } + } else { + n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE WHERE uuid = $1`, task.UUID) + if err != nil { + return xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("store indexing success: updated %d rows", n) + } } + return nil } diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index f6a09ee6e..730878350 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -317,6 +317,8 @@ func (I *IPNITask) TypeDetails() harmonytask.TaskTypeDetails { func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { // If IPNI is disabled then don't schedule any tasks + // Deals should already be marked as complete by task_indexing + // This check is to cover any edge case if I.cfg.Market.StorageMarketConfig.IPNI.Disable { return nil } From 32af2142aa755a736be9a0cf05b27e817185df50 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 6 Sep 2024 12:43:04 +0530 Subject: [PATCH 14/15] fix contextID type, move ipni in market --- harmony/harmonydb/sql/20240823-ipni.sql | 8 ++++---- {lib => market}/ipni/chunker/chunker.go | 3 +-- {lib => market}/ipni/ipni-provider/ipni-provider.go | 4 ++-- {lib => market}/ipni/ipniculib/ipniculib.go | 0 tasks/indexing/task_ipni.go | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) rename {lib => market}/ipni/chunker/chunker.go (98%) rename {lib => market}/ipni/ipni-provider/ipni-provider.go (99%) rename {lib => market}/ipni/ipniculib/ipniculib.go (100%) diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql index 56835d57d..04dbb93bf 100644 --- a/harmony/harmonydb/sql/20240823-ipni.sql +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -2,7 +2,7 @@ CREATE TABLE ipni ( order_number BIGSERIAL PRIMARY KEY, -- Unique increasing order number ad_cid TEXT NOT NULL, - context_id TEXT NOT NULL, -- abi.PieceInfo in Curio + context_id BYTEA NOT NULL, -- abi.PieceInfo in Curio -- metadata column in not required as Curio only supports one type of metadata(HTTP) is_rm BOOLEAN NOT NULL, @@ -39,7 +39,7 @@ CREATE TABLE ipni_head ( CREATE OR REPLACE FUNCTION insert_ad_and_update_head( _ad_cid TEXT, - _context_id TEXT, + _context_id BYTEA, _is_rm BOOLEAN, _provider TEXT, _addresses TEXT, @@ -71,7 +71,7 @@ CREATE OR REPLACE FUNCTION get_ad_chain( _ad_cid TEXT ) RETURNS TABLE ( ad_cid TEXT, - context_id TEXT, + context_id BYTEA, is_rm BOOLEAN, previous TEXT, provider TEXT, @@ -98,7 +98,7 @@ END; $$ LANGUAGE plpgsql; -- IPNI pipeline is kept separate from rest for robustness --- and reuse. This allows for removing, recreating as using CLI. +-- and reuse. This allows for removing, recreating ads using CLI. CREATE TABLE ipni_task ( sp_id BIGINT NOT NULL, sector BIGINT NOT NULL, diff --git a/lib/ipni/chunker/chunker.go b/market/ipni/chunker/chunker.go similarity index 98% rename from lib/ipni/chunker/chunker.go rename to market/ipni/chunker/chunker.go index 4731169f2..d9a32d1e4 100644 --- a/lib/ipni/chunker/chunker.go +++ b/market/ipni/chunker/chunker.go @@ -7,6 +7,7 @@ import ( "io" "slices" + "github.com/filecoin-project/curio/market/ipni/ipniculib" lru "github.com/hashicorp/golang-lru/v2" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-car/v2/index" @@ -14,8 +15,6 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipni/go-libipni/ingest/schema" "github.com/multiformats/go-multihash" - - "github.com/filecoin-project/curio/lib/ipni/ipniculib" ) var log = logging.Logger("chunker") diff --git a/lib/ipni/ipni-provider/ipni-provider.go b/market/ipni/ipni-provider/ipni-provider.go similarity index 99% rename from lib/ipni/ipni-provider/ipni-provider.go rename to market/ipni/ipni-provider/ipni-provider.go index bfd685632..beb2626b4 100644 --- a/lib/ipni/ipni-provider/ipni-provider.go +++ b/market/ipni/ipni-provider/ipni-provider.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/filecoin-project/curio/market/ipni/chunker" + "github.com/filecoin-project/curio/market/ipni/ipniculib" "github.com/gorilla/mux" lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" @@ -35,8 +37,6 @@ import ( "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/harmony/harmonydb" - "github.com/filecoin-project/curio/lib/ipni/chunker" - "github.com/filecoin-project/curio/lib/ipni/ipniculib" "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/urltomultiaddr" diff --git a/lib/ipni/ipniculib/ipniculib.go b/market/ipni/ipniculib/ipniculib.go similarity index 100% rename from lib/ipni/ipniculib/ipniculib.go rename to market/ipni/ipniculib/ipniculib.go diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 730878350..b6502b5ac 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -27,12 +27,12 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/ffi" - "github.com/filecoin-project/curio/lib/ipni/chunker" - "github.com/filecoin-project/curio/lib/ipni/ipniculib" "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/ipni/chunker" + "github.com/filecoin-project/curio/market/ipni/ipniculib" ) var ilog = logging.Logger("ipni") From 4d22e9a5aa27345a5ba72d14a99e37628cb8f41e Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 17 Sep 2024 21:36:59 +0400 Subject: [PATCH 15/15] gen, remove old market sql --- cmd/curio/guidedsetup/shared_cbor_gen.go | 1 - deps/config/doc_gen.go | 9 +- .../default-curio-configuration.md | 18 +- documentation/en/curio-cli/curio.md | 20 - .../sql/20240730-market-migration.sql | 362 ------------------ lib/paths/mocks/pf.go | 3 +- lib/paths/mocks/store.go | 6 +- market/ipni/chunker/chunker.go | 3 +- market/ipni/ipni-provider/ipni-provider.go | 4 +- tasks/storage-market/task_commp.go | 2 +- tasks/storage-market/task_find_deal.go | 2 +- 11 files changed, 24 insertions(+), 406 deletions(-) delete mode 100644 harmony/harmonydb/sql/20240730-market-migration.sql diff --git a/cmd/curio/guidedsetup/shared_cbor_gen.go b/cmd/curio/guidedsetup/shared_cbor_gen.go index 1f2effd07..c88b9e31f 100644 --- a/cmd/curio/guidedsetup/shared_cbor_gen.go +++ b/cmd/curio/guidedsetup/shared_cbor_gen.go @@ -10,7 +10,6 @@ import ( storiface "github.com/filecoin-project/curio/lib/storiface" sector "github.com/filecoin-project/curio/lib/types/sector" - abi "github.com/filecoin-project/go-state-types/abi" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 7b15ad7b3..7dc32400d 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -712,6 +712,7 @@ also be bounded by resources available on the machine.`, { Name: "EnableDealMarket", Type: "bool", + Comment: `EnableDealMarket enabled the deal market on the node. This would also enable libp2p on the node, if configured.`, }, { @@ -896,14 +897,6 @@ This will be used to fail the deals which cannot be sealed on time.`, Comment: `SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain Warning: If this check is skipped and there is a commP mismatch, all deals in the -sector will need to be sent again`, - }, - { - Name: "SkipCommP", - Type: "bool", - - Comment: `SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain -Warning: If this check is skipped and there is a commP mismatch, all deals in the sector will need to be sent again`, }, }, diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index c35b1a14c..056b02def 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -521,12 +521,18 @@ description: The default curio configuration # type: []string #NoAnnounceAddresses = [] - # SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain - # Warning: If this check is skipped and there is a commP mismatch, all deals in the - # sector will need to be sent again - # - # type: bool - #SkipCommP = false + [Market.HTTP] + # ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400" + # + # type: string + #ListenAddress = "0.0.0.0:12400" + + # AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node. + # Curio allows running more than one node for HTTP server and thus all addressed can be announced + # simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"] + # + # type: []string + #AnnounceAddresses = [] [Ingest] diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index 48932e662..f46bbf73c 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -639,15 +639,9 @@ USAGE: curio market command [command options] [arguments...] COMMANDS: -<<<<<<< HEAD seal start sealing a deal sector early add-url Add URL to fetch data for offline deals help, h Shows a list of commands or help for one command -======= - seal start sealing a deal sector early - import-data Import data for offline deal - help, h Shows a list of commands or help for one command ->>>>>>> 48e953d (poller redesign) OPTIONS: --help, -h show help @@ -667,7 +661,6 @@ OPTIONS: --help, -h show help ``` -<<<<<<< HEAD ### curio market add-url ``` NAME: @@ -681,19 +674,6 @@ OPTIONS: --header HEADER, -H HEADER [ --header HEADER, -H HEADER ] Custom HEADER to include in the HTTP request --url URL, -u URL URL to send the request to --help, -h show help -======= -### curio market import-data -``` -NAME: - curio market import-data - Import data for offline deal - -USAGE: - curio market import-data [command options] - -OPTIONS: - --actor value Specify actor address to start sealing sectors for - --help, -h show help ->>>>>>> 48e953d (poller redesign) ``` ## curio fetch-params diff --git a/harmony/harmonydb/sql/20240730-market-migration.sql b/harmony/harmonydb/sql/20240730-market-migration.sql deleted file mode 100644 index 39fa0a768..000000000 --- a/harmony/harmonydb/sql/20240730-market-migration.sql +++ /dev/null @@ -1,362 +0,0 @@ --- Table for Mk12 or Boost deals -CREATE TABLE market_mk12_deals ( - uuid TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', - sp_id BIGINT NOT NULL, - - signed_proposal_cid TEXT NOT NULL, - proposal_signature BYTEA NOT NULL, - proposal jsonb NOT NULL, - - offline BOOLEAN NOT NULL, - verified BOOLEAN NOT NULL, - - start_epoch BIGINT NOT NULL, - end_epoch BIGINT NOT NULL, - - client_peer_id TEXT NOT NULL, - - chain_deal_id BIGINT DEFAULT NULL, - publish_cid TEXT DEFAULT NULL, - - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, - - fast_retrieval BOOLEAN NOT NULL, - announce_to_ipni BOOLEAN NOT NULL, - - url TEXT DEFAULT NULL, - url_headers jsonb NOT NULL DEFAULT '{}', - - error TEXT DEFAULT NULL, - - primary key (uuid, sp_id, piece_cid, signed_proposal_cid), - unique (uuid), - unique (signed_proposal_cid) -); - --- This table is used for storing piece metadata (piece indexing) -CREATE TABLE market_piece_metadata ( - piece_cid TEXT NOT NULL PRIMARY KEY, - - version INT NOT NULL DEFAULT 2, - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', - - indexed BOOLEAN NOT NULL DEFAULT FALSE, - indexed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC', - - constraint market_piece_meta_identity_key - unique (piece_cid) -); - --- This table binds the piece metadata to specific deals (piece indexing) -CREATE TABLE market_piece_deal ( - id TEXT NOT NULL, -- (UUID for new deals, PropCID for old) - piece_cid TEXT NOT NULL, - - boost_deal BOOLEAN NOT NULL, - legacy_deal BOOLEAN NOT NULL DEFAULT FALSE, - - chain_deal_id BIGINT NOT NULL DEFAULT 0, - - sp_id BIGINT NOT NULL, - sector_num BIGINT NOT NULL, - - piece_offset BIGINT NOT NULL, - piece_length BIGINT NOT NULL, - raw_size BIGINT NOT NULL, - - primary key (sp_id, piece_cid, deal), - constraint market_piece_deal_identity_key - unique (sp_id, id) -); - --- This function is used to insert piece metadata and piece deal (piece indexing) -CREATE OR REPLACE FUNCTION process_piece_deal( - _id TEXT, - _piece_cid TEXT, - _boost_deal BOOLEAN, - _legacy_deal BOOLEAN DEFAULT FALSE, - _chain_deal_id BIGINT DEFAULT 0, - _sp_id BIGINT, - _sector_num BIGINT, - _piece_offset BIGINT, - _piece_length BIGINT, - _raw_size BIGINT -) -RETURNS VOID AS $$ -BEGIN - -- Update or insert into market_piece_metadata -INSERT INTO market_piece_metadata (piece_cid, indexed, indexed_at) -VALUES (_piece_cid, TRUE, CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - ON CONFLICT (piece_cid) DO UPDATE - SET indexed = TRUE, - indexed_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - --- Insert into market_piece_deal -INSERT INTO market_piece_deal ( - id, piece_cid, boost_deal, legacy_deal, chain_deal_id, - sp_id, sector_num, piece_offset, piece_length, raw_size -) -VALUES ( - _id, _piece_cid, _boost_deal, _legacy_deal, _chain_deal_id, - _sp_id, _sector_num, _piece_offset, _piece_length, _raw_size - ) - ON CONFLICT (sp_id, piece_cid, id) DO NOTHING; -END; -$$ LANGUAGE plpgsql; - --- Storage Ask for ask protocol -CREATE TABLE market_mk12_storage_ask ( - sp_id BIGINT NOT NULL, - - price BIGINT NOT NULL, - verified_price BIGINT NOT NULL, - - min_size BIGINT NOT NULL, - max_size BIGINT NOT NULL, - - created_at BIGINT NOT NULL, - expiry BIGINT NOT NULL, - - sequence BIGINT NOT NULL, - unique (sp_id) -); - --- Used for processing Mk12 deals -CREATE TABLE market_mk12_deal_pipeline ( - uuid TEXT NOT NULL, - sp_id BIGINT NOT NULL, - - started BOOLEAN DEFAULT FALSE, - - piece_cid TEXT NOT NULL, - piece_size BOOLEAN NOT NULL, - raw_size BIGINT DEFAULT NULL, - - offline BOOLEAN NOT NULL, - - url TEXT DEFAULT NULL, - headers jsonb NOT NULL DEFAULT '{}', - - commp_task_id BIGINT DEFAULT NULL, - after_commp BOOLEAN DEFAULT FALSE, - - psd_task_id BIGINT DEFAULT NULL, - after_psd BOOLEAN DEFAULT FALSE, - - psd_wait_time TIMESTAMPTZ, - - find_deal_task_id BIGINT DEFAULT NULL, - after_find_deal BOOLEAN DEFAULT FALSE, - - sector BIGINT, - reg_seal_proof INT NOT NULL, - sector_offset BIGINT, - - sealed BOOLEAN DEFAULT FALSE, - - should_index BOOLEAN DEFAULT FALSE, - indexing_created_at TIMESTAMPTZ, - indexing_task_id BIGINT DEFAULT NULL, - indexed BOOLEAN DEFAULT FALSE, - - complete BOOLEAN NOT NULL DEFAULT FALSE, - - constraint market_mk12_deal_pipeline_identity_key unique (uuid) -); - --- This function creates indexing task based from move_storage tasks -CREATE OR REPLACE FUNCTION create_indexing_task(task_id BIGINT, sealing_table TEXT) -RETURNS VOID AS $$ -DECLARE -query TEXT; -- Holds the dynamic SQL query - pms RECORD; -- Holds each row returned by the query in the loop -BEGIN - -- Construct the dynamic SQL query based on the sealing_table - IF sealing_table = 'sectors_sdr_pipeline' THEN - query := format( - 'SELECT - dp.uuid, - ssp.reg_seal_proof - FROM - %I ssp - JOIN - market_mk12_deal_pipeline dp ON ssp.sp_id = dp.sp_id AND ssp.sector_num = dp.sector - WHERE - ssp.task_id_move_storage = $1', sealing_table); - ELSIF sealing_table = 'sectors_snap_pipeline' THEN - query := format( - 'SELECT - dp.uuid, - (SELECT reg_seal_proof FROM sectors_meta WHERE sp_id = ssp.sp_id AND sector_num = ssp.sector_num) AS reg_seal_proof - FROM - %I ssp - JOIN - market_mk12_deal_pipeline dp ON ssp.sp_id = dp.sp_id AND ssp.sector_num = dp.sector - WHERE - ssp.task_id_move_storage = $1', sealing_table); -ELSE - RAISE EXCEPTION 'Invalid sealing_table name: %', sealing_table; -END IF; - - -- Execute the dynamic SQL query with the task_id parameter -FOR pms IN EXECUTE query USING task_id - LOOP - -- Update the market_mk12_deal_pipeline table with the reg_seal_proof and indexing_created_at values -UPDATE market_mk12_deal_pipeline -SET - reg_seal_proof = pms.reg_seal_proof, - indexing_created_at = NOW() AT TIME ZONE 'UTC' -WHERE - uuid = pms.uuid; -END LOOP; - - -- If everything is successful, simply exit - RETURN; - -EXCEPTION - WHEN OTHERS THEN - -- Rollback the transaction and raise the exception for Go to catch - ROLLBACK; - RAISE EXCEPTION 'Failed to create indexing task: %', SQLERRM; -END; -$$ LANGUAGE plpgsql; - --- This table can be used to track remote piece for offline deals --- The entries must be created by users -CREATE TABLE market_offline_urls ( - uuid TEXT NOT NULL, - - url TEXT NOT NULL, - headers jsonb NOT NULL DEFAULT '{}', - - raw_size BIGINT NOT NULL, - - CONSTRAINT market_offline_urls_uuid_fk FOREIGN KEY (uuid) - REFERENCES market_mk12_deal_pipeline (uuid) - ON DELETE CASCADE, - CONSTRAINT market_offline_urls_uuid_unique UNIQUE (uuid) -); - --- This table is used for coordinating libp2p nodes -CREATE TABLE libp2p ( - sp_id BIGINT NOT NULL, - priv_key BYTEA NOT NULL, - listen_address TEXT NOT NULL, - announce_address TEXT NOT NULL, - no_announce_address TEXT NOT NULL, -); - -CREATE TABLE direct_deals ( - id TEXT, - created_at TIMESTAMPTZ, - piece_cid TEXT, - piece_size BIGINT, - cleanup_data BOOLEAN, - client_address TEXT, - provider_address TEXT, - allocation_id BIGINT, - start_epoch BIGINT, - end_epoch BIGINT, - inbound_file_path TEXT, - inbound_file_size BIGINT, - sector_id BIGINT, - offset BIGINT, - length BIGINT, - announce_to_ipni BOOLEAN, - keep_unsealed_copy BOOLEAN -); - --- -- Function used to update the libp2p table -CREATE OR REPLACE FUNCTION insert_or_update_libp2p( - _sp_id BIGINT, - _listen_address TEXT, - _announce_address TEXT, - _no_announce_address TEXT, - _running_on TEXT -) -RETURNS BYTEA AS $$ -DECLARE -_priv_key BYTEA; - _current_running_on TEXT; - _current_updated_at TIMESTAMPTZ; -BEGIN - -- Check if the sp_id exists and retrieve the current values - SELECT priv_key, running_on, updated_at INTO _priv_key, _current_running_on, _current_updated_at - FROM libp2p - WHERE sp_id = _sp_id; - - -- Raise an exception if no row was found - IF NOT FOUND THEN - RAISE EXCEPTION 'libp2p key for sp_id "%" does not exist', _sp_id; - END IF; - - -- If the sp_id exists and running_on is NULL or matches _running_on - IF _current_running_on IS NULL OR _current_running_on = _running_on THEN - -- Update the record with the provided values and set updated_at to NOW - UPDATE libp2p - SET - listen_address = _listen_address, - announce_address = _announce_address, - no_announce_address = _no_announce_address, - running_on = _running_on, - updated_at = NOW() AT TIME ZONE 'UTC' - WHERE sp_id = _sp_id; - ELSIF _current_updated_at > NOW() - INTERVAL '10 seconds' THEN - -- Raise an exception if running_on is different and updated_at is recent - RAISE EXCEPTION 'Libp2p node already running on "%"', _current_running_on; - ELSE - -- Update running_on and other columns if updated_at is older than 10 seconds - UPDATE libp2p - SET - listen_address = _listen_address, - announce_address = _announce_address, - no_announce_address = _no_announce_address, - running_on = _running_on, - updated_at = NOW() AT TIME ZONE 'UTC' - WHERE sp_id = _sp_id; - END IF; - -RETURN _priv_key; -END; -$$ LANGUAGE plpgsql; - --- Add host column to allow local file based --- piece park -ALTER TABLE parked_piece_refs - ADD COLUMN host text; - --- Table for old lotus market deals. This is just for deal --- which are still alive. It should not be used for any processing -CREATE TABLE market_legacy_deals ( - signed_proposal_cid TEXT NOT NULL, - sp_id BIGINT NOT NULL, - client_peer_id TEXT NOT NULL, - - proposal_signature BYTEA NOT NULL, - proposal jsonb NOT NULL, - - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, - - offline BOOLEAN NOT NULL, - verified BOOLEAN NOT NULL, - - start_epoch BIGINT NOT NULL, - end_epoch BIGINT NOT NULL, - - publish_cid TEXT NOT NULL, - chain_deal_id BIGINT NOT NULL, - - fast_retrieval BOOLEAN NOT NULL, - - created_at TIMESTAMPTZ NOT NULL, - sector_num BIGINT NOT NULL, - - primary key (sp_id, piece_cid, signed_proposal_cid) -); - - - - diff --git a/lib/paths/mocks/pf.go b/lib/paths/mocks/pf.go index cb0bb126e..072e47b2e 100644 --- a/lib/paths/mocks/pf.go +++ b/lib/paths/mocks/pf.go @@ -8,11 +8,12 @@ import ( io "io" reflect "reflect" + gomock "github.com/golang/mock/gomock" + abi "github.com/filecoin-project/go-state-types/abi" partialfile "github.com/filecoin-project/curio/lib/partialfile" storiface "github.com/filecoin-project/curio/lib/storiface" - gomock "github.com/golang/mock/gomock" ) // MockPartialFileHandler is a mock of PartialFileHandler interface. diff --git a/lib/paths/mocks/store.go b/lib/paths/mocks/store.go index b44366c8d..e7c3c184c 100644 --- a/lib/paths/mocks/store.go +++ b/lib/paths/mocks/store.go @@ -8,14 +8,14 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + cid "github.com/ipfs/go-cid" + abi "github.com/filecoin-project/go-state-types/abi" storiface "github.com/filecoin-project/curio/lib/storiface" fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" - - gomock "github.com/golang/mock/gomock" - cid "github.com/ipfs/go-cid" ) // MockStore is a mock of Store interface. diff --git a/market/ipni/chunker/chunker.go b/market/ipni/chunker/chunker.go index d9a32d1e4..21ff1010e 100644 --- a/market/ipni/chunker/chunker.go +++ b/market/ipni/chunker/chunker.go @@ -7,7 +7,6 @@ import ( "io" "slices" - "github.com/filecoin-project/curio/market/ipni/ipniculib" lru "github.com/hashicorp/golang-lru/v2" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-car/v2/index" @@ -15,6 +14,8 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipni/go-libipni/ingest/schema" "github.com/multiformats/go-multihash" + + "github.com/filecoin-project/curio/market/ipni/ipniculib" ) var log = logging.Logger("chunker") diff --git a/market/ipni/ipni-provider/ipni-provider.go b/market/ipni/ipni-provider/ipni-provider.go index beb2626b4..0e745cfbf 100644 --- a/market/ipni/ipni-provider/ipni-provider.go +++ b/market/ipni/ipni-provider/ipni-provider.go @@ -11,8 +11,6 @@ import ( "strings" "time" - "github.com/filecoin-project/curio/market/ipni/chunker" - "github.com/filecoin-project/curio/market/ipni/ipniculib" "github.com/gorilla/mux" lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" @@ -40,6 +38,8 @@ import ( "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/urltomultiaddr" + "github.com/filecoin-project/curio/market/ipni/chunker" + "github.com/filecoin-project/curio/market/ipni/ipniculib" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/tasks/storage-market/task_commp.go b/tasks/storage-market/task_commp.go index bfa34c810..a4eea3151 100644 --- a/tasks/storage-market/task_commp.go +++ b/tasks/storage-market/task_commp.go @@ -9,7 +9,6 @@ import ( "net/url" "strconv" - "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -22,6 +21,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/storiface" diff --git a/tasks/storage-market/task_find_deal.go b/tasks/storage-market/task_find_deal.go index 5a681a89b..e71e601f6 100644 --- a/tasks/storage-market/task_find_deal.go +++ b/tasks/storage-market/task_find_deal.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" - "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -20,6 +19,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/promise" "github.com/filecoin-project/lotus/api"