From c6f85886bf260f521500ab7336ece3ad845c67dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 2 Dec 2020 00:32:01 +0100 Subject: [PATCH 1/3] miner: Add slow mode to proving check --- api/api_storage.go | 2 +- api/apistruct/struct.go | 6 ++-- cmd/lotus-storage-miner/proving.go | 6 +++- extern/sector-storage/faults.go | 54 ++++++++++++++++++++++++++++-- node/impl/storminer.go | 19 +++++++++-- storage/wdpost_run.go | 2 +- 6 files changed, 79 insertions(+), 10 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 2ddbb9d1261..5d0ec73a772 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -118,7 +118,7 @@ type StorageMiner interface { // the path specified when calling CreateBackup is within the base path CreateBackup(ctx context.Context, fpath string) error - CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) } type SealRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b1f569efbd1..3cf0647e371 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -364,7 +364,7 @@ type StorageMinerStruct struct { CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` - CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) `perm:"admin"` + CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) `perm:"admin"` } } @@ -1516,8 +1516,8 @@ func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) err return c.Internal.CreateBackup(ctx, fpath) } -func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) { - return c.Internal.CheckProvable(ctx, pp, sectors) +func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) { + return c.Internal.CheckProvable(ctx, pp, sectors, expensive) } // WorkerStruct diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index b930476e67a..5d176a754f1 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -386,6 +386,10 @@ var provingCheckProvableCmd = &cli.Command{ Usage: "print only bad sectors", Value: false, }, + &cli.BoolFlag{ + Name: "slow", + Usage: "run slower checks", + }, }, Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 1 { @@ -459,7 +463,7 @@ var provingCheckProvableCmd = &cli.Command{ }) } - bad, err := sapi.CheckProvable(ctx, pf, tocheck) + bad, err := sapi.CheckProvable(ctx, pf, tocheck, cctx.Bool("slow")) if err != nil { return err } diff --git a/extern/sector-storage/faults.go b/extern/sector-storage/faults.go index 7af81f0021f..0fd385250a9 100644 --- a/extern/sector-storage/faults.go +++ b/extern/sector-storage/faults.go @@ -2,13 +2,18 @@ package sectorstorage import ( "context" + "crypto/rand" "fmt" "os" "path/filepath" "golang.org/x/xerrors" + "github.com/ipfs/go-cid" + + ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" @@ -16,11 +21,13 @@ import ( // FaultTracker TODO: Track things more actively type FaultTracker interface { - CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg RGetter) (map[abi.SectorID]string, error) } +type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error) + // CheckProvable returns unprovable sectors -func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { +func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg RGetter) (map[abi.SectorID]string, error) { var bad = make(map[abi.SectorID]string) ssize, err := pp.SectorSize() @@ -83,6 +90,49 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, } } + if rg != nil { + wpp, err := sector.ProofType.RegisteredWindowPoStProof() + if err != nil { + return err + } + + var pr abi.PoStRandomness = make([]byte, abi.RandomnessLength) + _, _ = rand.Read(pr) + pr[31] &= 0x3f + + ch, err := ffi.GeneratePoStFallbackSectorChallenges(wpp, sector.ID.Miner, pr, []abi.SectorNumber{ + sector.ID.Number, + }) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err) + bad[sector.ID] = fmt.Sprintf("generating fallback challenges: %s", err) + return nil + } + + commr, err := rg(ctx, sector.ID) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err) + bad[sector.ID] = fmt.Sprintf("getting commR: %s", err) + return nil + } + + _, err = ffi.GenerateSingleVanillaProof(ffi.PrivateSectorInfo{ + SectorInfo: proof.SectorInfo{ + SealProof: sector.ProofType, + SectorNumber: sector.ID.Number, + SealedCID: commr, + }, + CacheDirPath: lp.Cache, + PoStProofType: wpp, + SealedSectorPath: lp.Sealed, + }, ch.Challenges[sector.ID.Number]) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err) + bad[sector.ID] = fmt.Sprintf("generating vanilla proof: %s", err) + return nil + } + } + return nil }() if err != nil { diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 488b80f8998..79dd898d03b 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -544,8 +544,23 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error return backup(sm.DS, fpath) } -func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef) (map[abi.SectorNumber]string, error) { - bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors) +func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) { + var rg sectorstorage.RGetter + if expensive { + rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) { + si, err := sm.Miner.GetSectorInfo(id.Number) + if err != nil { + return cid.Undef, err + } + if si.CommR == nil { + return cid.Undef, xerrors.Errorf("commr is nil") + } + + return *si.CommR, nil + } + } + + bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors, rg) if err != nil { return nil, err } diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 87dd8ad1552..4d3d4b726bc 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -214,7 +214,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B }) } - bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck) + bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck, nil) if err != nil { return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err) } From 95eaf13b5aa601b78bcb960e6427882f1a08e659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 2 Dec 2020 00:35:55 +0100 Subject: [PATCH 2/3] sectorstorage: Fix tests --- extern/sector-storage/faults.go | 8 ++------ extern/sector-storage/mock/mock.go | 2 +- extern/sector-storage/storiface/ffi.go | 4 ++++ node/impl/storminer.go | 2 +- storage/wdpost_run_test.go | 3 ++- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/extern/sector-storage/faults.go b/extern/sector-storage/faults.go index 0fd385250a9..fdd5f6b7d6c 100644 --- a/extern/sector-storage/faults.go +++ b/extern/sector-storage/faults.go @@ -9,8 +9,6 @@ import ( "golang.org/x/xerrors" - "github.com/ipfs/go-cid" - ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/runtime/proof" @@ -21,13 +19,11 @@ import ( // FaultTracker TODO: Track things more actively type FaultTracker interface { - CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg RGetter) (map[abi.SectorID]string, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) } -type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error) - // CheckProvable returns unprovable sectors -func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg RGetter) (map[abi.SectorID]string, error) { +func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) { var bad = make(map[abi.SectorID]string) ssize, err := pp.SectorSize() diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 59d7d05036d..47fb2b974f2 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -405,7 +405,7 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro return nil } -func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) (map[abi.SectorID]string, error) { +func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) { bad := map[abi.SectorID]string{} for _, sid := range ids { diff --git a/extern/sector-storage/storiface/ffi.go b/extern/sector-storage/storiface/ffi.go index 95d400e5248..31d5b2be55f 100644 --- a/extern/sector-storage/storiface/ffi.go +++ b/extern/sector-storage/storiface/ffi.go @@ -1,7 +1,9 @@ package storiface import ( + "context" "errors" + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-state-types/abi" ) @@ -15,3 +17,5 @@ func (i UnpaddedByteIndex) Padded() PaddedByteIndex { } type PaddedByteIndex uint64 + +type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 79dd898d03b..a1a6bec248d 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -545,7 +545,7 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error } func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) { - var rg sectorstorage.RGetter + var rg storiface.RGetter if expensive { rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) { si, err := sm.Miner.GetSectorInfo(id.Number) diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 80b4f66afe1..65b6b0183b5 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -27,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/journal" ) @@ -125,7 +126,7 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si type mockFaultTracker struct { } -func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { +func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) { // Returns "bad" sectors so just return empty map meaning all sectors are good return map[abi.SectorID]string{}, nil } From 289ef910a0f21a3f86fca774fd3f9ec43f6046f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 2 Dec 2020 00:39:55 +0100 Subject: [PATCH 3/3] fix imports, docsgen --- documentation/en/api-methods-miner.md | 3 ++- extern/sector-storage/storiface/ffi.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/en/api-methods-miner.md b/documentation/en/api-methods-miner.md index ebf0973ccbf..d3f9880aaec 100644 --- a/documentation/en/api-methods-miner.md +++ b/documentation/en/api-methods-miner.md @@ -232,7 +232,8 @@ Inputs: ```json [ 8, - null + null, + true ] ``` diff --git a/extern/sector-storage/storiface/ffi.go b/extern/sector-storage/storiface/ffi.go index 31d5b2be55f..f6b2cbdd31d 100644 --- a/extern/sector-storage/storiface/ffi.go +++ b/extern/sector-storage/storiface/ffi.go @@ -3,6 +3,7 @@ package storiface import ( "context" "errors" + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-state-types/abi"