From de726b190f8cc561e40c6ba362e21363194961fd Mon Sep 17 00:00:00 2001 From: Acha Bill Date: Wed, 11 Dec 2024 20:18:06 -0500 Subject: [PATCH 1/4] fix: add TestGetWrappedChunk --- pkg/feeds/getter_test.go | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 pkg/feeds/getter_test.go diff --git a/pkg/feeds/getter_test.go b/pkg/feeds/getter_test.go new file mode 100644 index 0000000000..cb40b0b8ce --- /dev/null +++ b/pkg/feeds/getter_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package feeds + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + + soctesting "github.com/ethersphere/bee/v2/pkg/soc/testing" + mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock" +) + +func TestGetWrappedChunk(t *testing.T) { + storer := mockstorer.New() + + // new format (wraps chunk) + ch := soctesting.GenerateMockSOC(t, []byte("data")).Chunk() + wch, err := GetWrappedChunk(context.Background(), storer.ChunkStore(), ch) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(wch.Data()[8:], []byte("data")) { + t.Fatal("data mismatch") + } + + // old format (ts + ref) + timestamp := make([]byte, 8) + binary.BigEndian.PutUint64(timestamp, 1) + ch = soctesting.GenerateMockSOC(t, append(timestamp, wch.Address().Bytes()...)).Chunk() + + err = storer.Put(context.Background(), wch) + if err != nil { + t.Fatal(err) + } + + wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(wch.Data()[8:], []byte("data")) { + t.Fatal("data mismatch") + } +} From 077e76389cc06565c8f59463c70184c32fdabd25 Mon Sep 17 00:00:00 2001 From: Acha Bill Date: Sat, 14 Dec 2024 04:35:33 -0500 Subject: [PATCH 2/4] chore: beekeeper --- .github/workflows/beekeeper.yml | 87 +++++++++++++++++---------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index 6b2bf8ac83..e7bb15b0f9 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -14,7 +14,7 @@ env: SETUP_CONTRACT_IMAGE: "ethersphere/bee-localchain" SETUP_CONTRACT_IMAGE_TAG: "0.9.2" BEELOCAL_BRANCH: "main" - BEEKEEPER_BRANCH: "master" + BEEKEEPER_BRANCH: "feat/feeds-check" BEEKEEPER_METRICS_ENABLED: false REACHABILITY_OVERRIDE_PUBLIC: true BATCHFACTOR_OVERRIDE_PUBLIC: 2 @@ -132,47 +132,50 @@ jobs: - name: Test fullconnectivity id: fullconnectivity run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks=ci-full-connectivity; do echo "waiting for full connectivity..."; sleep .3; done' - - name: Test settlements - id: settlements - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-settlements - - name: Sleep for time allowance to replenish - run: sleep 2 - - name: Test pss - id: pss - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pss - - name: Test soc - id: soc - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-soc - - name: Test gsoc - id: gsoc - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-gsoc - - name: Test pushsync (chunks) - id: pushsync-chunks-1 - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-chunks - - name: Test pushsync (light mode chunks) - id: pushsync-chunks-2 - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-light-chunks - - name: Test retrieval - id: retrieval - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-retrieval - - name: Test manifest - id: manifest - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest - - name: Test postage stamps - id: postage-stamps - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-postage - - name: Test staking - id: stake - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-stake - - name: Test withdraw - id: withdraw - run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-withdraw; do echo "waiting for withdraw..."; sleep .3; done' - - name: Test redundancy - id: redundancy - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-redundancy - - name: Test act - id: act - run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-act; do echo "waiting for act..."; sleep .3; done' +# - name: Test settlements +# id: settlements +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-settlements +# - name: Sleep for time allowance to replenish +# run: sleep 2 +# - name: Test pss +# id: pss +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pss +# - name: Test soc +# id: soc +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-soc +# - name: Test gsoc +# id: gsoc +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-gsoc +# - name: Test pushsync (chunks) +# id: pushsync-chunks-1 +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-chunks +# - name: Test pushsync (light mode chunks) +# id: pushsync-chunks-2 +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-light-chunks +# - name: Test retrieval +# id: retrieval +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-retrieval +# - name: Test manifest +# id: manifest +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest +# - name: Test postage stamps +# id: postage-stamps +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-postage +# - name: Test staking +# id: stake +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-stake +# - name: Test withdraw +# id: withdraw +# run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-withdraw; do echo "waiting for withdraw..."; sleep .3; done' +# - name: Test redundancy +# id: redundancy +# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-redundancy +# - name: Test act +# id: act +# run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-act; do echo "waiting for act..."; sleep .3; done' + - name: Test feeds + id: feeds + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-feed - name: Collect debug artifacts if: failure() run: | From 94c0ceed8b974045f48e8b5c7458f3875df5048c Mon Sep 17 00:00:00 2001 From: Acha Bill Date: Tue, 7 Jan 2025 18:05:29 -0500 Subject: [PATCH 3/4] fix: test encrypted --- pkg/feeds/getter_test.go | 46 +++++++++++++------ .../inmemchunkstore/inmemchunkstore.go | 23 ++++++---- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/pkg/feeds/getter_test.go b/pkg/feeds/getter_test.go index cb40b0b8ce..d09d5b837a 100644 --- a/pkg/feeds/getter_test.go +++ b/pkg/feeds/getter_test.go @@ -28,22 +28,40 @@ func TestGetWrappedChunk(t *testing.T) { t.Fatal("data mismatch") } - // old format (ts + ref) - timestamp := make([]byte, 8) - binary.BigEndian.PutUint64(timestamp, 1) - ch = soctesting.GenerateMockSOC(t, append(timestamp, wch.Address().Bytes()...)).Chunk() - - err = storer.Put(context.Background(), wch) - if err != nil { - t.Fatal(err) + // old format + tt := []struct { + name string + addr []byte + }{ + { + name: "unencrypted", + addr: wch.Address().Bytes(), + }, + { + name: "encrypted", + addr: append(wch.Address().Bytes(), wch.Address().Bytes()...), + }, } - wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch) - if err != nil { - t.Fatal(err) - } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + timestamp := make([]byte, 8) + binary.BigEndian.PutUint64(timestamp, 1) + ch = soctesting.GenerateMockSOC(t, append(timestamp, tc.addr...)).Chunk() - if !bytes.Equal(wch.Data()[8:], []byte("data")) { - t.Fatal("data mismatch") + err = storer.Put(context.Background(), wch) + if err != nil { + t.Fatal(err) + } + + wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(wch.Data()[8:], []byte("data")) { + t.Fatal("data mismatch") + } + }) } } diff --git a/pkg/storage/inmemchunkstore/inmemchunkstore.go b/pkg/storage/inmemchunkstore/inmemchunkstore.go index 3ec2b8e8a6..4f0465016a 100644 --- a/pkg/storage/inmemchunkstore/inmemchunkstore.go +++ b/pkg/storage/inmemchunkstore/inmemchunkstore.go @@ -32,7 +32,7 @@ func (c *ChunkStore) Get(_ context.Context, addr swarm.Address) (swarm.Chunk, er c.mu.Lock() defer c.mu.Unlock() - chunk, ok := c.chunks[addr.ByteString()] + chunk, ok := c.chunks[c.key(addr)] if !ok { return nil, storage.ErrNotFound } @@ -43,12 +43,12 @@ func (c *ChunkStore) Put(_ context.Context, ch swarm.Chunk) error { c.mu.Lock() defer c.mu.Unlock() - chunkCount, ok := c.chunks[ch.Address().ByteString()] + chunkCount, ok := c.chunks[c.key(ch.Address())] if !ok { chunkCount.chunk = swarm.NewChunk(ch.Address(), ch.Data()).WithStamp(ch.Stamp()) } chunkCount.count++ - c.chunks[ch.Address().ByteString()] = chunkCount + c.chunks[c.key(ch.Address())] = chunkCount return nil } @@ -57,7 +57,7 @@ func (c *ChunkStore) Has(_ context.Context, addr swarm.Address) (bool, error) { c.mu.Lock() defer c.mu.Unlock() - _, exists := c.chunks[addr.ByteString()] + _, exists := c.chunks[c.key(addr)] return exists, nil } @@ -66,12 +66,12 @@ func (c *ChunkStore) Delete(_ context.Context, addr swarm.Address) error { c.mu.Lock() defer c.mu.Unlock() - chunkCount := c.chunks[addr.ByteString()] + chunkCount := c.chunks[c.key(addr)] chunkCount.count-- if chunkCount.count <= 0 { delete(c.chunks, addr.ByteString()) } else { - c.chunks[addr.ByteString()] = chunkCount + c.chunks[c.key(addr)] = chunkCount } return nil @@ -81,12 +81,12 @@ func (c *ChunkStore) Replace(_ context.Context, ch swarm.Chunk, emplace bool) er c.mu.Lock() defer c.mu.Unlock() - chunkCount := c.chunks[ch.Address().ByteString()] + chunkCount := c.chunks[c.key(ch.Address())] chunkCount.chunk = ch if emplace { chunkCount.count++ } - c.chunks[ch.Address().ByteString()] = chunkCount + c.chunks[c.key(ch.Address())] = chunkCount return nil } @@ -111,3 +111,10 @@ func (c *ChunkStore) Iterate(_ context.Context, fn storage.IterateChunkFn) error func (c *ChunkStore) Close() error { return nil } + +func (c *ChunkStore) key(addr swarm.Address) string { + if len(addr.Bytes()) < swarm.HashSize { + return addr.ByteString() + } + return string(addr.Bytes()[:swarm.HashSize]) +} From 2f75b74f01da862134483c33a99beba83275fe4c Mon Sep 17 00:00:00 2001 From: Acha Bill Date: Mon, 13 Jan 2025 13:48:37 -0500 Subject: [PATCH 4/4] chore: use master beekeeper --- .github/workflows/beekeeper.yml | 84 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index e7bb15b0f9..a2e5c93639 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -14,7 +14,7 @@ env: SETUP_CONTRACT_IMAGE: "ethersphere/bee-localchain" SETUP_CONTRACT_IMAGE_TAG: "0.9.2" BEELOCAL_BRANCH: "main" - BEEKEEPER_BRANCH: "feat/feeds-check" + BEEKEEPER_BRANCH: "master" BEEKEEPER_METRICS_ENABLED: false REACHABILITY_OVERRIDE_PUBLIC: true BATCHFACTOR_OVERRIDE_PUBLIC: 2 @@ -132,47 +132,47 @@ jobs: - name: Test fullconnectivity id: fullconnectivity run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks=ci-full-connectivity; do echo "waiting for full connectivity..."; sleep .3; done' -# - name: Test settlements -# id: settlements -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-settlements -# - name: Sleep for time allowance to replenish -# run: sleep 2 -# - name: Test pss -# id: pss -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pss -# - name: Test soc -# id: soc -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-soc -# - name: Test gsoc -# id: gsoc -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-gsoc -# - name: Test pushsync (chunks) -# id: pushsync-chunks-1 -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-chunks -# - name: Test pushsync (light mode chunks) -# id: pushsync-chunks-2 -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-light-chunks -# - name: Test retrieval -# id: retrieval -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-retrieval -# - name: Test manifest -# id: manifest -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest -# - name: Test postage stamps -# id: postage-stamps -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-postage -# - name: Test staking -# id: stake -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-stake -# - name: Test withdraw -# id: withdraw -# run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-withdraw; do echo "waiting for withdraw..."; sleep .3; done' -# - name: Test redundancy -# id: redundancy -# run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-redundancy -# - name: Test act -# id: act -# run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-act; do echo "waiting for act..."; sleep .3; done' + - name: Test settlements + id: settlements + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-settlements + - name: Sleep for time allowance to replenish + run: sleep 2 + - name: Test pss + id: pss + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pss + - name: Test soc + id: soc + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-soc + - name: Test gsoc + id: gsoc + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-gsoc + - name: Test pushsync (chunks) + id: pushsync-chunks-1 + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-chunks + - name: Test pushsync (light mode chunks) + id: pushsync-chunks-2 + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-light-chunks + - name: Test retrieval + id: retrieval + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-retrieval + - name: Test manifest + id: manifest + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest + - name: Test postage stamps + id: postage-stamps + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-postage + - name: Test staking + id: stake + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-stake + - name: Test withdraw + id: withdraw + run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-withdraw; do echo "waiting for withdraw..."; sleep .3; done' + - name: Test redundancy + id: redundancy + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-redundancy + - name: Test act + id: act + run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-act; do echo "waiting for act..."; sleep .3; done' - name: Test feeds id: feeds run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-feed