Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storage serialization #425

Draft
wants to merge 19 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
hmac protected CONFIG
  • Loading branch information
poolpOrg committed Feb 7, 2025
commit df0bfe001c7c24f6ae7bb699bd824a9f34d8baa0
2 changes: 2 additions & 0 deletions api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
)

var lstore storage.Store
var lconfig storage.Configuration
var lrepository *repository.Repository

type Item[T any] struct {
Expand Down Expand Up @@ -92,6 +93,7 @@ func TokenAuthMiddleware(token string) func(http.Handler) http.Handler {

func SetupRoutes(server *http.ServeMux, repo *repository.Repository, token string) {
lstore = repo.Store()
lconfig = repo.Configuration()
lrepository = repo

authToken := TokenAuthMiddleware(token)
Expand Down
2 changes: 1 addition & 1 deletion api/api_params_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@

for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
lstore, err := storage.Create(c.location, *c.config)

Check failure on line 312 in api/api_params_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *c.config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to storage.Create
require.NoError(t, err, "creating storage")

ctx := appcontext.NewAppContext()
Expand All @@ -317,7 +317,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *c.config, nil)

Check failure on line 320 in api/api_params_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *c.config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to repository.New
require.NoError(t, err, "creating repository")

req, err := http.NewRequest("GET", "/path/{id}", nil)
Expand Down
20 changes: 10 additions & 10 deletions api/api_repository_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
// XXX: re-add once we move to non-mocked state object.
func _Test_RepositoryConfiguration(t *testing.T) {
config := ptesting.NewConfiguration()
lstore, err := storage.Create("/test/location", *config)

Check failure on line 28 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to storage.Create
require.NoError(t, err, "creating storage")

ctx := appcontext.NewAppContext()
Expand All @@ -33,7 +33,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)

Check failure on line 36 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to repository.New
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -293,7 +293,7 @@

for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
lstore, err := storage.Create(c.location, *c.config)

Check failure on line 296 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *c.config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to storage.Create
require.NoError(t, err, "creating storage")

ctx := appcontext.NewAppContext()
Expand All @@ -301,7 +301,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *c.config, nil)

Check failure on line 304 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *c.config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to repository.New
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -381,7 +381,7 @@
for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
config := ptesting.NewConfiguration()
lstore, err := storage.Create(c.location, *config)

Check failure on line 384 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to storage.Create
require.NoError(t, err, "creating storage")

ctx := appcontext.NewAppContext()
Expand All @@ -389,7 +389,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)

Check failure on line 392 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to repository.New
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -444,7 +444,7 @@

for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
lstore, err := storage.Create(c.location, *c.config)

Check failure on line 447 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *c.config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to storage.Create
require.NoError(t, err, "creating storage")

ctx := appcontext.NewAppContext()
Expand All @@ -452,7 +452,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *c.config, nil)

Check failure on line 455 in api/api_repository_test.go

View workflow job for this annotation

GitHub Actions / build (1.23.3)

cannot use *c.config (variable of type "github.com/PlakarKorp/plakar/storage".Configuration) as []byte value in argument to repository.New
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -509,7 +509,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *c.config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -571,7 +571,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -621,7 +621,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *c.config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -675,7 +675,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -720,7 +720,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *c.config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -810,7 +810,7 @@
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down
6 changes: 3 additions & 3 deletions api/api_snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func _TestSnapshotHeader(t *testing.T) {
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -199,7 +199,7 @@ func TestSnapshotHeaderErrors(t *testing.T) {
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
require.NoError(t, err, "creating repository")

var noToken string
Expand Down Expand Up @@ -246,7 +246,7 @@ func _TestSnapshotSign(t *testing.T) {
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
require.NoError(t, err, "creating repository")

token := "test-token"
Expand Down
3 changes: 1 addition & 2 deletions api/api_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ import (
)

func storageConfiguration(w http.ResponseWriter, r *http.Request) error {
configuration := lstore.Configuration()
return json.NewEncoder(w).Encode(configuration)
return json.NewEncoder(w).Encode(lconfig)
}

func storageStates(w http.ResponseWriter, r *http.Request) error {
Expand Down
4 changes: 2 additions & 2 deletions api/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestAuthMiddleware(t *testing.T) {
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -80,7 +80,7 @@ func Test_UnknownEndpoint(t *testing.T) {
defer cache.Close()
ctx.SetCache(cache)
ctx.SetLogger(logging.NewLogger(os.Stdout, os.Stderr))
repo, err := repository.New(ctx, lstore, nil)
repo, err := repository.New(ctx, lstore, *config, nil)
if err != nil {
t.Fatal(err)
}
Expand Down
26 changes: 16 additions & 10 deletions cmd/plakar/plakar.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,21 +259,27 @@ func entryPoint() int {
skipPassphrase = true
}

store, err := storage.Open(repositoryPath)
store, serializedConfig, err := storage.Open(repositoryPath)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s\n", flag.CommandLine.Name(), err)
return 1
}

if store.Configuration().Version != versioning.FromString(storage.VERSION) {
storeConfig, err := storage.NewConfigurationFromWrappedBytes(serializedConfig)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s\n", flag.CommandLine.Name(), err)
return 1
}

if storeConfig.Version != versioning.FromString(storage.VERSION) {
fmt.Fprintf(os.Stderr, "%s: incompatible repository version: %s != %s\n",
flag.CommandLine.Name(), store.Configuration().Version, storage.VERSION)
flag.CommandLine.Name(), storeConfig.Version, storage.VERSION)
return 1
}

var secret []byte
if !skipPassphrase {
if store.Configuration().Encryption != nil {
if storeConfig.Encryption != nil {
derived := false
envPassphrase := os.Getenv("PLAKAR_PASSPHRASE")
if ctx.KeyFromFile == "" {
Expand All @@ -288,11 +294,11 @@ func entryPoint() int {
passphrase = []byte(envPassphrase)
}

key, err := encryption.DeriveKey(store.Configuration().Encryption.KDFParams, passphrase)
key, err := encryption.DeriveKey(storeConfig.Encryption.KDFParams, passphrase)
if err != nil {
continue
}
if !encryption.VerifyCanary(key, store.Configuration().Encryption.Canary) {
if !encryption.VerifyCanary(key, storeConfig.Encryption.Canary) {
if envPassphrase != "" {
break
}
Expand All @@ -303,9 +309,9 @@ func entryPoint() int {
break
}
} else {
key, err := encryption.DeriveKey(store.Configuration().Encryption.KDFParams, []byte(ctx.KeyFromFile))
key, err := encryption.DeriveKey(storeConfig.Encryption.KDFParams, []byte(ctx.KeyFromFile))
if err == nil {
if encryption.VerifyCanary(key, store.Configuration().Encryption.Canary) {
if encryption.VerifyCanary(key, storeConfig.Encryption.Canary) {
secret = key
derived = true
}
Expand All @@ -321,13 +327,13 @@ func entryPoint() int {

var repo *repository.Repository
if opt_agentless && command != "server" {
repo, err = repository.New(ctx, store, secret)
repo, err = repository.New(ctx, store, serializedConfig, secret)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s\n", flag.CommandLine.Name(), err)
return 1
}
} else {
repo, err = repository.NewNoRebuild(ctx, store, secret)
repo, err = repository.NewNoRebuild(ctx, store, serializedConfig, secret)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s\n", flag.CommandLine.Name(), err)
return 1
Expand Down
4 changes: 2 additions & 2 deletions cmd/plakar/subcommands/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -575,14 +575,14 @@ func (cmd *Agent) ListenAndServe(ctx *appcontext.AppContext) error {
var repo *repository.Repository

if repositoryLocation != "" {
store, err := storage.Open(repositoryLocation)
store, serializedConfig, err := storage.Open(repositoryLocation)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to open storage: %s\n", err)
return
}
defer store.Close()

repo, err = repository.New(clientContext, store, clientContext.GetSecret())
repo, err = repository.New(clientContext, store, serializedConfig, clientContext.GetSecret())
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to open repository: %s\n", err)
return
Expand Down
10 changes: 8 additions & 2 deletions cmd/plakar/subcommands/clone/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,16 @@ func (cmd *Clone) Name() string {
func (cmd *Clone) Execute(ctx *appcontext.AppContext, repo *repository.Repository) (int, error) {
sourceStore := repo.Store()

configuration := sourceStore.Configuration()
configuration := repo.Configuration()
configuration.RepositoryID = uuid.Must(uuid.NewRandom())

cloneStore, err := storage.Create(cmd.Dest, configuration)
serializedConfig, err := configuration.ToBytes()
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to decode storage configuration: %s\n", err)
return 1, err
}

cloneStore, err := storage.Create(cmd.Dest, serializedConfig)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: could not create repository: %s\n", cmd.Dest, err)
return 1, err
Expand Down
26 changes: 24 additions & 2 deletions cmd/plakar/subcommands/create/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@
package create

import (
"bytes"
"flag"
"fmt"
"hash"
"io"
"os"
"path/filepath"

Expand All @@ -29,7 +32,9 @@ import (
"github.com/PlakarKorp/plakar/encryption"
"github.com/PlakarKorp/plakar/hashing"
"github.com/PlakarKorp/plakar/repository"
"github.com/PlakarKorp/plakar/resources"
"github.com/PlakarKorp/plakar/storage"
"github.com/PlakarKorp/plakar/versioning"
passwordvalidator "github.com/wagslane/go-password-validator"
)

Expand Down Expand Up @@ -79,6 +84,7 @@ func (cmd *Create) Execute(ctx *appcontext.AppContext, repo *repository.Reposito
hashingConfiguration := hashing.DefaultConfiguration()
storageConfiguration.Hashing = *hashingConfiguration

var hasher hash.Hash
if !cmd.NoEncryption {
storageConfiguration.Encryption.Algorithm = encryption.DefaultConfiguration().Algorithm

Expand Down Expand Up @@ -132,18 +138,34 @@ func (cmd *Create) Execute(ctx *appcontext.AppContext, repo *repository.Reposito
return 1, err
}
storageConfiguration.Encryption.Canary = canary
hasher = hashing.GetHasherHMAC(storage.DEFAULT_HASHING_ALGORITHM, key)
} else {
storageConfiguration.Encryption = nil
hasher = hashing.GetHasher(storage.DEFAULT_HASHING_ALGORITHM)
}

serializedConfig, err := storageConfiguration.ToBytes()
if err != nil {
return 1, err
}

rd, err := storage.Serialize(hasher, resources.RT_CONFIG, versioning.GetCurrentVersion(resources.RT_CONFIG), bytes.NewReader(serializedConfig))
if err != nil {
return 1, err
}
wrappedConfig, err := io.ReadAll(rd)
if err != nil {
return 1, err
}

if cmd.Location == "" {
repo, err := storage.Create(filepath.Join(ctx.HomeDir, ".plakar"), *storageConfiguration)
repo, err := storage.Create(filepath.Join(ctx.HomeDir, ".plakar"), wrappedConfig)
if err != nil {
return 1, err
}
repo.Close()
} else {
repo, err := storage.Create(cmd.Location, *storageConfiguration)
repo, err := storage.Create(cmd.Location, wrappedConfig)
if err != nil {
return 1, err
}
Expand Down
16 changes: 11 additions & 5 deletions cmd/plakar/subcommands/sync/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,35 +80,41 @@ func (cmd *Sync) Execute(ctx *appcontext.AppContext, repo *repository.Repository
return 1, fmt.Errorf("usage: sync [snapshotID] to|from repository")
}

peerStore, err := storage.Open(peerRepositoryPath)
peerStore, peerStoreSerializedConfig, err := storage.Open(peerRepositoryPath)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: could not open repository: %s\n", peerRepositoryPath, err)
return 1, err
}

peerStoreConfig, err := storage.NewConfigurationFromWrappedBytes(peerStoreSerializedConfig)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: could not parse configuration: %s\n", peerStore.Location(), err)
return 1, err
}

var peerSecret []byte
if peerStore.Configuration().Encryption != nil {
if peerStoreConfig.Encryption != nil {
for {
passphrase, err := utils.GetPassphrase("destination repository")
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}

key, err := encryption.DeriveKey(peerStore.Configuration().Encryption.KDFParams, passphrase)
key, err := encryption.DeriveKey(peerStoreConfig.Encryption.KDFParams, passphrase)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
if !encryption.VerifyCanary(key, peerStore.Configuration().Encryption.Canary) {
if !encryption.VerifyCanary(key, peerStoreConfig.Encryption.Canary) {
fmt.Fprintf(os.Stderr, "invalid passphrase\n")
continue
}
peerSecret = key
break
}
}
peerRepository, err := repository.New(ctx, peerStore, peerSecret)
peerRepository, err := repository.New(ctx, peerStore, peerStoreSerializedConfig, peerSecret)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: could not open repository: %s\n", peerStore.Location(), err)
return 1, err
Expand Down
Loading