logs for backend manager and backends

build-index/cmd/cmd.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -136,7 +136,7 @@ func Run(flags *Flags, opts ...Option) {
 		log.Fatalf("Error creating simple store: %s", err)
 	}
 
-	backends, err := backend.NewManager(config.Backends, config.Auth, stats)
+	backends, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats)
 	if err != nil {
 		log.Fatalf("Error creating backend manager: %s", err)
 	}

build-index/cmd/config.go

@@ -34,6 +34,7 @@ import (
 type Config struct {
 	ZapLogging     zap.Config                   `yaml:"zap"`
 	Metrics        metrics.Config               `yaml:"metrics"`
+  BackendManager backend.ManagerConfig   `yaml:"backend_manager"`
 	Backends       []backend.Config             `yaml:"backends"`
 	Auth           backend.AuthConfig           `yaml:"auth"`
 	TagServer      tagserver.Config             `yaml:"tagserver"`

lib/backend/client.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,15 +17,16 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/uber/kraken/core"
 	"github.com/uber-go/tally"
+	"github.com/uber/kraken/core"
+	"go.uber.org/zap"
 )
 
 var _factories = make(map[string]ClientFactory)
 
 // ClientFactory creates backend client given name.
 type ClientFactory interface {
-	Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope) (Client, error)
+	Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope, logger *zap.SugaredLogger) (Client, error)
 }
 
 // Register registers new Factory with corresponding backend client name.

lib/backend/fixtures.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,11 +13,13 @@
 // limitations under the License.
 package backend
 
-import "github.com/uber-go/tally"
+import (
+	"github.com/uber-go/tally"
+)
 
 // ManagerFixture returns a Manager with no clients for testing purposes.
 func ManagerFixture() *Manager {
-	m, err := NewManager(nil, AuthConfig{}, tally.NoopScope)
+	m, err := NewManager(ManagerConfig{}, nil, AuthConfig{}, tally.NoopScope)
 	if err != nil {
 		panic(err)
 	}

lib/backend/gcsbackend/client.go

@@ -31,6 +31,7 @@ import (
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	"gopkg.in/yaml.v2"
+  "go.uber.org/zap"
 )
 
 const _gcs = "gcs"
@@ -42,7 +43,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

lib/backend/gcsbackend/client_test.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -85,7 +86,7 @@ func TestClientFactory(t *testing.T) {
 	userAuth := UserAuthConfig{"test-user": auth}
 	masterAuth := backend.AuthConfig{_gcs: userAuth}
 	f := factory{}
-	_, err := f.Create(config, masterAuth, tally.NoopScope)
+	_, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar())
 	fmt.Println(err.Error())
 	require.True(strings.Contains(err.Error(), "invalid gcs credentials"))
 }

lib/backend/hdfsbackend/client.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,6 +30,7 @@ import (
 	"github.com/uber/kraken/utils/log"
 
 	"github.com/satori/go.uuid"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
@@ -42,7 +43,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

lib/backend/hdfsbackend/client_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -64,7 +65,7 @@ func TestClientFactory(t *testing.T) {
 		testing:       true,
 	}
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

lib/backend/httpbackend/http.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,6 +27,7 @@ import (
 	"github.com/uber/kraken/utils/httputil"
 
 	"gopkg.in/yaml.v2"
+	"go.uber.org/zap"
 )
 
 const _http = "http"
@@ -38,7 +39,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

lib/backend/httpbackend/http_test.go

@@ -28,14 +28,15 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestClientFactory(t *testing.T) {
 	require := require.New(t)
 
 	config := Config{}
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

lib/backend/manager.go

@@ -53,8 +53,19 @@ type Manager struct {
 	backends []*backend
 }
 
+// ManagerConfig is config for backend manager.
+type ManagerConfig struct {
+	Log log.Config `yaml:"log"`
+}
+
 // NewManager creates a new backend Manager.
-func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) {
+func NewManager(managerConfig ManagerConfig, configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) {
+	logger, err := log.New(managerConfig.Log, nil)
+	if err != nil {
+		return nil, fmt.Errorf("log: %s", err)
+	}
+	slogger := logger.Sugar()
+
 	var backends []*backend
 	for _, config := range configs {
 		config = config.applyDefaults()
@@ -71,7 +82,7 @@ func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager,
 		if err != nil {
 			return nil, fmt.Errorf("get backend client factory: %s", err)
 		}
-		c, err = factory.Create(backendConfig, auth, stats)
+		c, err = factory.Create(backendConfig, auth, stats, slogger)
 		if err != nil {
 			return nil, fmt.Errorf("create backend client: %s", err)
 		}

lib/backend/manager_test.go

@@ -107,7 +107,7 @@ func TestManagerNamespaceOrdering(t *testing.T) {
 	var configs []Config
 	require.NoError(yaml.Unmarshal([]byte(configStr), &configs))
 
-	m, err := NewManager(configs, AuthConfig{}, tally.NoopScope)
+	m, err := NewManager(ManagerConfig{}, configs, AuthConfig{}, tally.NoopScope)
 	require.NoError(err)
 
 	for ns, expected := range map[string]string{
@@ -128,18 +128,20 @@ func TestManagerNamespaceOrdering(t *testing.T) {
 func TestManagerBandwidth(t *testing.T) {
 	require := require.New(t)
 
-	m, err := NewManager([]Config{{
-		Namespace: ".*",
-		Bandwidth: bandwidth.Config{
-			EgressBitsPerSec:  10,
-			IngressBitsPerSec: 50,
-			TokenSize:         1,
-			Enable:            true,
-		},
-		Backend: map[string]interface{}{
-			"testfs": testfs.Config{Addr: "test-addr", NamePath: namepath.Identity},
-		},
-	}}, AuthConfig{}, tally.NoopScope)
+	m, err := NewManager(
+		ManagerConfig{},
+		[]Config{{
+			Namespace: ".*",
+			Bandwidth: bandwidth.Config{
+				EgressBitsPerSec:  10,
+				IngressBitsPerSec: 50,
+				TokenSize:         1,
+				Enable:            true,
+			},
+			Backend: map[string]interface{}{
+				"testfs": testfs.Config{Addr: "test-addr", NamePath: namepath.Identity},
+			},
+		}}, AuthConfig{}, tally.NoopScope)
 	require.NoError(err)
 
 	checkBandwidth := func(egress, ingress int64) {

lib/backend/registrybackend/blobclient.go

@@ -28,6 +28,7 @@ import (
 	"github.com/uber/kraken/utils/httputil"
 	"github.com/uber/kraken/utils/log"
 	yaml "gopkg.in/yaml.v2"
+	"go.uber.org/zap"
 )
 
 const _registryblob = "registry_blob"
@@ -39,7 +40,7 @@ func init() {
 type blobClientFactory struct{}
 
 func (f *blobClientFactory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

lib/backend/registrybackend/blobclient_test.go

@@ -28,14 +28,15 @@ import (
 	"github.com/uber/kraken/utils/memsize"
 	"github.com/uber/kraken/utils/randutil"
 	"github.com/uber/kraken/utils/testutil"
+	"go.uber.org/zap"
 )
 
 func TestClientFactory(t *testing.T) {
 	require := require.New(t)
 
 	config := Config{}
 	f := blobClientFactory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

lib/backend/registrybackend/tagclient.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -28,6 +28,7 @@ import (
 	"github.com/uber/kraken/lib/backend/registrybackend/security"
 	"github.com/uber/kraken/utils/dockerutil"
 	"github.com/uber/kraken/utils/httputil"
+	"go.uber.org/zap"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -40,7 +41,7 @@ func init() {
 type tagClientFactory struct{}
 
 func (f *tagClientFactory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

lib/backend/s3backend/client.go

@@ -34,6 +34,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"gopkg.in/yaml.v2"
+	"go.uber.org/zap"
 )
 
 const _s3 = "s3"
@@ -45,7 +46,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

lib/backend/s3backend/client_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -82,7 +83,7 @@ func TestClientFactory(t *testing.T) {
 	userAuth := UserAuthConfig{"test-user": auth}
 	masterAuth := backend.AuthConfig{_s3: userAuth}
 	f := factory{}
-	_, err := f.Create(config, masterAuth, tally.NoopScope)
+	_, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

lib/backend/shadowbackend/client.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,6 +27,7 @@ import (
 	"github.com/uber/kraken/lib/backend/sqlbackend"
 	"github.com/uber/kraken/lib/backend/testfs"
 	"github.com/uber/kraken/utils/log"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
@@ -43,7 +44,7 @@ func (f *factory) Name() string {
 }
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {