From 106609d8a8746d60a06f157e2b93a963d385a2ef Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 25 Oct 2022 20:21:47 +0900 Subject: [PATCH 01/17] GDrive: Support service_account --- server/storage/gdrive.go | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 61e9f48c..55447957 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -45,13 +45,20 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir } // If modifying these scopes, delete your previously saved client_secret.json. - config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) - if err != nil { - return nil, err + var httpClient *http.Client + + if strings.Contains(string(b), `"type": "service_account"`) { + logger.Println("service account") + httpClient = JWTConfigFromJSON(b, logger).Client(ctx) + } else { + logger.Println("user account") + config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) + if err != nil { + return nil, err + } + httpClient = getGDriveClient(ctx, config, localConfigPath, logger) } - httpClient := getGDriveClient(ctx, config, localConfigPath, logger) - srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) if err != nil { return nil, err @@ -323,6 +330,25 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader return nil } +// Retrieve a ServiceAccountJson, return a jwt.Config +func JWTConfigFromJSON(b []byte, logger *log.Logger) *jwt.Config { + var c = struct { + Email string `json:"client_email"` + PrivateKey string `json:"private_key"` + }{} + json.Unmarshal(b, &c) + config := &jwt.Config{ + Email: c.Email, + PrivateKey: []byte(c.PrivateKey), + Scopes: []string{ + drive.DriveScope, + drive.DriveMetadataScope, + }, + TokenURL: google.JWTTokenURL, + } + return config +} + // Retrieve a token, saves the token, then returns the generated client. func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile) From 9cd6e4f59478b0224c17de25e6feca5dc850391f Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 25 Oct 2022 20:23:17 +0900 Subject: [PATCH 02/17] GDrive: *FIX* fieldNotWritable &drive.File Size => fieldNotWritable error --- server/storage/gdrive.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 55447957..8b87184c 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -303,7 +303,6 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader Name: token, Parents: []string{s.rootID}, MimeType: gDriveDirectoryMimeType, - Size: int64(contentLength), } di, err := s.service.Files.Create(dir).Fields("id").Do() From f4f04b967ee94b56476575c9b01884cfa1a04396 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 25 Oct 2022 20:25:34 +0900 Subject: [PATCH 03/17] GDrive: Support Shared drive SupportsAllDrives(true) => all action IncludeItemsFromAllDrives(true) => List --- server/storage/gdrive.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 8b87184c..9b5f5495 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -91,7 +91,7 @@ func (s *GDrive) setupRoot() error { MimeType: gDriveDirectoryMimeType, } - di, err := s.service.Files.Create(dir).Fields("id").Do() + di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do() if err != nil { return err } @@ -110,7 +110,7 @@ func (s *GDrive) hasChecksum(f *drive.File) bool { } func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) { - return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do() + return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Do() } func (s *GDrive) findID(filename string, token string) (string, error) { @@ -191,7 +191,7 @@ func (s *GDrive) Head(ctx context.Context, token string, filename string) (conte } var fi *drive.File - if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil { + if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").SupportsAllDrives(true).Do(); err != nil { return } @@ -209,7 +209,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader } var fi *drive.File - fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do() + fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").SupportsAllDrives(true).Do() if err != nil { return } @@ -234,7 +234,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader // Delete removes a file from storage func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) { metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token) - _ = s.service.Files.Delete(metadata).Do() + _ = s.service.Files.Delete(metadata).SupportsAllDrives(true).Do() var fileID string fileID, err = s.findID(filename, token) @@ -242,7 +242,7 @@ func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err return } - err = s.service.Files.Delete(fileID).Context(ctx).Do() + err = s.service.Files.Delete(fileID).Context(ctx).SupportsAllDrives(true).Do() return } @@ -259,7 +259,7 @@ func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) { for 0 < len(l.Files) { for _, fi := range l.Files { - err = s.service.Files.Delete(fi.Id).Context(ctx).Do() + err = s.service.Files.Delete(fi.Id).Context(ctx).SupportsAllDrives(true).Do() if err != nil { return } @@ -305,7 +305,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader MimeType: gDriveDirectoryMimeType, } - di, err := s.service.Files.Create(dir).Fields("id").Do() + di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do() if err != nil { return err } @@ -320,7 +320,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader MimeType: contentType, } - _, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do() + _, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).SupportsAllDrives(true).Do() if err != nil { return err From f31820735e295050ffe57293a8281c86a8d0863d Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 25 Oct 2022 20:37:52 +0900 Subject: [PATCH 04/17] GDrive: jwt import --- server/storage/gdrive.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 9b5f5495..484d98c3 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -15,6 +15,7 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" "google.golang.org/api/option" From 26753509c1499e75e8f5bb401bc5818ff7c1ba18 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 25 Oct 2022 20:41:51 +0900 Subject: [PATCH 05/17] GDrive: credential type logging Done --- server/storage/gdrive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 484d98c3..31212451 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -49,10 +49,10 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir var httpClient *http.Client if strings.Contains(string(b), `"type": "service_account"`) { - logger.Println("service account") + logger.Println("GDrive: using Service Account credentials") httpClient = JWTConfigFromJSON(b, logger).Client(ctx) } else { - logger.Println("user account") + logger.Println("GDrive: using OAuth2 credentials") config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) if err != nil { return nil, err From cd9c56e96237721fd295cea848ca80aab5a5a385 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Wed, 26 Oct 2022 09:44:26 +0900 Subject: [PATCH 06/17] GDrive: Check gdrive-local-config-path on Oauth2 Check localConfigPath == "" on NEWGDriveStorage() --- cmd/cmd.go | 3 +-- server/storage/gdrive.go | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 180ec8dd..99d925d3 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -471,11 +471,10 @@ func New() *Cmd { } case "gdrive": chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024 + localConfigPath = c.String("gdrive-local-config-path") if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" { panic("client-json-filepath not set.") - } else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" { - panic("local-config-path not set.") } else if basedir := c.String("basedir"); basedir == "" { panic("basedir not set.") } else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil { diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 31212451..0e414869 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -52,6 +52,9 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir logger.Println("GDrive: using Service Account credentials") httpClient = JWTConfigFromJSON(b, logger).Client(ctx) } else { + if localConfigPath == "" { + panic("local-config-path not set.") + } logger.Println("GDrive: using OAuth2 credentials") config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) if err != nil { From 73ac766cd7486ae5c452383e7961d71da1350a56 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Wed, 26 Oct 2022 09:45:38 +0900 Subject: [PATCH 07/17] Revert "GDrive: Check gdrive-local-config-path on Oauth2" This reverts commit cd9c56e96237721fd295cea848ca80aab5a5a385. --- cmd/cmd.go | 3 ++- server/storage/gdrive.go | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 99d925d3..180ec8dd 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -471,10 +471,11 @@ func New() *Cmd { } case "gdrive": chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024 - localConfigPath = c.String("gdrive-local-config-path") if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" { panic("client-json-filepath not set.") + } else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" { + panic("local-config-path not set.") } else if basedir := c.String("basedir"); basedir == "" { panic("basedir not set.") } else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil { diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 0e414869..31212451 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -52,9 +52,6 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir logger.Println("GDrive: using Service Account credentials") httpClient = JWTConfigFromJSON(b, logger).Client(ctx) } else { - if localConfigPath == "" { - panic("local-config-path not set.") - } logger.Println("GDrive: using OAuth2 credentials") config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) if err != nil { From fb87a5b32f8acda48d5a7f8ec582ce4f5bdee8c8 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Wed, 26 Oct 2022 13:03:40 +0900 Subject: [PATCH 08/17] Update getGDriveClientFromServiceAccount JWTConfigFromJson to getGDriveClientFromServiceAccount --- server/storage/gdrive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 31212451..521aae82 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -50,7 +50,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir if strings.Contains(string(b), `"type": "service_account"`) { logger.Println("GDrive: using Service Account credentials") - httpClient = JWTConfigFromJSON(b, logger).Client(ctx) + httpClient = getGDriveClientFromServiceAccount(b).Client(ctx) } else { logger.Println("GDrive: using OAuth2 credentials") config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) @@ -331,7 +331,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader } // Retrieve a ServiceAccountJson, return a jwt.Config -func JWTConfigFromJSON(b []byte, logger *log.Logger) *jwt.Config { +func getGDriveClientFromServiceAccount(b []byte) *jwt.Config { var c = struct { Email string `json:"client_email"` PrivateKey string `json:"private_key"` From f0963001c3e330dd2c3ef0cefe796382c58a6cb4 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Wed, 26 Oct 2022 13:04:07 +0900 Subject: [PATCH 09/17] GDrive: getGDriveClientFromToken --- server/storage/gdrive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 521aae82..b1a4127a 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -57,7 +57,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir if err != nil { return nil, err } - httpClient = getGDriveClient(ctx, config, localConfigPath, logger) + httpClient = getGDriveClientFromToken(ctx, config, localConfigPath, logger) } srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) @@ -350,7 +350,7 @@ func getGDriveClientFromServiceAccount(b []byte) *jwt.Config { } // Retrieve a token, saves the token, then returns the generated client. -func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { +func getGDriveClientFromToken(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile) tok, err := gDriveTokenFromFile(tokenFile) if err != nil { From 71d8858570c93f2c403901fbb1e589b36078cb4a Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Thu, 27 Oct 2022 11:16:16 +0900 Subject: [PATCH 10/17] GDrive: SupportsAllDrives on download --- server/storage/gdrive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index b1a4127a..6e630924 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -222,7 +222,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader contentLength = uint64(fi.Size) var res *http.Response - res, err = s.service.Files.Get(fileID).Context(ctx).Download() + res, err = s.service.Files.Get(fileID).Context(ctx).SupportsAllDrives(true).Download() if err != nil { return } From f73922d04a788aaeee4c03eb1bdbc15d51814146 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Thu, 27 Oct 2022 14:41:40 +0900 Subject: [PATCH 11/17] GDrive: authType on GDrive struct --- server/storage/gdrive.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 6e630924..d11a1c0a 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -27,6 +27,7 @@ type GDrive struct { rootID string basedir string localConfigPath string + authType string chunkSize int logger *log.Logger } @@ -47,11 +48,16 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir // If modifying these scopes, delete your previously saved client_secret.json. var httpClient *http.Client + var AuthType string if strings.Contains(string(b), `"type": "service_account"`) { + AuthType = "service_account" + logger.Println("GDrive: using Service Account credentials") httpClient = getGDriveClientFromServiceAccount(b).Client(ctx) } else { + AuthType = "user_account" + logger.Println("GDrive: using OAuth2 credentials") config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) if err != nil { @@ -65,7 +71,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir return nil, err } - storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger} + storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, authType: AuthType, chunkSize: chunkSize, logger: logger} err = storage.setupRoot() if err != nil { return nil, err From f517bdc01be0bffdac0802009afe757f465c45f8 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Thu, 27 Oct 2022 15:06:28 +0900 Subject: [PATCH 12/17] GDrive: JWT to google.JWTConfigFromJSON --- server/storage/gdrive.go | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index d11a1c0a..f087b3c4 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -15,7 +15,6 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/google" - "golang.org/x/oauth2/jwt" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" "google.golang.org/api/option" @@ -54,7 +53,11 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir AuthType = "service_account" logger.Println("GDrive: using Service Account credentials") - httpClient = getGDriveClientFromServiceAccount(b).Client(ctx) + config, err := google.JWTConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) + if err != nil { + return nil, err + } + httpClient = config.Client(ctx) } else { AuthType = "user_account" @@ -336,25 +339,6 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader return nil } -// Retrieve a ServiceAccountJson, return a jwt.Config -func getGDriveClientFromServiceAccount(b []byte) *jwt.Config { - var c = struct { - Email string `json:"client_email"` - PrivateKey string `json:"private_key"` - }{} - json.Unmarshal(b, &c) - config := &jwt.Config{ - Email: c.Email, - PrivateKey: []byte(c.PrivateKey), - Scopes: []string{ - drive.DriveScope, - drive.DriveMetadataScope, - }, - TokenURL: google.JWTTokenURL, - } - return config -} - // Retrieve a token, saves the token, then returns the generated client. func getGDriveClientFromToken(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile) From 03271222cc933fbb851d82ffba129ef364a14bd9 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Thu, 27 Oct 2022 15:06:54 +0900 Subject: [PATCH 13/17] GDrive: Token Redirecturl to urn:ietf:wg:oauth:2.0:oob --- server/storage/gdrive.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index f087b3c4..ec167c43 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -353,10 +353,13 @@ func getGDriveClientFromToken(ctx context.Context, config *oauth2.Config, localC // Request a token from the web, then returns the retrieved token. func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token { + config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob" authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) + fmt.Printf("Go to the following link in your browser then type the "+ - "authorization code: \n%v\n", authURL) + "authorization code.\n%v\n", authURL) + fmt.Printf("Authorization code: ") var authCode string if _, err := fmt.Scan(&authCode); err != nil { logger.Fatalf("Unable to read authorization code %v", err) From be00bb8afa1aeed07c3a7fdf75d3b3d362402ef6 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Thu, 27 Oct 2022 15:27:46 +0900 Subject: [PATCH 14/17] GDrive: basedir as rootID --- cmd/cmd.go | 7 +++--- server/storage/gdrive.go | 53 ++++++++++++++-------------------------- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 180ec8dd..29af87c4 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -2,12 +2,12 @@ package cmd import ( "fmt" - "github.com/dutchcoders/transfer.sh/server/storage" "log" "os" "strings" "github.com/dutchcoders/transfer.sh/server" + "github.com/dutchcoders/transfer.sh/server/storage" "github.com/fatih/color" "github.com/urfave/cli" "google.golang.org/api/googleapi" @@ -471,11 +471,10 @@ func New() *Cmd { } case "gdrive": chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024 + localConfigPath := c.String("gdrive-local-config-path") if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" { - panic("client-json-filepath not set.") - } else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" { - panic("local-config-path not set.") + panic("gdrive-client-json-filepath not set.") } else if basedir := c.String("basedir"); basedir == "" { panic("basedir not set.") } else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil { diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index ec167c43..73bb845a 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -24,14 +24,12 @@ import ( type GDrive struct { service *drive.Service rootID string - basedir string localConfigPath string authType string chunkSize int logger *log.Logger } -const gDriveRootConfigFile = "root_id.conf" const gDriveTokenJSONFile = "token.json" const gDriveDirectoryMimeType = "application/vnd.google-apps.folder" @@ -50,7 +48,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir var AuthType string if strings.Contains(string(b), `"type": "service_account"`) { - AuthType = "service_account" + AuthType = "service" logger.Println("GDrive: using Service Account credentials") config, err := google.JWTConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) @@ -59,7 +57,11 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir } httpClient = config.Client(ctx) } else { - AuthType = "user_account" + AuthType = "oauth" + + if localConfigPath == "" { + return nil, fmt.Errorf("gdrive-local-config-path not set") + } logger.Println("GDrive: using OAuth2 credentials") config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) @@ -74,8 +76,8 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir return nil, err } - storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, authType: AuthType, chunkSize: chunkSize, logger: logger} - err = storage.setupRoot() + storage := &GDrive{service: srv, rootID: basedir, localConfigPath: localConfigPath, authType: AuthType, chunkSize: chunkSize, logger: logger} + err = storage.checkRoot() if err != nil { return nil, err } @@ -83,36 +85,17 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir return storage, nil } -func (s *GDrive) setupRoot() error { - rootFileConfig := filepath.Join(s.localConfigPath, gDriveRootConfigFile) - - rootID, err := ioutil.ReadFile(rootFileConfig) - if err != nil && !os.IsNotExist(err) { - return err - } - - if string(rootID) != "" { - s.rootID = string(rootID) - return nil - } - - dir := &drive.File{ - Name: s.basedir, - MimeType: gDriveDirectoryMimeType, - } - - di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do() - if err != nil { - return err - } - - s.rootID = di.Id - err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600)) - if err != nil { - return err +func (s *GDrive) checkRoot() error { + if s.rootID == "root" { + switch s.authType { + case "service": + return fmt.Errorf("GDrive: Folder \"root\" is not available when using Service Account credentials") + case "oauth": + s.logger.Println("GDrive: Warning: Folder \"root\" is not recommended.") + } } - - return nil + _, err := s.service.Files.Get(s.rootID).SupportsAllDrives(true).Do() + return err } func (s *GDrive) hasChecksum(f *drive.File) bool { From 91492e152d851b4f0cd6d66fb75bfe8056ed89a6 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Mon, 31 Oct 2022 09:19:23 +0900 Subject: [PATCH 15/17] GDrive: Add gdrive-auth-type --- cmd/cmd.go | 10 +++++++++- server/storage/gdrive.go | 18 +++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 29af87c4..ab5280d8 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -169,6 +169,12 @@ var globalFlags = []cli.Flag{ Value: "", EnvVar: "GDRIVE_CLIENT_JSON_FILEPATH", }, + cli.StringFlag{ + Name: "gdrive-auth-type", + Usage: "oauth2|service", + Value: "", + EnvVar: "GDRIVE_AUTH_TYPE", + }, cli.StringFlag{ Name: "gdrive-local-config-path", Usage: "", @@ -477,7 +483,9 @@ func New() *Cmd { panic("gdrive-client-json-filepath not set.") } else if basedir := c.String("basedir"); basedir == "" { panic("basedir not set.") - } else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil { + } else if authType := c.String("gdrive-auth-type"); authType == "" { + panic("gdrive-auth-type not set.") + } else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, authType, chunkSize, logger); err != nil { panic(err) } else { options = append(options, server.UseStorage(store)) diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index 73bb845a..ca0fa370 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -34,7 +34,7 @@ const gDriveTokenJSONFile = "token.json" const gDriveDirectoryMimeType = "application/vnd.google-apps.folder" // NewGDriveStorage is the factory for GDrive -func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) { +func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, authType string, chunkSize int, logger *log.Logger) (*GDrive, error) { ctx := context.TODO() @@ -45,20 +45,17 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir // If modifying these scopes, delete your previously saved client_secret.json. var httpClient *http.Client - var AuthType string - - if strings.Contains(string(b), `"type": "service_account"`) { - AuthType = "service" + switch authType { + case "service": // Using Service Account credentials logger.Println("GDrive: using Service Account credentials") config, err := google.JWTConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) if err != nil { return nil, err } httpClient = config.Client(ctx) - } else { - AuthType = "oauth" + case "oauth2": // Using OAuth2 credentials if localConfigPath == "" { return nil, fmt.Errorf("gdrive-local-config-path not set") } @@ -69,6 +66,9 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir return nil, err } httpClient = getGDriveClientFromToken(ctx, config, localConfigPath, logger) + + default: + return nil, fmt.Errorf("invalid gdrive-auth-type: %s", authType) } srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) @@ -76,7 +76,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir return nil, err } - storage := &GDrive{service: srv, rootID: basedir, localConfigPath: localConfigPath, authType: AuthType, chunkSize: chunkSize, logger: logger} + storage := &GDrive{service: srv, rootID: basedir, localConfigPath: localConfigPath, authType: authType, chunkSize: chunkSize, logger: logger} err = storage.checkRoot() if err != nil { return nil, err @@ -90,7 +90,7 @@ func (s *GDrive) checkRoot() error { switch s.authType { case "service": return fmt.Errorf("GDrive: Folder \"root\" is not available when using Service Account credentials") - case "oauth": + case "oauth2": s.logger.Println("GDrive: Warning: Folder \"root\" is not recommended.") } } From 478341c15280f520dec5438e08dcb18ea73830a4 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 1 Nov 2022 10:22:23 +0900 Subject: [PATCH 16/17] GDrive: service to service_account --- cmd/cmd.go | 2 +- server/storage/gdrive.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index ab5280d8..320a040e 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -171,7 +171,7 @@ var globalFlags = []cli.Flag{ }, cli.StringFlag{ Name: "gdrive-auth-type", - Usage: "oauth2|service", + Usage: "oauth2|service_account", Value: "", EnvVar: "GDRIVE_AUTH_TYPE", }, diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index ca0fa370..e72ed9e1 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -47,7 +47,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir var httpClient *http.Client switch authType { - case "service": // Using Service Account credentials + case "service_account": // Using Service Account credentials logger.Println("GDrive: using Service Account credentials") config, err := google.JWTConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) if err != nil { @@ -88,7 +88,7 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir func (s *GDrive) checkRoot() error { if s.rootID == "root" { switch s.authType { - case "service": + case "service_account": return fmt.Errorf("GDrive: Folder \"root\" is not available when using Service Account credentials") case "oauth2": s.logger.Println("GDrive: Warning: Folder \"root\" is not recommended.") From c96cf410c27fcaae7b0b85390b9922611cfa6d31 Mon Sep 17 00:00:00 2001 From: DoyunShin Date: Tue, 1 Nov 2022 10:22:37 +0900 Subject: [PATCH 17/17] GDrive: README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50099a2e..faa89c59 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ proxy-path | path prefix when service is run behind a proxy | | PROXY_PATH | proxy-port | port of the proxy when the service is run behind a proxy | | PROXY_PORT | email-contact | email contact for the front end | | EMAIL_CONTACT | ga-key | google analytics key for the front end | | GA_KEY | -provider | which storage provider to use | (s3, storj, gdrive or local) | +provider | which storage provider to use | (s3, storj, gdrive or local) | PROVIDER | uservoice-key | user voice key for the front end | | USERVOICE_KEY | aws-access-key | aws access key | | AWS_ACCESS_KEY | aws-secret-key | aws access key | | AWS_SECRET_KEY | @@ -107,9 +107,10 @@ storj-bucket | Bucket to use within the project | | STORJ_BUCKET | basedir | path storage for local/gdrive provider | | BASEDIR | gdrive-client-json-filepath | path to oauth client json config for gdrive provider | | GDRIVE_CLIENT_JSON_FILEPATH | gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| | GDRIVE_LOCAL_CONFIG_PATH | +gdrive-auth-type | which auth type to use for gdrive provider | (oauth or service_account) | GDRIVE_AUTH_TYPE | gdrive-chunk-size | chunk size for gdrive upload in megabytes, must be lower than available memory (8 MB) | | GDRIVE_CHUNK_SIZE | lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated) | | HOSTS | -log | path to log file| | LOG | +log | path to log file | | LOG | cors-domains | comma separated list of domains for CORS, setting it enable CORS | | CORS_DOMAINS | clamav-host | host for clamav feature | | CLAMAV_HOST | perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN |