Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GDrive: Fix fieldNotWritable, Add SupportSharedDrive, Add service_account method #512

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open

GDrive: Fix fieldNotWritable, Add SupportSharedDrive, Add service_account method #512

Changes from 5 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
106609d
GDrive: Support service_account
DoyunShin Oct 25, 2022
9cd6e4f
GDrive: *FIX* fieldNotWritable
DoyunShin Oct 25, 2022
f4f04b9
GDrive: Support Shared drive
DoyunShin Oct 25, 2022
f318207
GDrive: jwt import
DoyunShin Oct 25, 2022
2675350
GDrive: credential type logging
DoyunShin Oct 25, 2022
cd9c56e
GDrive: Check gdrive-local-config-path on Oauth2
DoyunShin Oct 26, 2022
73ac766
Revert "GDrive: Check gdrive-local-config-path on Oauth2"
DoyunShin Oct 26, 2022
fb87a5b
Update getGDriveClientFromServiceAccount
DoyunShin Oct 26, 2022
f096300
GDrive: getGDriveClientFromToken
DoyunShin Oct 26, 2022
71d8858
GDrive: SupportsAllDrives on download
DoyunShin Oct 27, 2022
f73922d
GDrive: authType on GDrive struct
DoyunShin Oct 27, 2022
f517bdc
GDrive: JWT to google.JWTConfigFromJSON
DoyunShin Oct 27, 2022
0327122
GDrive: Token Redirecturl to urn:ietf:wg:oauth:2.0:oob
DoyunShin Oct 27, 2022
be00bb8
GDrive: basedir as rootID
DoyunShin Oct 27, 2022
91492e1
GDrive: Add gdrive-auth-type
DoyunShin Oct 31, 2022
478341c
GDrive: service to service_account
DoyunShin Nov 1, 2022
c96cf41
GDrive: README.md
DoyunShin Nov 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 41 additions & 15 deletions server/storage/gdrive.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"golang.org/x/oauth2/jwt"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
Expand Down Expand Up @@ -45,13 +46,20 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir
}

// If modifying these scopes, delete your previously saved client_secret.json.
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
var httpClient *http.Client

if strings.Contains(string(b), `"type": "service_account"`) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's have a separate cli param for service account file

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

param like --gdrive-type=service / oauth2?
or we can actually check in json.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oauth2
{ "installed": { "client_id": "clientid", "project_id": "projectname", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_secret": "clientsecret" } }

Service Account
{ "type": "service_account", "project_id": "projectname", "private_key_id": "keyid", "private_key": "-----BEGIN PRIVATE KEY-----\nKEYSECRET\n-----END PRIVATE KEY-----\n", "client_email": "[email protected]", "client_id": "clientid", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/transfersh%40projectname.iam.gserviceaccount.com" }

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

param like --gdrive-type=service / oauth2?
or we can actually check in json.

param like gdrive-client-json-filepath: gdrive-service-account-json-filepath

only one of the two can be set

in the factory you can translate this to a single param with the path and a new param for the type (service account/oauth2): it is up to you

logger.Println("GDrive: using Service Account credentials")
httpClient = JWTConfigFromJSON(b, logger).Client(ctx)
DoyunShin marked this conversation as resolved.
Show resolved Hide resolved
} else {
logger.Println("GDrive: using OAuth2 credentials")
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
}
httpClient = getGDriveClient(ctx, config, localConfigPath, logger)
DoyunShin marked this conversation as resolved.
Show resolved Hide resolved
}

httpClient := getGDriveClient(ctx, config, localConfigPath, logger)

srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient))
if err != nil {
return nil, err
Expand Down Expand Up @@ -84,7 +92,7 @@ func (s *GDrive) setupRoot() error {
MimeType: gDriveDirectoryMimeType,
}

di, err := s.service.Files.Create(dir).Fields("id").Do()
di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do()
DoyunShin marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return err
}
Expand All @@ -103,7 +111,7 @@ func (s *GDrive) hasChecksum(f *drive.File) bool {
}

func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Do()
DoyunShin marked this conversation as resolved.
Show resolved Hide resolved
}

func (s *GDrive) findID(filename string, token string) (string, error) {
Expand Down Expand Up @@ -184,7 +192,7 @@ func (s *GDrive) Head(ctx context.Context, token string, filename string) (conte
}

var fi *drive.File
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil {
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").SupportsAllDrives(true).Do(); err != nil {
return
}

Expand All @@ -202,7 +210,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader
}

var fi *drive.File
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").SupportsAllDrives(true).Do()
if err != nil {
return
}
Expand All @@ -227,15 +235,15 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader
// Delete removes a file from storage
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
_ = s.service.Files.Delete(metadata).Do()
_ = s.service.Files.Delete(metadata).SupportsAllDrives(true).Do()

var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}

err = s.service.Files.Delete(fileID).Context(ctx).Do()
err = s.service.Files.Delete(fileID).Context(ctx).SupportsAllDrives(true).Do()
return
}

Expand All @@ -252,7 +260,7 @@ func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {

for 0 < len(l.Files) {
for _, fi := range l.Files {
err = s.service.Files.Delete(fi.Id).Context(ctx).Do()
err = s.service.Files.Delete(fi.Id).Context(ctx).SupportsAllDrives(true).Do()
if err != nil {
return
}
Expand Down Expand Up @@ -296,10 +304,9 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
Name: token,
Parents: []string{s.rootID},
MimeType: gDriveDirectoryMimeType,
Size: int64(contentLength),
}

di, err := s.service.Files.Create(dir).Fields("id").Do()
di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do()
if err != nil {
return err
}
Expand All @@ -314,7 +321,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
MimeType: contentType,
}

_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).SupportsAllDrives(true).Do()

if err != nil {
return err
Expand All @@ -323,6 +330,25 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
return nil
}

// Retrieve a ServiceAccountJson, return a jwt.Config
func JWTConfigFromJSON(b []byte, logger *log.Logger) *jwt.Config {
var c = struct {
Email string `json:"client_email"`
PrivateKey string `json:"private_key"`
}{}
json.Unmarshal(b, &c)
config := &jwt.Config{
Email: c.Email,
PrivateKey: []byte(c.PrivateKey),
Scopes: []string{
drive.DriveScope,
drive.DriveMetadataScope,
},
TokenURL: google.JWTTokenURL,
}
return config
}

// Retrieve a token, saves the token, then returns the generated client.
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile)
Expand Down