AWS uses SongImporter

Removed dependency on listing S3 bucket.

dcc-repository-aws/README.md

@@ -1,9 +1,6 @@
 # ICGC DCC - Repository - AWS
 
-This is ICGC AWS S3 repository file import module. 
-
-It's responsible for indexing the state of the S3 bucket managed by [dcc-storage-server](/../../dcc-storage/README.md). This includes the `id` of the file, its size and last modified date. The state and related metadata is derived from [ICGC-TCGA-PanCancer](https://github.com/ICGC-TCGA-PanCancer/s3-transfer-operations).
-
+This is ICGC AWS S3 repository file import module. It talks to the SONG server deployed in AWS. 
 ## Build
 
 To compile, test and package the module, execute the following from the root of the repository:

dcc-repository-aws/pom.xml

@@ -40,9 +40,9 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>dcc-repository-cloud</artifactId>
+      <artifactId>dcc-repository-song</artifactId>
       <version>${project.version}</version>
-    </dependency>    
+    </dependency>
 
   </dependencies>
 

dcc-repository-aws/src/main/java/org/icgc/dcc/repository/aws/AWSImporter.java

@@ -17,54 +17,22 @@
  */
 package org.icgc.dcc.repository.aws;
 
-import static org.icgc.dcc.repository.core.model.RepositorySource.AWS;
-
-import java.io.File;
-
-import org.icgc.dcc.repository.aws.s3.AWSClientFactory;
-import org.icgc.dcc.repository.cloud.CloudImporter;
-import org.icgc.dcc.repository.cloud.core.CloudFileProcessor;
-import org.icgc.dcc.repository.cloud.s3.CloudS3BucketReader;
-import org.icgc.dcc.repository.cloud.transfer.CloudTransferJobReader;
 import org.icgc.dcc.repository.core.RepositoryFileContext;
 import org.icgc.dcc.repository.core.model.Repositories;
 
 import lombok.NonNull;
-import lombok.val;
 import lombok.extern.slf4j.Slf4j;
+import org.icgc.dcc.repository.song.SongImporter;
 
 @Slf4j
-public class AWSImporter extends CloudImporter {
-
-  /**
-   * Constants.
-   */
-  private static final String BUCKET_NAME = "oicr.icgc";
-  private static final String BUCKET_KEY_PREFIX = "data";
-
-  private static final String GIT_REPO_URL = "https://github.com/ICGC-TCGA-PanCancer/s3-transfer-operations.git";
-  private static final String GIT_REPO_DIR_GLOB = "s3-transfer-jobs-*";
-  private static final File GIT_REPO_DIR = new File("/tmp/dcc-repository-aws");
+public class AWSImporter extends SongImporter {
 
   public AWSImporter(@NonNull RepositoryFileContext context) {
-    super(AWS, context, log);
-  }
-
-  @Override
-  protected CloudTransferJobReader createJobReader() {
-    return new CloudTransferJobReader(GIT_REPO_URL, GIT_REPO_DIR, GIT_REPO_DIR_GLOB);
-  }
-
-  @Override
-  protected CloudS3BucketReader createBucketReader() {
-    val s3 = AWSClientFactory.createS3Client();
-    return new CloudS3BucketReader(BUCKET_NAME, BUCKET_KEY_PREFIX, s3);
-  }
-
-  @Override
-  protected CloudFileProcessor createFileProcessor() {
-    val awsRepository = Repositories.getAWSRepository();
-    return new CloudFileProcessor(context, awsRepository);
+    super(
+      context,
+      Repositories.getCollabRepository(),
+      context.getAwsUrl(),
+      context.getAwsToken());
   }
 
 }

dcc-repository-client/src/main/java/org/icgc/dcc/repository/client/config/ClientConfig.java

@@ -73,7 +73,9 @@ public RepositoryFileContext context(ClientProperties properties) {
         .dccIdResolver(new DCCDonorIdResolver())
         .importMongoUri(properties.getImports().getMongoUri())
         .collabUrl(url(properties.getImports().getCollabUrl()))
-        .collabToken(properties.getImports().getCollabToken());
+        .collabToken(properties.getImports().getCollabToken())
+        .awsUrl(url(properties.getImports().getAwsUrl()))
+        .awsToken(properties.getImports().getAwsToken());
     // Outputs
     context
         .repoMongoUri(properties.getRepository().getMongoUri())

dcc-repository-client/src/main/java/org/icgc/dcc/repository/client/config/ClientProperties.java

@@ -78,6 +78,9 @@ public static class ImportsProperties {
     MongoClientURI mongoUri;
     String collabUrl;
     String collabToken;
+    String awsUrl;
+    String awsToken;
+
   }
 
   @Data

dcc-repository-core/src/main/java/org/icgc/dcc/repository/core/RepositoryFileContext.java

@@ -56,6 +56,12 @@ public class RepositoryFileContext {
   @NonNull
   @Getter
   private final String collabToken;
+  @NonNull
+  @Getter
+  private final URL awsUrl;
+  @NonNull
+  @Getter
+  private final String awsToken;
   @Getter
   @NonNull
   private final URI archiveUri;

dcc-repository-core/src/main/java/org/icgc/dcc/repository/core/RepositoryFileContextBuilder.java

@@ -73,6 +73,12 @@ public final class RepositoryFileContextBuilder {
   private String collabToken = "";
   @Setter
   @Accessors(chain = true, fluent = true)
+  private URL awsUrl = url("http://localhost:8080");
+  @Setter
+  @Accessors(chain = true, fluent=true)
+  private String awsToken = "";
+  @Setter
+  @Accessors(chain = true, fluent = true)
   private URI esUri = URIs.getUri("es://localhost:9300");
   @Setter
   @Accessors(chain = true, fluent = true)
@@ -118,7 +124,7 @@ public RepositoryFileContext build() {
     val idClient = createIdClient();
     val tcgaMappings = new TCGAMappingsReader().readMappings();
 
-    return new RepositoryFileContext(repoMongoUri, esUri, collabUrl, collabToken, archiveUri, indexAlias, skipImport, sources, readOnly,
+    return new RepositoryFileContext(repoMongoUri, esUri, collabUrl, collabToken, awsUrl, awsToken, archiveUri, indexAlias, skipImport, sources, readOnly,
         primarySites, idClient, tcgaMappings, pcawgIdResolver, dccIdResolver, report);
   }
 

pom.xml

@@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
   <parent>
     <groupId>org.icgc.dcc</groupId>
     <artifactId>dcc-parent</artifactId>
-    <version>31</version>
+    <version>35</version>
   </parent>
 
   <artifactId>dcc-repository</artifactId>