Skip to content

Commit

Permalink
Merge pull request #1310 from github/accept-archive-paths-with-github…
Browse files Browse the repository at this point in the history 
…-storage-in-gei

Accept archive paths when using GitHub storage
  • Loading branch information
@ArinGhazarian
ArinGhazarian authored Dec 11, 2024
2 parents 5ef3b27 + 3c678c5 commit 20bbba7
Show file tree
Hide file tree
Showing 5 changed files with 253 additions and 12 deletions.
1 change: 1 addition & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
- Add progress report to `gh [gei|bbs2gh] migrate-repo` command when uploading migration archives to Azure Blob or AWS S3
- `gh gei migrate-repo` now allows using either `--ghes-api-url` or archive paths (`--git-archive-path` and `--metadata-archive-path`).
74 changes: 72 additions & 2 deletions src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,77 @@ public void It_Falls_Back_To_Github_Target_Pat_If_Github_Source_Pat_Is_Not_Provi
}

[Fact]
public void Aws_Bucket_Name_Without_Ghes_Api_Url_Throws()
public void AwsBucketName_Validates_With_GhesApiUrl()
{
var args = new MigrateRepoCommandArgs
{
GithubSourceOrg = SOURCE_ORG,
SourceRepo = SOURCE_REPO,
GithubTargetOrg = TARGET_ORG,
AwsBucketName = AWS_BUCKET_NAME,
GhesApiUrl = GHES_API_URL
};

args.Validate(_mockOctoLogger.Object);

args.TargetRepo.Should().Be(SOURCE_REPO);
}

[Fact]
public void AwsBucketName_Validates_With_ArchivePaths()
{
var args = new MigrateRepoCommandArgs
{
GithubSourceOrg = SOURCE_ORG,
SourceRepo = SOURCE_REPO,
GithubTargetOrg = TARGET_ORG,
AwsBucketName = AWS_BUCKET_NAME,
GitArchivePath = GIT_ARCHIVE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_PATH
};

args.Validate(_mockOctoLogger.Object);

args.TargetRepo.Should().Be(SOURCE_REPO);
}

[Fact]
public void UseGithubStorage_Validates_With_GhesApiUrl()
{
var args = new MigrateRepoCommandArgs
{
GithubSourceOrg = SOURCE_ORG,
SourceRepo = SOURCE_REPO,
GithubTargetOrg = TARGET_ORG,
GhesApiUrl = GHES_API_URL,
UseGithubStorage = true
};

args.Validate(_mockOctoLogger.Object);

args.TargetRepo.Should().Be(SOURCE_REPO);
}

[Fact]
public void UseGithubStorage_Validates_With_ArchivePaths()
{
var args = new MigrateRepoCommandArgs
{
GithubSourceOrg = SOURCE_ORG,
SourceRepo = SOURCE_REPO,
GithubTargetOrg = TARGET_ORG,
GitArchivePath = GIT_ARCHIVE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_PATH,
UseGithubStorage = true
};

args.Validate(_mockOctoLogger.Object);

args.TargetRepo.Should().Be(SOURCE_REPO);
}

[Fact]
public void Aws_Bucket_Name_Without_GhesApiUrl_Or_ArchivePaths_Throws()
{
var args = new MigrateRepoCommandArgs
{
Expand All @@ -73,7 +143,7 @@ public void Aws_Bucket_Name_Without_Ghes_Api_Url_Throws()
}

[Fact]
public void UseGithubStorage_Without_Ghes_Api_Url_Throws()
public void UseGithubStorage_Without_GhesApiUrl_Or_ArchivePaths_Throws()
{
var args = new MigrateRepoCommandArgs
{
Expand Down
169 changes: 169 additions & 0 deletions src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1510,6 +1510,175 @@ await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
.WithMessage("*AWS S3*--aws-bucket-name*");
}

[Fact]
public async Task GitArchivePath_With_Both_Azure_Storage_Connection_String_And_Aws_Bucket_Name_Throws()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AzureStorageConnectionString = AZURE_CONNECTION_STRING,
AwsBucketName = AWS_BUCKET_NAME
}))
.Should()
.ThrowAsync<OctoshiftCliException>();
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Access_Key_Id_Throws()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AwsBucketName = AWS_BUCKET_NAME,
AwsSecretKey = AWS_SECRET_ACCESS_KEY
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("*--aws-access-key*");
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Secret_Key_Throws()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AwsBucketName = AWS_BUCKET_NAME,
AwsAccessKey = AWS_ACCESS_KEY_ID
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("*--aws-secret-key*");
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Region_Throws()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AwsBucketName = AWS_BUCKET_NAME,
AwsAccessKey = AWS_ACCESS_KEY_ID,
AwsSecretKey = AWS_SECRET_ACCESS_KEY,
AwsSessionToken = AWS_SESSION_TOKEN
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("Either --aws-region or AWS_REGION environment variable must be set.");
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Not_Provided_But_Aws_Access_Key_Provided()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AzureStorageConnectionString = AZURE_CONNECTION_STRING,
AwsAccessKey = AWS_ACCESS_KEY_ID
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("*AWS S3*--aws-bucket-name*");
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Not_Provided_But_Aws_Secret_Key_Provided()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AzureStorageConnectionString = AZURE_CONNECTION_STRING,
AwsSecretKey = AWS_SECRET_ACCESS_KEY
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("*AWS S3*--aws-bucket-name*");
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Not_Provided_But_Aws_Session_Token_Provided()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AzureStorageConnectionString = AZURE_CONNECTION_STRING,
AwsSessionToken = AWS_SECRET_ACCESS_KEY
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("*AWS S3*--aws-bucket-name*");
}

[Fact]
public async Task GitArchivePath_When_Aws_Bucket_Name_Not_Provided_But_Aws_Region_Provided()
{
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);

await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
{
SourceRepo = SOURCE_REPO,
GithubSourceOrg = SOURCE_ORG,
GithubTargetOrg = TARGET_ORG,
TargetRepo = TARGET_REPO,
GitArchivePath = GIT_ARCHIVE_FILE_PATH,
MetadataArchivePath = METADATA_ARCHIVE_FILE_PATH,
AzureStorageConnectionString = AZURE_CONNECTION_STRING,
AwsRegion = AWS_REGION
}))
.Should()
.ThrowAsync<OctoshiftCliException>()
.WithMessage("*AWS S3*--aws-bucket-name*");
}

[Fact]
public async Task Keep_Archive_Does_Not_Call_DeleteIfExists()
{
Expand Down
13 changes: 7 additions & 6 deletions src/gei/Commands/MigrateRepo/MigrateRepoCommandArgs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,14 @@ public override void Validate(OctoLogger log)

if (GhesApiUrl.IsNullOrWhiteSpace())
{
if (AwsBucketName.HasValue())
if (AwsBucketName.HasValue() && GitArchivePath.IsNullOrWhiteSpace())
{
throw new OctoshiftCliException("--ghes-api-url must be specified when --aws-bucket-name is specified.");
throw new OctoshiftCliException("When using --aws-bucket-name, you must provide --ghes-api-url, or --git-archive-path and --metadata-archive-path");
}

if (UseGithubStorage && GitArchivePath.IsNullOrWhiteSpace())
{
throw new OctoshiftCliException("When using --use-github-storage, you must provide --ghes-api-url, or --git-archive-path and --metadata-archive-path");
}

if (NoSslVerify)
Expand All @@ -79,10 +84,6 @@ public override void Validate(OctoLogger log)
{
throw new OctoshiftCliException("--ghes-api-url must be specified when --keep-archive is specified.");
}
if (UseGithubStorage)
{
throw new OctoshiftCliException("--ghes-api-url must be specified when --use-github-storage is specified.");
}
}

if (AwsBucketName.HasValue() && UseGithubStorage)
Expand Down
8 changes: 4 additions & 4 deletions src/gei/Commands/MigrateRepo/MigrateRepoCommandHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@ public async Task Handle(MigrateRepoCommandArgs args)

_log.LogInformation("Migrating Repo...");

var blobCredentialsRequired = await _ghesVersionChecker.AreBlobCredentialsRequired(args.GhesApiUrl);
var blobCredentialsRequired = args.GitArchivePath.HasValue() || await _ghesVersionChecker.AreBlobCredentialsRequired(args.GhesApiUrl);

if (args.GhesApiUrl.HasValue())
if (args.GhesApiUrl.HasValue() || args.GitArchivePath.HasValue())
{
ValidateGHESOptions(args, blobCredentialsRequired);
ValidateUploadOptions(args, blobCredentialsRequired);
}

if (args.GhesApiUrl.HasValue())
Expand Down Expand Up @@ -402,7 +402,7 @@ private async Task<string> WaitForArchiveGeneration(GithubApi githubApi, string

private string GetGithubRepoUrl(string org, string repo, string baseUrl) => $"{baseUrl ?? DEFAULT_GITHUB_BASE_URL}/{org.EscapeDataString()}/{repo.EscapeDataString()}";

private void ValidateGHESOptions(MigrateRepoCommandArgs args, bool cloudCredentialsRequired)
private void ValidateUploadOptions(MigrateRepoCommandArgs args, bool cloudCredentialsRequired)
{
var shouldUseAzureStorage = GetAzureStorageConnectionString(args).HasValue();
var shouldUseAwsS3 = args.AwsBucketName.HasValue();
Expand Down

0 comments on commit 20bbba7

Please sign in to comment.