Skip to content

Commit

Permalink
Merge pull request #1311 from github/add-upload-archive-progress-repo…
Browse files Browse the repository at this point in the history 
…rt-for-aws-and-azure

Add progress report when uploading migration archives to Azure Blob or AWS S3
  • Loading branch information
@ArinGhazarian
ArinGhazarian authored Dec 11, 2024
2 parents 802bc1f + 79ee21c commit 5ef3b27
Show file tree
Hide file tree
Showing 10 changed files with 187 additions and 53 deletions.
2 changes: 1 addition & 1 deletion RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@

- Add progress report to `gh [gei|bbs2gh] migrate-repo` command when uploading migration archives to Azure Blob or AWS S3
19 changes: 19 additions & 0 deletions src/Octoshift/Extensions/NumericExtensions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
namespace OctoshiftCLI.Extensions;

public static class NumericExtensions
{
public static string ToLogFriendlySize(this long size)
{
const int kilobyte = 1024;
const int megabyte = 1024 * kilobyte;
const int gigabyte = 1024 * megabyte;

return size switch
{
< kilobyte => $"{size:n0} bytes",
< megabyte => $"{size / (double)kilobyte:n0} KB",
< gigabyte => $"{size / (double)megabyte:n0} MB",
_ => $"{size / (double)gigabyte:n2} GB"
};
}
}
60 changes: 52 additions & 8 deletions src/Octoshift/Services/AwsApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@ namespace OctoshiftCLI.Services;
public class AwsApi : IDisposable
{
private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10;

private readonly ITransferUtility _transferUtility;
private readonly object _mutex = new();
private readonly OctoLogger _log;
private DateTime _nextProgressReport = DateTime.Now;

public AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility;
public AwsApi(ITransferUtility transferUtility, OctoLogger log)
{
_transferUtility = transferUtility;
_log = log;
}

#pragma warning disable CA2000
public AwsApi(string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
: this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)))
public AwsApi(OctoLogger log, string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
: this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)), log)
#pragma warning restore CA2000
{
}
Expand Down Expand Up @@ -51,20 +59,37 @@ public virtual async Task<string> UploadToBucket(string bucketName, string fileN
{
try
{
await _transferUtility.UploadAsync(fileName, bucketName, keyName);
var uploadRequest = new TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = keyName,
FilePath = fileName
};
return await UploadToBucket(uploadRequest);
}
catch (Exception ex) when (ex is TaskCanceledException or TimeoutException)
{
throw new OctoshiftCliException($"Upload of archive \"{fileName}\" to AWS timed out", ex);
}

return GetPreSignedUrlForFile(bucketName, keyName);
}

public virtual async Task<string> UploadToBucket(string bucketName, Stream content, string keyName)
{
await _transferUtility.UploadAsync(content, bucketName, keyName);
return GetPreSignedUrlForFile(bucketName, keyName);
var uploadRequest = new TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = keyName,
InputStream = content
};
return await UploadToBucket(uploadRequest);
}

private async Task<string> UploadToBucket(TransferUtilityUploadRequest uploadRequest)
{
uploadRequest.UploadProgressEvent += (_, args) => LogProgress(args.PercentDone, args.TransferredBytes, args.TotalBytes);
await _transferUtility.UploadAsync(uploadRequest);

return GetPreSignedUrlForFile(uploadRequest.BucketName, uploadRequest.Key);
}

private string GetPreSignedUrlForFile(string bucketName, string keyName)
Expand All @@ -81,6 +106,25 @@ private string GetPreSignedUrlForFile(string bucketName, string keyName)
return _transferUtility.S3Client.GetPreSignedURL(urlRequest);
}

private void LogProgress(int percentDone, long uploadedBytes, long totalBytes)
{
lock (_mutex)
{
if (DateTime.Now < _nextProgressReport)
{
return;
}

_nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS);
}

var progressMessage = uploadedBytes > 0
? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentDone}%) completed"
: "";

_log.LogInformation($"Archive upload in progress{progressMessage}...");
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
Expand Down
32 changes: 32 additions & 0 deletions src/Octoshift/Services/AzureApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using Azure.Storage.Sas;
using OctoshiftCLI.Extensions;

namespace OctoshiftCLI.Services;

Expand All @@ -14,9 +15,12 @@ public class AzureApi
private readonly HttpClient _client;
private readonly BlobServiceClient _blobServiceClient;
private readonly OctoLogger _log;
private readonly object _mutex = new();
private const string CONTAINER_PREFIX = "migration-archives";
private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
private const int DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024;
private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10;
private DateTime _nextProgressReport = DateTime.Now;

public AzureApi(HttpClient client, BlobServiceClient blobServiceClient, OctoLogger log)
{
Expand Down Expand Up @@ -48,16 +52,24 @@ public virtual async Task<Uri> UploadToBlob(string fileName, byte[] content)

public virtual async Task<Uri> UploadToBlob(string fileName, Stream content)
{
ArgumentNullException.ThrowIfNull(fileName, nameof(fileName));
ArgumentNullException.ThrowIfNull(content);

var containerClient = await CreateBlobContainerAsync();
var blobClient = containerClient.GetBlobClient(fileName);

var progress = new Progress<long>();
var archiveSize = content.Length;
progress.ProgressChanged += (_, uploadedBytes) => LogProgress(uploadedBytes, archiveSize);

var options = new BlobUploadOptions
{
TransferOptions = new Azure.Storage.StorageTransferOptions()
{
InitialTransferSize = DEFAULT_BLOCK_SIZE,
MaximumTransferSize = DEFAULT_BLOCK_SIZE
},
ProgressHandler = progress
};

await blobClient.UploadAsync(content, options);
Expand Down Expand Up @@ -89,4 +101,24 @@ private Uri GetServiceSasUriForBlob(BlobClient blobClient)

return blobClient.GenerateSasUri(sasBuilder);
}

private void LogProgress(long uploadedBytes, long totalBytes)
{
lock (_mutex)
{
if (DateTime.Now < _nextProgressReport)
{
return;
}

_nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS);
}

var percentage = (int)(uploadedBytes * 100L / totalBytes);
var progressMessage = uploadedBytes > 0
? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentage}%) completed"
: "";

_log.LogInformation($"Archive upload in progress{progressMessage}...");
}
}
74 changes: 48 additions & 26 deletions src/Octoshift/Services/OctoLogger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public class OctoLogger
private readonly string _logFilePath;
private readonly string _verboseFilePath;
private readonly bool _debugMode;
private readonly object _mutex = new();

private readonly Action<string> _writeToLog;
private readonly Action<string> _writeToVerboseLog;
Expand Down Expand Up @@ -103,20 +104,32 @@ private string Redact(string msg)
return result;
}

public virtual void LogInformation(string msg) => Log(msg, LogLevel.INFO);
public virtual void LogInformation(string msg)
{
lock (_mutex)
{
Log(msg, LogLevel.INFO);
}
}

public virtual void LogWarning(string msg)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Log(msg, LogLevel.WARNING);
Console.ResetColor();
lock (_mutex)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Log(msg, LogLevel.WARNING);
Console.ResetColor();
}
}

public virtual void LogError(string msg)
{
Console.ForegroundColor = ConsoleColor.Red;
Log(msg, LogLevel.ERROR);
Console.ResetColor();
lock (_mutex)
{
Console.ForegroundColor = ConsoleColor.Red;
Log(msg, LogLevel.ERROR);
Console.ResetColor();
}
}

public virtual void LogError(Exception ex)
Expand All @@ -126,30 +139,36 @@ public virtual void LogError(Exception ex)
throw new ArgumentNullException(nameof(ex));
}

var verboseMessage = ex is HttpRequestException httpEx ? $"[HTTP ERROR {(int?)httpEx.StatusCode}] {ex}" : ex.ToString();
var logMessage = Verbose ? verboseMessage : ex is OctoshiftCliException ? ex.Message : GENERIC_ERROR_MESSAGE;
lock (_mutex)
{
var verboseMessage = ex is HttpRequestException httpEx ? $"[HTTP ERROR {(int?)httpEx.StatusCode}] {ex}" : ex.ToString();
var logMessage = Verbose ? verboseMessage : ex is OctoshiftCliException ? ex.Message : GENERIC_ERROR_MESSAGE;

var output = Redact(FormatMessage(logMessage, LogLevel.ERROR));
var output = Redact(FormatMessage(logMessage, LogLevel.ERROR));

Console.ForegroundColor = ConsoleColor.Red;
_writeToConsoleError(output);
Console.ResetColor();
Console.ForegroundColor = ConsoleColor.Red;
_writeToConsoleError(output);
Console.ResetColor();

_writeToLog(output);
_writeToVerboseLog(Redact(FormatMessage(verboseMessage, LogLevel.ERROR)));
_writeToLog(output);
_writeToVerboseLog(Redact(FormatMessage(verboseMessage, LogLevel.ERROR)));
}
}

public virtual void LogVerbose(string msg)
{
if (Verbose)
lock (_mutex)
{
Console.ForegroundColor = ConsoleColor.Gray;
Log(msg, LogLevel.VERBOSE);
Console.ResetColor();
}
else
{
_writeToVerboseLog(Redact(FormatMessage(msg, LogLevel.VERBOSE)));
if (Verbose)
{
Console.ForegroundColor = ConsoleColor.Gray;
Log(msg, LogLevel.VERBOSE);
Console.ResetColor();
}
else
{
_writeToVerboseLog(Redact(FormatMessage(msg, LogLevel.VERBOSE)));
}
}
}

Expand All @@ -163,9 +182,12 @@ public virtual void LogDebug(string msg)

public virtual void LogSuccess(string msg)
{
Console.ForegroundColor = ConsoleColor.Green;
Log(msg, LogLevel.SUCCESS);
Console.ResetColor();
lock (_mutex)
{
Console.ForegroundColor = ConsoleColor.Green;
Log(msg, LogLevel.SUCCESS);
Console.ResetColor();
}
}

public virtual void RegisterSecret(string secret) => _secrets.Add(secret);
Expand Down
Loading

0 comments on commit 5ef3b27

Please sign in to comment.