Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
kekyo committed May 28, 2024
2 parents 62262b2 + a2eb69e commit 5c8763b
Show file tree
Hide file tree
Showing 20 changed files with 395 additions and 202 deletions.
6 changes: 5 additions & 1 deletion FSharp.GitReader/Primitive/RepositoryFactoryExtension.fs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
namespace GitReader.Primitive

open GitReader
open GitReader.IO
open System.Threading

[<AutoOpen>]
Expand All @@ -18,4 +19,7 @@ module public RepositoryFactoryExtension =
type RepositoryFactory with
member _.openPrimitive(path: string, ?ct: CancellationToken) =
PrimitiveRepositoryFacade.OpenPrimitiveAsync(
path, unwrapCT ct) |> Async.AwaitTask
path, new StandardFileSystem(65536), unwrapCT ct) |> Async.AwaitTask
member _.openPrimitive(path: string, fileSystem: IFileSystem, ?ct: CancellationToken) =
PrimitiveRepositoryFacade.OpenPrimitiveAsync(
path, fileSystem, unwrapCT ct) |> Async.AwaitTask
6 changes: 5 additions & 1 deletion FSharp.GitReader/Structures/RepositoryFactoryExtension.fs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
namespace GitReader.Structures

open GitReader
open GitReader.IO
open System.Threading

[<AutoOpen>]
Expand All @@ -18,4 +19,7 @@ module public RepositoryFactoryExtension =
type RepositoryFactory with
member _.openStructured(path: string, ?ct: CancellationToken) =
StructuredRepositoryFacade.OpenStructuredAsync(
path, unwrapCT ct) |> Async.AwaitTask
path, new StandardFileSystem(65536), unwrapCT ct) |> Async.AwaitTask
member _.openStructured(path: string, fileSystem: IFileSystem, ?ct: CancellationToken) =
StructuredRepositoryFacade.OpenStructuredAsync(
path, fileSystem, unwrapCT ct) |> Async.AwaitTask
8 changes: 5 additions & 3 deletions GitReader.Core/IO/DeltaDecodedStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -612,11 +612,13 @@ public override void Flush() =>
#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP
public static async ValueTask<DeltaDecodedStream> CreateAsync(
Stream baseObjectStream, Stream deltaStream,
BufferPool pool, CancellationToken ct)
BufferPool pool, IFileSystem fileSystem,
CancellationToken ct)
#else
public static async Task<DeltaDecodedStream> CreateAsync(
Stream baseObjectStream, Stream deltaStream,
BufferPool pool, CancellationToken ct)
BufferPool pool, IFileSystem fileSystem,
CancellationToken ct)
#endif
{
void Throw(int step) =>
Expand Down Expand Up @@ -653,7 +655,7 @@ void Throw(int step) =>
}

return new(
await MemoizedStream.CreateAsync(baseObjectStream, (long)baseObjectLength, pool, ct),
await MemoizedStream.CreateAsync(baseObjectStream, (long)baseObjectLength, pool, fileSystem, ct),
deltaStream,
preloadBuffer.Detach(), preloadIndex, read, (long)decodedObjectLength);
}
Expand Down
27 changes: 20 additions & 7 deletions GitReader.Core/IO/FileStreamCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices.ComTypes;
using System.Threading;
using System.Threading.Tasks;

Expand All @@ -25,15 +26,14 @@ private sealed class CachedStream : Stream
// and the timing is managed by FileStreamCache.

private FileStreamCache parent;
private FileStream rawStream;
private Stream rawStream;
internal readonly string path;

public CachedStream(FileStreamCache parent, string path)
public CachedStream(FileStreamCache parent, string path, Stream rawStream)
{
this.parent = parent;
this.path = path;
this.rawStream = new(
path, FileMode.Open, FileAccess.Read, FileShare.Read, 65536, true);
this.rawStream = rawStream;
}

public override bool CanRead =>
Expand Down Expand Up @@ -118,9 +118,13 @@ public override void Flush() =>

internal static readonly int MaxReservedStreams = Environment.ProcessorCount * 2;

private readonly IFileSystem fileSystem;
private readonly Dictionary<string, LinkedList<CachedStream>> reserved = new();
private readonly LinkedList<CachedStream> streamsLRU = new();

public FileStreamCache(IFileSystem fileSystem) =>
this.fileSystem = fileSystem;

public void Dispose()
{
lock (this.reserved)
Expand Down Expand Up @@ -172,9 +176,15 @@ private void Return(CachedStream stream)
}
}

public Stream Open(string path)
#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP
public async ValueTask<Stream> OpenAsync(
string path, CancellationToken ct)
#else
public async Task<Stream> OpenAsync(
string path, CancellationToken ct)
#endif
{
var fullPath = Path.GetFullPath(path);
var fullPath = this.fileSystem.GetFullPath(path);

lock (this.reserved)
{
Expand Down Expand Up @@ -202,8 +212,11 @@ public Stream Open(string path)
{
this.RemoveLastReserved();
}
return new CachedStream(this, fullPath);
}
}

var stream2 = await this.fileSystem.OpenAsync(path, true, ct);

return new CachedStream(this, path, stream2);
}
}
191 changes: 191 additions & 0 deletions GitReader.Core/IO/IFileSystem.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
////////////////////////////////////////////////////////////////////////////
//
// GitReader - Lightweight Git local repository traversal library.
// Copyright (c) Kouji Matsui (@kozy_kekyo, @[email protected])
//
// Licensed under Apache-v2: https://opensource.org/licenses/Apache-2.0
//
////////////////////////////////////////////////////////////////////////////

using GitReader.Internal;
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace GitReader.IO;

public readonly struct TemporaryFileDescriptor
{
public readonly string Path;
public readonly Stream Stream;

public TemporaryFileDescriptor(string path, Stream stream)
{
this.Path = path;
this.Stream = stream;
}

public void Deconstruct(out string path, out Stream stream)
{
path = this.Path;
stream = this.Stream;
}
}

public interface IFileSystem
{
string Combine(params string[] paths);

string GetDirectoryPath(string path);

string GetFullPath(string path);

bool IsPathRooted(string path);

string ResolveRelativePath(string basePath, string path);

Task<bool> IsFileExistsAsync(
string path, CancellationToken ct);

Task<string[]> GetFilesAsync(
string basePath, string match, CancellationToken ct);

Task<Stream> OpenAsync(
string path, bool isSeekable, CancellationToken ct);

Task<TemporaryFileDescriptor> CreateTemporaryAsync(
CancellationToken ct);
}

public sealed class StandardFileSystem : IFileSystem
{
private static readonly bool isWindows =
#if NETSTANDARD1_6
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("HOMEDRIVE"));
#else
Environment.OSVersion.Platform.ToString().Contains("Win");
#endif

private static readonly string homePath =
Path.GetFullPath(isWindows ?
$"{Environment.GetEnvironmentVariable("HOMEDRIVE") ?? "C:"}{Environment.GetEnvironmentVariable("HOMEPATH") ?? "\\"}" :
(Environment.GetEnvironmentVariable("HOME") ?? "/"));

private readonly int bufferSize;

public StandardFileSystem(int bufferSize) =>
this.bufferSize = bufferSize;

#if NET35
public string Combine(params string[] paths) =>
paths.Aggregate(Path.Combine);
#else
public string Combine(params string[] paths) =>
Path.Combine(paths);
#endif

public string GetDirectoryPath(string path) =>
Path.GetDirectoryName(path) switch
{
// Not accurate in Windows, but a compromise...
null => Path.DirectorySeparatorChar.ToString(),
"" => string.Empty,
var dp => dp,
};

public string GetFullPath(string path) =>
Path.GetFullPath(path);

public bool IsPathRooted(string path) =>
Path.IsPathRooted(path);

public string ResolveRelativePath(string basePath, string path) =>
Path.GetFullPath(Path.IsPathRooted(path) ?
path :
path.StartsWith("~/") ?
Combine(homePath, path.Substring(2)) :
Combine(basePath, path));

public Task<bool> IsFileExistsAsync(string path, CancellationToken ct) =>
Utilities.FromResult(File.Exists(path));

public Task<string[]> GetFilesAsync(
string basePath, string match, CancellationToken ct) =>
Utilities.FromResult(Directory.Exists(basePath) ?
Directory.GetFiles(basePath, match, SearchOption.AllDirectories) :
Utilities.Empty<string>());

public async Task<Stream> OpenAsync(
string path, bool isSeekable, CancellationToken ct)
{
// Many Git clients are supposed to be OK to use at the same time.
// If we try to open a file with the FileShare.Read share flag (i.e., write-protected),
// an error will occur when another Git client is opening (with non-read-sharable) the file.
// Retry here as this situation is expected to take a short time to complete.
// However, if multiple files are opened sequentially,
// a deadlock may occur depending on the order in which they are opened.
// Because they are not processed as transactions.
// If a constraint is imposed by the number of open attempts,
// and if the file cannot be opened by any means,
// degrade to FileShare.ReadWrite and attempt to open it.
// (In this case it might read the wrong, that is the value in the process of writing...)

Random? r = null;

for (var count = 0; count < 20; count++)
{
try
{
return new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
this.bufferSize, true);
}
catch (FileNotFoundException)
{
throw;
}
catch (IOException)
{
}

if (r == null)
{
r = new Random();
}

await Utilities.Delay(TimeSpan.FromMilliseconds(r.Next(10, 500)), ct);
}

// Gave up and will try to open with read-write...
return new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite,
65536,
true);
}

public Task<TemporaryFileDescriptor> CreateTemporaryAsync(
CancellationToken ct)
{
var path = this.Combine(
Path.GetTempPath(),
Path.GetTempFileName());

var stream = new FileStream(
path,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
this.bufferSize,
true);

return Utilities.FromResult(new TemporaryFileDescriptor(path, stream));
}
}
14 changes: 11 additions & 3 deletions GitReader.Core/IO/MemoizedStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -309,15 +309,23 @@ public override void Write(byte[] buffer, int offset, int count) =>

#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP
public static async ValueTask<MemoizedStream> CreateAsync(
Stream parent, long parentLength, BufferPool pool, CancellationToken ct)
Stream parent,
long parentLength,
BufferPool pool,
IFileSystem fileSystem,
CancellationToken ct)
#else
public static async Task<MemoizedStream> CreateAsync(
Stream parent, long parentLength, BufferPool pool, CancellationToken ct)
Stream parent,
long parentLength,
BufferPool pool,
IFileSystem fileSystem,
CancellationToken ct)
#endif
{
if (parentLength >= memoizeToFileSize)
{
var temporaryFile = TemporaryFile.CreateFile();
var temporaryFile = await TemporaryFile.CreateFileAsync(fileSystem, ct);
return new(parent, parentLength, temporaryFile, temporaryFile.Stream, pool);
}
else if (parentLength < 0)
Expand Down
23 changes: 12 additions & 11 deletions GitReader.Core/IO/TemporaryFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;



#if !NETSTANDARD1_6
using System.Runtime.ConstrainedExecution;
Expand Down Expand Up @@ -72,18 +76,15 @@ public void Dispose()
public string Path =>
(string)this.pathHandle.Target!;

public static TemporaryFile CreateFile()
#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP
public static async ValueTask<TemporaryFile> CreateFileAsync(
IFileSystem fileSystem, CancellationToken ct)
#else
public static async Task<TemporaryFile> CreateFileAsync(
IFileSystem fileSystem, CancellationToken ct)
#endif
{
var path = Utilities.Combine(
System.IO.Path.GetTempPath(),
System.IO.Path.GetTempFileName());

var stream = new FileStream(
path,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None);

var (path, stream) = await fileSystem.CreateTemporaryAsync(ct);
return new TemporaryFile(path, stream);
}
}
Loading

0 comments on commit 5c8763b

Please sign in to comment.