Skip to content

Commit

Permalink
Update Git to v2.43.0 (#4618)
Browse files Browse the repository at this point in the history 
* Update Git to a new version
Add feature flag to use different Git versions in Worker and Listener
Separate Git config and Git usage
  • Loading branch information
@ivanduplenskikh
ivanduplenskikh authored Feb 20, 2024
1 parent bcc0324 commit 0905479
Show file tree
Hide file tree
Showing 12 changed files with 417 additions and 93 deletions.
59 changes: 47 additions & 12 deletions src/Agent.Plugins/GitCliManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,12 @@

namespace Agent.Plugins.Repository
{
public class GitCliManager
public interface IGitCliManager
{
Task<int> GitConfig(AgentTaskPluginExecutionContext context, string repositoryPath, string configKey, string configValue);
}

public class GitCliManager : IGitCliManager
{
private static Encoding _encoding
{
Expand Down Expand Up @@ -77,27 +82,57 @@ public bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch)
return gitLfsVersion >= requiredVersion;
}

public (string gitPath, string gitLfsPath) GetInternalGitPaths(
AgentTaskPluginExecutionContext context,
bool useLatestGitVersion)
{
string agentHomeDir = context.Variables.GetValueOrDefault("agent.homedirectory")?.Value;
ArgUtil.NotNullOrEmpty(agentHomeDir, nameof(agentHomeDir));

string gitPath;

if (useLatestGitVersion)
{
gitPath = Path.Combine(agentHomeDir, "externals", "ff_git", "cmd", $"git.exe");
}
else
{
gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe");
}

context.Debug($@"The useLatestGitVersion property is set to ""{useLatestGitVersion}"" therefore the Git path is ""{gitPath}""");

string gitLfsPath;

if (PlatformUtil.BuiltOnX86)
{
gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", "mingw32", "bin", "git-lfs.exe");
}
else
{
gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", "mingw64", "bin", "git-lfs.exe");
}

return (gitPath, gitLfsPath);
}

public virtual async Task LoadGitExecutionInfo(AgentTaskPluginExecutionContext context, bool useBuiltInGit)
{
// There is no built-in git for OSX/Linux
gitPath = null;
gitLfsPath = null;

// Resolve the location of git.
if (useBuiltInGit && PlatformUtil.RunningOnWindows)
{
string agentHomeDir = context.Variables.GetValueOrDefault("agent.homedirectory")?.Value;
ArgUtil.NotNullOrEmpty(agentHomeDir, nameof(agentHomeDir));
context.Debug("Git paths are resolving from internal dependencies");

if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(context).AsBoolean())
{
gitPath = Path.Combine(agentHomeDir, "externals", "ff_git", "cmd", $"git.exe");
}
else
{
gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe");
}
var (resolvedGitPath, resolvedGitLfsPath) = GetInternalGitPaths(
context,
AgentKnobs.UseLatestGitVersion.GetValue(context).AsBoolean());

gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", PlatformUtil.BuiltOnX86 ? "mingw32" : "mingw64", "bin", "git-lfs.exe");
gitPath = resolvedGitPath;
gitLfsPath = resolvedGitLfsPath;

// Prepend the PATH.
context.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", "Path", Path.GetFileName(gitPath)));
Expand Down
40 changes: 28 additions & 12 deletions src/Agent.Plugins/GitSourceProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ public async Task GetSourceAsync(

// Make sure the build machine met all requirements for the git repository
// For now, the requirement we have are:
// 1. git version greater than 2.9 and git-lfs version greater than 2.1 for on-prem tfsgit
// 1. git version greater than 2.9 and git-lfs version greater than 2.1 for on-prem tfsgit
// 2. git version greater than 2.14.2 if use SChannel for SSL backend (Windows only)
RequirementCheck(executionContext, repository, gitCommandManager);
string username = string.Empty;
Expand Down Expand Up @@ -684,17 +684,7 @@ public async Task GetSourceAsync(
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching.");
}

if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1");
await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.longpaths", "true");
}
SetGitFeatureFlagsConfiguration(executionContext, gitCommandManager, targetPath);

// always remove any possible left extraheader setting from git config.
if (await gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
Expand Down Expand Up @@ -1315,6 +1305,32 @@ public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionC
}
}

public async void SetGitFeatureFlagsConfiguration(
AgentTaskPluginExecutionContext executionContext,
IGitCliManager gitCommandManager,
string targetPath)
{
if (AgentKnobs.UseGitSingleThread.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1");
}

if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m");
}

if (AgentKnobs.UseGitLongPaths.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "core.longpaths", "true");
}
}

protected virtual GitCliManager GetCliManager(Dictionary<string, string> gitEnv = null)
{
return new GitCliManager(gitEnv);
Expand Down
21 changes: 21 additions & 0 deletions src/Agent.Sdk/Knob/AgentKnobs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,27 @@ public class AgentKnobs
new EnvironmentKnobSource("FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob UseGitLongPaths = new Knob(
nameof(UseGitLongPaths),
"When true, set core.longpaths to true",
new RuntimeKnobSource("USE_GIT_LONG_PATHS"),
new EnvironmentKnobSource("USE_GIT_LONG_PATHS"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob UseGitSingleThread = new Knob(
nameof(UseGitSingleThread),
"When true, spawn only one thread searching for best delta matches",
new RuntimeKnobSource("USE_GIT_SINGLE_THREAD"),
new EnvironmentKnobSource("USE_GIT_SINGLE_THREAD"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob UseLatestGitVersion = new Knob(
nameof(UseLatestGitVersion),
"When true, set path to the latest git version",
new RuntimeKnobSource("USE_LATEST_GIT_VERSION"),
new EnvironmentKnobSource("USE_LATEST_GIT_VERSION"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob TfVCUseSecureParameterPassing = new Knob(
nameof(TfVCUseSecureParameterPassing),
"If true, don't pass auth token in TFVC parameters",
Expand Down
44 changes: 42 additions & 2 deletions src/Agent.Worker/Build/GitCommandManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ public interface IGitCommandManager : IAgentService

bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch);

(string resolvedGitPath, string resolvedGitLfsPath) GetInternalGitPaths(IExecutionContext context, bool useLatestGitVersion);

// setup git execution info, git location, version, useragent, execpath
Task LoadGitExecutionInfo(IExecutionContext context, bool useBuiltInGit, Dictionary<string, string> gitEnv = null);

Expand Down Expand Up @@ -160,6 +162,38 @@ public bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch)
return _gitLfsVersion >= requiredVersion;
}

public (string resolvedGitPath, string resolvedGitLfsPath) GetInternalGitPaths(IExecutionContext context, bool useLatestGitVersion)
{
string externalsDirectoryPath = HostContext.GetDirectory(WellKnownDirectory.Externals);
ArgUtil.NotNullOrEmpty(externalsDirectoryPath, nameof(WellKnownDirectory.Externals));

string gitPath;

if (useLatestGitVersion)
{
gitPath = Path.Combine(externalsDirectoryPath, "ff_git", "cmd", $"git.exe");
}
else
{
gitPath = Path.Combine(externalsDirectoryPath, "git", "cmd", $"git.exe");
}

context.Debug($@"The useLatestGitVersion property is set to ""{useLatestGitVersion}"" therefore the Git path is ""{gitPath}""");

string gitLfsPath;

if (PlatformUtil.BuiltOnX86)
{
gitLfsPath = Path.Combine(externalsDirectoryPath, "git", "mingw32", "bin", $"git-lfs.exe");
}
else
{
gitLfsPath = Path.Combine(externalsDirectoryPath, "git", "mingw64", "bin", $"git-lfs.exe");
}

return (gitPath, gitLfsPath);
}

public async Task LoadGitExecutionInfo(IExecutionContext context, bool useBuiltInGit, Dictionary<string, string> gitEnv = null)
{
if (gitEnv != null)
Expand All @@ -181,8 +215,14 @@ public async Task LoadGitExecutionInfo(IExecutionContext context, bool useBuiltI
// The Windows agent ships a copy of Git
if (PlatformUtil.RunningOnWindows)
{
_gitPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "git", "cmd", $"git{IOUtil.ExeExtension}");
_gitLfsPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "git", PlatformUtil.BuiltOnX86 ? "mingw32" : "mingw64", "bin", "git-lfs.exe");
context.Debug("Git paths are resolving from internal dependencies");

var (resolvedGitPath, resolvedGitLfsPath) = GetInternalGitPaths(
context,
AgentKnobs.UseLatestGitVersion.GetValue(context).AsBoolean());

_gitPath = resolvedGitPath;
_gitLfsPath = resolvedGitLfsPath;

// Prepend the PATH.
context.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", Constants.PathVariable, Path.GetFileName(_gitPath)));
Expand Down
28 changes: 28 additions & 0 deletions src/Agent.Worker/Build/GitSourceProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -657,6 +657,8 @@ public async Task GetSourceAsync(
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching.");
}

SetGitFeatureFlagsConfiguration(executionContext, _gitCommandManager, targetPath);

// always remove any possible left extraheader setting from git config.
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
{
Expand Down Expand Up @@ -1229,6 +1231,32 @@ public override async Task RunMaintenanceOperations(IExecutionContext executionC
}
}

public async void SetGitFeatureFlagsConfiguration(
IExecutionContext executionContext,
IGitCommandManager gitCommandManager,
string targetPath)
{
if (AgentKnobs.UseGitSingleThread.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1");
}

if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m");
await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m");
}

if (AgentKnobs.UseGitLongPaths.GetValue(executionContext).AsBoolean())
{
await gitCommandManager.GitConfig(executionContext, targetPath, "core.longpaths", "true");
}
}

public override void SetVariablesInEndpoint(IExecutionContext executionContext, ServiceEndpoint endpoint)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
Expand Down
8 changes: 8 additions & 0 deletions src/Microsoft.VisualStudio.Services.Agent/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ public static class Agent
public static readonly string ContainerMapping = "agent.containermapping";
public static readonly string ContainerNetwork = "agent.containernetwork";
public static readonly string Diagnostic = "agent.diagnostic";
public static readonly string FixPossibleGitOutOfMemoryProblem = "FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM";
public static readonly string HomeDirectory = "agent.homedirectory";
public static readonly string Id = "agent.id";
public static readonly string IsSelfHosted = "agent.isselfhosted";
Expand Down Expand Up @@ -351,6 +352,9 @@ public static class Agent
public static readonly string SslSkipCertValidation = "agent.skipcertvalidation";
public static readonly string TempDirectory = "agent.TempDirectory";
public static readonly string ToolsDirectory = "agent.ToolsDirectory";
public static readonly string UseGitLongPaths = "USE_GIT_LONG_PATHS";
public static readonly string UseGitSingleThread = "USE_GIT_SINGLE_THREAD";
public static readonly string UseLatestGitVersion = "USE_LATEST_GIT_VERSION";
public static readonly string Version = "agent.version";
public static readonly string WorkFolder = "agent.workfolder";
public static readonly string WorkingDirectory = "agent.WorkingDirectory";
Expand Down Expand Up @@ -522,6 +526,7 @@ public static class Task
Agent.ContainerMapping,
Agent.ContainerNetwork,
Agent.Diagnostic,
Agent.FixPossibleGitOutOfMemoryProblem,
Agent.GitUseSChannel,
Agent.HomeDirectory,
Agent.Id,
Expand Down Expand Up @@ -551,6 +556,9 @@ public static class Task
Agent.SslSkipCertValidation,
Agent.TempDirectory,
Agent.ToolsDirectory,
Agent.UseGitLongPaths,
Agent.UseGitSingleThread,
Agent.UseLatestGitVersion,
Agent.Version,
Agent.WorkFolder,
Agent.WorkingDirectory,
Expand Down
2 changes: 1 addition & 1 deletion src/Misc/externals.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ NODE10_VERSION="10.24.1"
NODE16_VERSION="16.20.2"
NODE20_VERSION="20.9.0"
MINGIT_VERSION="2.39.1"
FF_MINGIT_VERSION="2.42.0.2"
FF_MINGIT_VERSION="2.43.0"
LFS_VERSION="3.3.0"

get_abs_path() {
Expand Down
Loading

0 comments on commit 0905479

Please sign in to comment.