diff --git a/Build-SFPkgs.ps1 b/Build-SFPkgs.ps1 index 4020e442..7290c7dc 100644 --- a/Build-SFPkgs.ps1 +++ b/Build-SFPkgs.ps1 @@ -23,11 +23,11 @@ function Build-SFPkg { try { Push-Location $scriptPath - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.SelfContained.3.2.12" "$scriptPath\bin\release\FabricObserver\linux-x64\self-contained\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.FrameworkDependent.3.2.12" "$scriptPath\bin\release\FabricObserver\linux-x64\framework-dependent\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.SelfContained.3.2.13" "$scriptPath\bin\release\FabricObserver\linux-x64\self-contained\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.FrameworkDependent.3.2.13" "$scriptPath\bin\release\FabricObserver\linux-x64\framework-dependent\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.12" "$scriptPath\bin\release\FabricObserver\win-x64\self-contained\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.FrameworkDependent.3.2.12" "$scriptPath\bin\release\FabricObserver\win-x64\framework-dependent\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.13" "$scriptPath\bin\release\FabricObserver\win-x64\self-contained\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.FrameworkDependent.3.2.13" "$scriptPath\bin\release\FabricObserver\win-x64\framework-dependent\FabricObserverType" } finally { Pop-Location diff --git a/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt b/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt index 9bfce406..d2747f7b 100644 --- a/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt +++ b/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt @@ -68,5 +68,5 @@ cd C:\Users\me\source\repos\service-fabric-observer ./Build-FabricObserver ./Build-NugetPackages -The output from the above commands, FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.12.nupkg, would be located in +The output from the above commands, FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.13.nupkg, would be located in C:\Users\me\source\repos\service-fabric-observer\bin\release\FabricObserver\Nugets. \ No newline at end of file diff --git a/Documentation/Deployment/service-fabric-observer.json b/Documentation/Deployment/service-fabric-observer.json index 85dbed8f..9b5b612d 100644 --- a/Documentation/Deployment/service-fabric-observer.json +++ b/Documentation/Deployment/service-fabric-observer.json @@ -11,16 +11,16 @@ }, "applicationTypeVersionFabricObserver": { "type": "string", - "defaultValue": "3.2.12", + "defaultValue": "3.2.13", "metadata": { - "description": "Provide the app version number of FabricObserver. This must be identical to the version, 3.2.12, in the referenced sfpkg specified in packageUrlFabricObserver." + "description": "Provide the app version number of FabricObserver. This must be identical to the version, 3.2.13, in the referenced sfpkg specified in packageUrlFabricObserver." } }, "packageUrlFabricObserver": { "type": "string", "defaultValue": "", "metadata": { - "description": "This has to be a public accessible URL for the sfpkg file which contains the FabricObserver app package. Example: https://github.com/microsoft/service-fabric-observer/releases/download/[xxxxxxxx]/Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.12.sfpkg" + "description": "This has to be a public accessible URL for the sfpkg file which contains the FabricObserver app package. Example: https://github.com/microsoft/service-fabric-observer/releases/download/[xxxxxxxx]/Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.13.sfpkg" } } }, diff --git a/Documentation/Deployment/service-fabric-observer.v3.2.12.parameters.json b/Documentation/Deployment/service-fabric-observer.v3.2.13.parameters.json similarity index 90% rename from Documentation/Deployment/service-fabric-observer.v3.2.12.parameters.json rename to Documentation/Deployment/service-fabric-observer.v3.2.13.parameters.json index 8a27a87c..6700cd6e 100644 --- a/Documentation/Deployment/service-fabric-observer.v3.2.12.parameters.json +++ b/Documentation/Deployment/service-fabric-observer.v3.2.13.parameters.json @@ -1,15 +1,15 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterName": { - "value": "" - }, - "applicationTypeVersionFabricObserver": { - "value": "3.2.12" - }, - "packageUrlFabricObserver": { - "value": "" - } - } +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "value": "" + }, + "applicationTypeVersionFabricObserver": { + "value": "3.2.13" + }, + "packageUrlFabricObserver": { + "value": "" + } + } } \ No newline at end of file diff --git a/Documentation/OperationalTelemetry.md b/Documentation/OperationalTelemetry.md index 2659f337..6164b898 100644 --- a/Documentation/OperationalTelemetry.md +++ b/Documentation/OperationalTelemetry.md @@ -18,7 +18,7 @@ As with most of FabricObserver's application settings, you can also do this with Connect-ServiceFabricCluster ... $appParams = @{ "ObserverManagerEnableOperationalFOTelemetry" = "false"; } -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationParameter $appParams -ApplicationTypeVersion 3.2.12 -UnMonitoredAuto +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationParameter $appParams -ApplicationTypeVersion 3.2.13 -UnMonitoredAuto ``` @@ -44,7 +44,7 @@ Here is a full example of exactly what is sent in one of these telemetry events, "ClusterId": "00000000-1111-1111-0000-00f00d000d", "ClusterType": "SFRP", "NodeNameHash": "3e83569d4c6aad78083cd081215dafc81e5218556b6a46cb8dd2b183ed0095ad", - "FOVersion": "3.2.12", + "FOVersion": "3.2.13", "HasPlugins": "False", "SFRuntimeVersion":"9.0.1028.9590" "UpTime": "1.00:30:18.8058379", diff --git a/Documentation/Plugins.md b/Documentation/Plugins.md index b3c4577d..1eb8111f 100644 --- a/Documentation/Plugins.md +++ b/Documentation/Plugins.md @@ -72,5 +72,5 @@ cd C:\Users\me\source\repos\service-fabric-observer ./Build-FabricObserver ./Build-NugetPackages ``` -The output from the above commands contains FabricObserver platform-specific nupkgs and a nupkg you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.12.nupkg. Nuget packages will be located in +The output from the above commands contains FabricObserver platform-specific nupkgs and a nupkg you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.13.nupkg. Nuget packages will be located in C:\Users\me\source\repos\service-fabric-observer\bin\release\FabricObserver\Nugets. \ No newline at end of file diff --git a/Documentation/Using.md b/Documentation/Using.md index 4c4d1f77..5851f8a4 100644 --- a/Documentation/Using.md +++ b/Documentation/Using.md @@ -710,7 +710,7 @@ $appParams = @{ "FabricSystemObserverEnabled" = "true"; "FabricSystemObserverMem Then execute the application upgrade with ```Powershell -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.12 -ApplicationParameter $appParams -Monitored -FailureAction rollback +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.13 -ApplicationParameter $appParams -Monitored -FailureAction rollback ``` **Important**: This action will overwrite previous app paramemter changes that were made in an earlier application upgrade, for example. If you want to preserve any earlier changes, then you will need to @@ -718,7 +718,7 @@ supply those parameter values again along with the new ones. You do this in the ```PowerShell $appName = "fabric:/FabricObserver" -$appVersion = "3.2.12" +$appVersion = "3.2.13" $application = Get-ServiceFabricApplication -ApplicationName $appName $appParamCollection = $application.ApplicationParameters $applicationParameterMap = @{} diff --git a/FabricObserver.Extensibility.nuspec.template b/FabricObserver.Extensibility.nuspec.template index c829ffe8..4b2aa67c 100644 --- a/FabricObserver.Extensibility.nuspec.template +++ b/FabricObserver.Extensibility.nuspec.template @@ -2,11 +2,9 @@ %PACKAGE_ID% - 3.2.12 + 3.2.13 -Note: This is library is required for observer plugins that target FabricObserver 3.2.12. -- Updated LVID count monitor to support ESE database performance category change in SF 10.x versions. -- Adding a warning health event to inform customer if there is an issue with loading plugins. +Note: This is library is required for observer plugins that target FabricObserver 3.2.13. Microsoft MIT diff --git a/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj b/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj index e74877f5..5f82fdbb 100644 --- a/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj +++ b/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj @@ -5,8 +5,8 @@ FabricObserver Copyright © 2023 FabricObserver - 3.2.12 - 3.2.12 + 3.2.13 + 3.2.13 CA1416 diff --git a/FabricObserver.Extensibility/Utilities/ObserverConstants.cs b/FabricObserver.Extensibility/Utilities/ObserverConstants.cs index 971eb7f3..8553dc10 100644 --- a/FabricObserver.Extensibility/Utilities/ObserverConstants.cs +++ b/FabricObserver.Extensibility/Utilities/ObserverConstants.cs @@ -193,5 +193,33 @@ public sealed class ObserverConstants public const string DefaultValue = "DefaultValue"; public const string Parameter = "Parameter"; public const string Parameters = "Parameters"; + + // AppManifest Thresholds + public const string AppManifestMemoryWarningLimitMb = "FabricObserver_MemoryWarningLimitMb"; + public const string AppManifestMemoryErrorLimitMb = "FabricObserver_MemoryErrorLimitMb"; + public const string AppManifestMemoryWarningLimitPercent = "FabricObserver_MemoryWarningLimitPercent"; + public const string AppManifestMemoryErrorLimitPercent = "FabricObserver_MemoryErrorLimitPercent"; + public const string AppManifestCpuErrorLimitPercent = "FabricObserver_CpuErrorLimitPercent"; + public const string AppManifestCpuWarningLimitPercent = "FabricObserver_CpuWarningLimitPercent"; + public const string AppManifestNetworkErrorActivePorts = "FabricObserver_NetworkErrorActivePorts"; + public const string AppManifestNetworkWarningActivePorts = "FabricObserver_NetworkWarningActivePorts"; + public const string AppManifestNetworkErrorEphemeralPorts = "FabricObserver_NetworkErrorEphemeralPorts"; + public const string AppManifestNetworkWarningEphemeralPorts = "FabricObserver_NetworkWarningEphemeralPorts"; + public const string AppManifestNetworkErrorEphemeralPortsPercent = "FabricObserver_NetworkErrorEphemeralPortsPercent"; + public const string AppManifestNetworkWarningEphemeralPortsPercent = "FabricObserver_NetworkWarningEphemeralPortsPercent"; + public const string AppManifestDumpProcessOnError = "FabricObserver_DumpProcessOnError"; + public const string AppManifestDumpProcessOnWarning = "FabricObserver_DumpProcessOnWarning"; + public const string AppManifestErrorOpenFileHandles = "FabricObserver_ErrorOpenFileHandles"; + public const string AppManifestWarningOpenFileHandles = "FabricObserver_WarningOpenFileHandles"; + public const string AppManifestErrorHandleCount = "FabricObserver_ErrorHandleCount"; + public const string AppManifestWarningHandleCount = "FabricObserver_WarningHandleCount"; + public const string AppManifestErrorThreadCount = "FabricObserver_ErrorThreadCount"; + public const string AppManifestWarningThreadCount = "FabricObserver_WarningThreadCount"; + public const string AppManifestWarningPrivateBytesMb = "FabricObserver_WarningPrivateBytesMb"; + public const string AppManifestErrorPrivateBytesMb = "FabricObserver_ErrorPrivateBytesMb"; + public const string AppManifestWarningPrivateBytesPercent = "FabricObserver_WarningPrivateBytesPercent"; + public const string AppManifestErrorPrivateBytesPercent = "FabricObserver_ErrorPrivateBytesPercent"; + public const string AppManifestWarningRGMemoryLimitPercent = "FabricObserver_WarningRGMemoryLimitPercent"; + public const string AppManifestWarningRGCpuLimitPercent = "FabricObserver_WarningRGCpuLimitPercent"; } } diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index d6697888..918cdc7f 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -2,10 +2,9 @@ %PACKAGE_ID% - 3.2.12 + 3.2.13 -- ESE LVID count monitor updated to support ESE database performance category change in SF 10.x versions. -- Adding a warning health event to inform customer if there is an issue with loading plugins. +- Added feature for applications to set AppObserver thresholds in Application Manifest. This release only allows for specifying RG memory threshold settings for AppObserver in FO's ApplicationManifest. Support for other AppObserver Application Parameter settings will come in future releases. Microsoft MIT @@ -14,7 +13,7 @@ icon.png fonuget.md en-US - This package contains the FabricObserver(FO) Application - built for .NET 6.0 and SF Runtime 9.x. FO a highly configurable and extensible resource usage watchdog service that is designed to be run in Azure Service Fabric Windows and Linux clusters. This package contains the entire application and can be used to build .NET Standard 2.0 observer plugins. NOTE: If you want to target .NET 6 for your plugins, then you must use Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.12 nuget package to build them. + This package contains the FabricObserver(FO) Application - built for .NET 6.0 and SF Runtime 9.x. FO a highly configurable and extensible resource usage watchdog service that is designed to be run in Azure Service Fabric Windows and Linux clusters. This package contains the entire application and can be used to build .NET Standard 2.0 observer plugins. NOTE: If you want to target .NET 6 for your plugins, then you must use Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.13 nuget package to build them. diff --git a/FabricObserver.sln b/FabricObserver.sln index 447d6a31..491785e5 100644 --- a/FabricObserver.sln +++ b/FabricObserver.sln @@ -35,9 +35,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md SECURITY.md = SECURITY.md Documentation\Deployment\service-fabric-cluster-observer.json = Documentation\Deployment\service-fabric-cluster-observer.json - Documentation\Deployment\service-fabric-observer.json = Documentation\Deployment\service-fabric-observer.json Documentation\Deployment\service-fabric-cluster-observer.v2.2.7.parameters.json = Documentation\Deployment\service-fabric-cluster-observer.v2.2.7.parameters.json - Documentation\Deployment\service-fabric-observer.v3.2.12.parameters.json = Documentation\Deployment\service-fabric-observer.v3.2.12.parameters.json + Documentation\Deployment\service-fabric-observer.json = Documentation\Deployment\service-fabric-observer.json + Documentation\Deployment\service-fabric-observer.v3.2.13.parameters.json = Documentation\Deployment\service-fabric-observer.v3.2.13.parameters.json Documentation\Using.md = Documentation\Using.md EndProjectSection EndProject diff --git a/FabricObserver/FabricObserver.csproj b/FabricObserver/FabricObserver.csproj index 94aed618..72842c2e 100644 --- a/FabricObserver/FabricObserver.csproj +++ b/FabricObserver/FabricObserver.csproj @@ -11,8 +11,8 @@ True Copyright © 2022 FabricObserver - 3.2.12 - 3.2.12 + 3.2.13 + 3.2.13 true true FabricObserver.Program diff --git a/FabricObserver/Observers/AppObserver.cs b/FabricObserver/Observers/AppObserver.cs index 1939e9be..f15e0267 100644 --- a/FabricObserver/Observers/AppObserver.cs +++ b/FabricObserver/Observers/AppObserver.cs @@ -1028,6 +1028,10 @@ public async Task InitializeAsync() // Support for specifying single configuration JSON object for all applications. await ProcessGlobalThresholdSettingsAsync(); + + // Support for specifying thresholds in application's manifest + await ProcessAppManifestThresholdSettingsAsync(); + SetConcurrentMonitoringState(); int settingsFail = 0; @@ -1213,6 +1217,339 @@ private void SetConcurrentMonitoringState() }; } + private bool PopulateAppInfoWithAppManifestThresholds(DeployedApplication deployedApp, ApplicationInfo appInfo) + { + string appTypeVersion = null; + ApplicationParameterList appParameters = null; + ApplicationParameterList defaultParameters = null; + + ApplicationList appList = + FabricClientInstance.QueryManager.GetApplicationListAsync( + deployedApp.ApplicationName, + ConfigurationSettings.AsyncTimeout, + Token).Result; + + ApplicationTypeList applicationTypeList = + FabricClientInstance.QueryManager.GetApplicationTypeListAsync( + deployedApp.ApplicationTypeName, + ConfigurationSettings.AsyncTimeout, + Token).Result; + + if (appList?.Count > 0) + { + try + { + if (appList.Any(app => app.ApplicationTypeName == deployedApp.ApplicationTypeName)) + { + appTypeVersion = appList.First(app => app.ApplicationTypeName == deployedApp.ApplicationTypeName).ApplicationTypeVersion; + appParameters = appList.First(app => app.ApplicationTypeName == deployedApp.ApplicationTypeName).ApplicationParameters; + } + + if (applicationTypeList.Any(app => app.ApplicationTypeVersion == appTypeVersion)) + { + defaultParameters = applicationTypeList.First(app => app.ApplicationTypeVersion == appTypeVersion).DefaultParameters; + } + } + catch (Exception e) when (e is ArgumentException or InvalidOperationException) + { + + } + } + + ApplicationParameterList parameters = new(); + FabricClientUtilities.AddParametersIfNotExists(parameters, appParameters); + FabricClientUtilities.AddParametersIfNotExists(parameters, defaultParameters); + + bool informationAdded = false; + + // Working Set + if (parameters.Contains(ObserverConstants.AppManifestMemoryWarningLimitMb)) + { + if (long.TryParse(parameters[ObserverConstants.AppManifestMemoryWarningLimitMb].Value, out long memoryMbWarningThreshold) && memoryMbWarningThreshold > 0) + { + appInfo.MemoryWarningLimitMb = memoryMbWarningThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestMemoryErrorLimitMb)) + { + if (long.TryParse(parameters[ObserverConstants.AppManifestMemoryErrorLimitMb].Value, out long memoryMbErrorThreshold) && memoryMbErrorThreshold > 0) + { + appInfo.MemoryErrorLimitMb = memoryMbErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestMemoryWarningLimitPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestMemoryWarningLimitPercent].Value, out double memoryPctWarningThreshold) && memoryPctWarningThreshold > 0) + { + appInfo.MemoryWarningLimitPercent = memoryPctWarningThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestMemoryErrorLimitPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestMemoryErrorLimitPercent].Value, out double memoryPctErrorThreshold) && memoryPctErrorThreshold > 0) + { + appInfo.MemoryErrorLimitPercent = memoryPctErrorThreshold; + informationAdded = true; + } + } + // CPU + if (parameters.Contains(ObserverConstants.AppManifestCpuErrorLimitPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestCpuErrorLimitPercent].Value, out double cpuPctErrorThreshold) && cpuPctErrorThreshold > 0) + { + appInfo.CpuErrorLimitPercent = cpuPctErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestCpuWarningLimitPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestCpuWarningLimitPercent].Value, out double cpuPctWarningThreshold) && cpuPctWarningThreshold > 0) + { + appInfo.CpuWarningLimitPercent = cpuPctWarningThreshold; + informationAdded = true; + } + } + // Active Ports + if (parameters.Contains(ObserverConstants.AppManifestNetworkErrorActivePorts)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestNetworkErrorActivePorts].Value, out int activePortsErrorThreshold) && activePortsErrorThreshold > 0) + { + appInfo.NetworkErrorActivePorts = activePortsErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestNetworkWarningActivePorts)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestNetworkWarningActivePorts].Value, out int activePortsWarningThreshold) && activePortsWarningThreshold > 0) + { + appInfo.NetworkWarningActivePorts = activePortsWarningThreshold; + informationAdded = true; + } + } + // Ephemeral Ports + if (parameters.Contains(ObserverConstants.AppManifestNetworkErrorEphemeralPorts)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestNetworkErrorEphemeralPorts].Value, out int ephemeralPortsErrorThreshold) && ephemeralPortsErrorThreshold > 0) + { + appInfo.NetworkErrorEphemeralPorts = ephemeralPortsErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestNetworkWarningEphemeralPorts)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestNetworkWarningEphemeralPorts].Value, out int ephemeralPortsWarningThreshold) && ephemeralPortsWarningThreshold > 0) + { + appInfo.NetworkWarningEphemeralPorts = ephemeralPortsWarningThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestNetworkErrorEphemeralPortsPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestNetworkErrorEphemeralPortsPercent].Value, out double ephemeralPortsPctErrorThreshold) && ephemeralPortsPctErrorThreshold > 0) + { + appInfo.NetworkErrorEphemeralPortsPercent = ephemeralPortsPctErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestNetworkWarningEphemeralPortsPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestNetworkWarningEphemeralPortsPercent].Value, out double ephemeralPortsPctWarningThreshold) && ephemeralPortsPctWarningThreshold > 0) + { + appInfo.NetworkWarningEphemeralPortsPercent = ephemeralPortsPctWarningThreshold; + informationAdded = true; + } + } + // Dump Process + if (parameters.Contains(ObserverConstants.AppManifestDumpProcessOnError)) + { + if (bool.TryParse(parameters[ObserverConstants.AppManifestDumpProcessOnError].Value, out bool dumpProcessOnError)) + { + appInfo.DumpProcessOnError = dumpProcessOnError; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestDumpProcessOnWarning)) + { + if (bool.TryParse(parameters[ObserverConstants.AppManifestDumpProcessOnWarning].Value, out bool dumpProcessOnWarning)) + { + appInfo.DumpProcessOnWarning = dumpProcessOnWarning; + informationAdded = true; + } + } + // Handles + if (parameters.Contains(ObserverConstants.AppManifestErrorOpenFileHandles)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestErrorOpenFileHandles].Value, out int openFileHandlesErrorThreshold) && openFileHandlesErrorThreshold > 0) + { + appInfo.ErrorOpenFileHandles = openFileHandlesErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestWarningOpenFileHandles)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestWarningOpenFileHandles].Value, out int openFileHandlesWarningThreshold) && openFileHandlesWarningThreshold > 0) + { + appInfo.WarningOpenFileHandles = openFileHandlesWarningThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestErrorHandleCount)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestErrorHandleCount].Value, out int handleCountErrorThreshold) && handleCountErrorThreshold > 0) + { + appInfo.ErrorHandleCount = handleCountErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestWarningHandleCount)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestWarningHandleCount].Value, out int handleCountWarningThreshold) && handleCountWarningThreshold > 0) + { + appInfo.WarningHandleCount = handleCountWarningThreshold; + informationAdded = true; + } + } + // Threads + if (parameters.Contains(ObserverConstants.AppManifestErrorThreadCount)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestErrorThreadCount].Value, out int threadCountErrorThreshold) && threadCountErrorThreshold > 0) + { + appInfo.ErrorThreadCount = threadCountErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestWarningThreadCount)) + { + if (int.TryParse(parameters[ObserverConstants.AppManifestWarningThreadCount].Value, out int threadCountWarningThreshold) && threadCountWarningThreshold > 0) + { + appInfo.WarningThreadCount = threadCountWarningThreshold; + informationAdded = true; + } + } + // Private Bytes + if (parameters.Contains(ObserverConstants.AppManifestWarningPrivateBytesMb)) + { + if (long.TryParse(parameters[ObserverConstants.AppManifestWarningPrivateBytesMb].Value, out long privateBytesMbWarningThreshold) && privateBytesMbWarningThreshold > 0) + { + appInfo.WarningPrivateBytesMb = privateBytesMbWarningThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestErrorPrivateBytesMb)) + { + if (long.TryParse(parameters[ObserverConstants.AppManifestErrorPrivateBytesMb].Value, out long privateBytesMbErrorThreshold) && privateBytesMbErrorThreshold > 0) + { + appInfo.ErrorPrivateBytesMb = privateBytesMbErrorThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestWarningPrivateBytesPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestWarningPrivateBytesPercent].Value, out double privateBytesPctWarningThreshold) && privateBytesPctWarningThreshold > 0) + { + appInfo.WarningPrivateBytesPercent = privateBytesPctWarningThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestErrorPrivateBytesPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestErrorPrivateBytesPercent].Value, out double privateBytesPctErrorThreshold) && privateBytesPctErrorThreshold > 0) + { + appInfo.ErrorPrivateBytesPercent = privateBytesPctErrorThreshold; + informationAdded = true; + } + } + // RG monitoring + if (parameters.Contains(ObserverConstants.AppManifestWarningRGMemoryLimitPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestWarningRGMemoryLimitPercent].Value, out double rgMemoryThreshold) && rgMemoryThreshold > 0) + { + appInfo.WarningRGMemoryLimitPercent = rgMemoryThreshold; + informationAdded = true; + } + } + if (parameters.Contains(ObserverConstants.AppManifestWarningRGCpuLimitPercent)) + { + if (double.TryParse(parameters[ObserverConstants.AppManifestWarningRGCpuLimitPercent].Value, out double rgCpuThreshold) && rgCpuThreshold > 0) + { + appInfo.WarningRGCpuLimitPercent = rgCpuThreshold; + informationAdded = true; + } + } + return informationAdded; + } + + private async Task ProcessAppManifestThresholdSettingsAsync() + { + for (int i = 0; i < deployedApps.Count; i++) + { + Token.ThrowIfCancellationRequested(); + + var app = deployedApps[i]; + + try + { + // Make sure deployed app is not a containerized app. + var codepackages = await FabricClientInstance.QueryManager.GetDeployedCodePackageListAsync(NodeName, app.ApplicationName, null, null, ConfigurationSettings.AsyncTimeout, Token); + + if (codepackages.Count == 0) + { + continue; + } + + int containerHostCount = codepackages.Count(c => c.HostType == HostType.ContainerHost); + + // Ignore containerized apps. ContainerObserver is designed for those types of services. + if (containerHostCount > 0) + { + continue; + } + + // AppObserver does not monitor SF system services. + if (app.ApplicationName.OriginalString == "fabric:/System") + { + continue; + } + } + catch (FabricException fe) + { + ObserverLogger.LogWarning($"Handled FabricException from GetDeployedCodePackageListAsync call for app {app.ApplicationName.OriginalString}: {fe.Message}."); + continue; + } + // Don't create a brand new entry for an existing (specified in configuration) app target/type. Just update the appConfig instance with data supplied in the application's manifest. + // If a threshold is supplied in the application's manifest, it will override all other settings. + if (userTargetList.Any(a => a.TargetApp == app.ApplicationName.OriginalString || a.TargetAppType == app.ApplicationTypeName)) + { + var existingAppConfig = userTargetList.FindAll(a => a.TargetApp == app.ApplicationName.OriginalString || a.TargetAppType == app.ApplicationTypeName); + + if (existingAppConfig == null || existingAppConfig.Count == 0) + { + continue; + } + + for (int j = 0; j < existingAppConfig.Count; j++) + { + PopulateAppInfoWithAppManifestThresholds(app, existingAppConfig[j]); + } + } + else + { + var appConfig = new ApplicationInfo + { + TargetApp = app.ApplicationName.OriginalString + }; + + if (PopulateAppInfoWithAppManifestThresholds(app, appConfig)) + { + userTargetList.Add(appConfig); + } + } + } + } + private async Task ProcessGlobalThresholdSettingsAsync() { if (userTargetList == null || userTargetList.Count == 0) diff --git a/FabricObserver/Observers/ObserverManager.cs b/FabricObserver/Observers/ObserverManager.cs index 51fbcd08..56a8019e 100644 --- a/FabricObserver/Observers/ObserverManager.cs +++ b/FabricObserver/Observers/ObserverManager.cs @@ -54,7 +54,7 @@ private List Observers private CancellationTokenSource linkedSFRuntimeObserverTokenSource; // Folks often use their own version numbers. This is for internal diagnostic telemetry. - private const string InternalVersionNumber = "3.2.12"; + private const string InternalVersionNumber = "3.2.13"; private static FabricClient FabricClientInstance => FabricClientUtilities.FabricClientSingleton; diff --git a/FabricObserver/PackageRoot/Data/Plugins/Readme.txt b/FabricObserver/PackageRoot/Data/Plugins/Readme.txt index ea5ce63c..fad8a443 100644 --- a/FabricObserver/PackageRoot/Data/Plugins/Readme.txt +++ b/FabricObserver/PackageRoot/Data/Plugins/Readme.txt @@ -7,8 +7,8 @@ Note that the observer API lives in its own library, FabricObserver.Extensibilit 1. Create a new .NET 6 Library project. 2. Install the same version of the Microsoft.ServiceFabricApps.FabricObserver.Extensibility nupkg from https://www.nuget.org/profiles/ServiceFabricApps as the version of FabricObserver you are deploying. - E.g., 3.2.12 if you are going to deploy FO 3.2.12. - NOTE: You can also consume the entire FabricObserver 3.2.12 nupkg to build your plugin. Please see the SampleObserverPlugin project's csproj file for more information. + E.g., 3.2.13 if you are going to deploy FO 3.2.13. + NOTE: You can also consume the entire FabricObserver 3.2.13 nupkg to build your plugin. Please see the SampleObserverPlugin project's csproj file for more information. 3. Write an observer! @@ -68,5 +68,5 @@ cd C:\Users\me\source\repos\service-fabric-observer ./Build-FabricObserver ./Build-NugetPackages -The output from the above commands contains FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.12.nupkg. Nupkg files from above command would be located in +The output from the above commands contains FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.13.nupkg. Nupkg files from above command would be located in C:\Users\me\source\repos\service-fabric-observer\bin\release\FabricObserver\Nugets. \ No newline at end of file diff --git a/FabricObserver/PackageRoot/ServiceManifest.xml b/FabricObserver/PackageRoot/ServiceManifest.xml index 09e87d61..e7646843 100644 --- a/FabricObserver/PackageRoot/ServiceManifest.xml +++ b/FabricObserver/PackageRoot/ServiceManifest.xml @@ -1,6 +1,6 @@  @@ -9,7 +9,7 @@ This name must match the string used in RegisterServiceType call in Program.cs. --> - + install_lvid_perfcounter.bat @@ -25,10 +25,10 @@ - + - + \ No newline at end of file diff --git a/FabricObserver/PackageRoot/ServiceManifest_linux.xml b/FabricObserver/PackageRoot/ServiceManifest_linux.xml index 65b54348..26226e2a 100644 --- a/FabricObserver/PackageRoot/ServiceManifest_linux.xml +++ b/FabricObserver/PackageRoot/ServiceManifest_linux.xml @@ -1,6 +1,6 @@  @@ -11,7 +11,7 @@ - + setcaps.sh @@ -27,10 +27,10 @@ - + - + \ No newline at end of file diff --git a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml index 96048f7f..e42477a5 100644 --- a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml +++ b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml @@ -1,6 +1,6 @@  - + @@ -236,7 +236,7 @@ should match the Name and Version attributes of the ServiceManifest element defined in the ServiceManifest.xml file. --> - + diff --git a/FabricObserverTests/ObserverTests.cs b/FabricObserverTests/ObserverTests.cs index d751b829..9ab5f647 100644 --- a/FabricObserverTests/ObserverTests.cs +++ b/FabricObserverTests/ObserverTests.cs @@ -1247,8 +1247,77 @@ public async Task AppObserver_ObserveAsync_Successful_RGMemoryLimitWarningGenera JsonConfigPath = Path.Combine(Environment.CurrentDirectory, "PackageRoot", "Config", "AppObserver_rg_memory_warning.config.json"), }; + // Upgrade RG memory warning threshold App parameters. + ApplicationUpgradeDescription appUpgradeDescription = new() + { + ApplicationName = new Uri("fabric:/Voting"), + TargetApplicationTypeVersion = "1.0.0", + UpgradePolicyDescription = new RollingUpgradePolicyDescription() { UpgradeMode = RollingUpgradeMode.UnmonitoredAuto } + }; + + appUpgradeDescription.ApplicationParameters.Add(ObserverConstants.AppManifestWarningRGMemoryLimitPercent, "1"); + + try + { + await FabricClientSingleton.ApplicationManager.UpgradeApplicationAsync( + appUpgradeDescription, + TimeSpan.FromSeconds(60), + Token); + } + catch (FabricException) + { + // This can happen if the parameter value is the same/already in place. Ignore it. + // If it is for some other reason, then this test function will naturally fail. + } + + Stopwatch stopwatch = Stopwatch.StartNew(); + + // Wait for the upgrade to complete. + while (stopwatch.Elapsed <= TimeSpan.FromSeconds(30)) + { + var progress = await FabricClientSingleton.ApplicationManager.GetApplicationUpgradeProgressAsync(new Uri("fabric:/Voting")); + + if (progress.UpgradeState == ApplicationUpgradeState.RollingForwardCompleted) + { + break; + } + + await Task.Delay(TimeSpan.FromSeconds(3)); + } + await obs.ObserveAsync(Token); + // Revert previous upgrade. + appUpgradeDescription.ApplicationParameters[ObserverConstants.AppManifestWarningRGMemoryLimitPercent] = "0"; + + try + { + await FabricClientSingleton.ApplicationManager.UpgradeApplicationAsync( + appUpgradeDescription, + TimeSpan.FromSeconds(60), + Token); + } + catch (FabricException) + { + // This can happen if the parameter value is the same/already in place. Ignore it. + // If it is for some other reason, then this test function will naturally fail. + } + + stopwatch = Stopwatch.StartNew(); + + // Wait for the upgrade to complete. + while (stopwatch.Elapsed <= TimeSpan.FromSeconds(30)) + { + var progress = await FabricClientSingleton.ApplicationManager.GetApplicationUpgradeProgressAsync(new Uri("fabric:/Voting")); + + if (progress.UpgradeState == ApplicationUpgradeState.RollingForwardCompleted) + { + break; + } + + await Task.Delay(TimeSpan.FromSeconds(3)); + } + // observer ran to completion with no errors. Assert.IsTrue(obs.LastRunDateTime > startDateTime); diff --git a/FabricObserverTests/PackageRoot/Config/AppObserver_rg_memory_warning.config.json b/FabricObserverTests/PackageRoot/Config/AppObserver_rg_memory_warning.config.json index 12507ae7..c7f687ac 100644 --- a/FabricObserverTests/PackageRoot/Config/AppObserver_rg_memory_warning.config.json +++ b/FabricObserverTests/PackageRoot/Config/AppObserver_rg_memory_warning.config.json @@ -1,6 +1,6 @@ [ { "targetApp": "Voting", - "warningRGMemoryLimitPercent": 1 + "warningRGMemoryLimitPercent": 90 } ] \ No newline at end of file diff --git a/FabricObserverTests/VotingApp.zip b/FabricObserverTests/VotingApp.zip index 7508bdff..2f0ea101 100644 Binary files a/FabricObserverTests/VotingApp.zip and b/FabricObserverTests/VotingApp.zip differ diff --git a/README.md b/README.md index 09a481d5..64709215 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## FabricObserver 3.2.12 +## FabricObserver 3.2.13 [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fmicrosoft%2Fservice-fabric-observer%2Fmain%2FDocumentation%2FDeployment%2Fservice-fabric-observer.json) @@ -87,7 +87,7 @@ see [FOAzurePipeline.yaml](/FOAzurePipeline.yaml) for msazure devops build tasks .net6 installed (if you deploy VM images from Azure gallery, then they will not have .net6 installed), then you must deploy the SelfContained package. ### Deploy FabricObserver -**Note: You must deploy this version (3.2.12) to clusters that are running SF 9.0 and above. This version also requires .NET 6.** +**Note: You must deploy this version (3.2.13) to clusters that are running SF 9.0 and above. This version also requires .NET 6.** You can deploy FabricObserver (and ClusterObserver) using Visual Studio (if you build the sources yourself), PowerShell or ARM. Please note that this version of FabricObserver no longer supports the DefaultServices node in ApplicationManifest.xml. This means that should you deploy using PowerShell, you must create an instance of the service as the last command in your script. This was done to support ARM deployment, specifically. The StartupServices.xml file you see in the FabricHealerApp project now contains the service information once held in ApplicationManifest's DefaultServices node. Note that this information is primarily useful for deploying from Visual Studio. @@ -127,15 +127,15 @@ Connect-ServiceFabricCluster -ConnectionEndpoint @('sf-win-cluster.westus2.cloud #Copy $path contents (FO app package) to server: -Copy-ServiceFabricApplicationPackage -ApplicationPackagePath $path -CompressPackage -ApplicationPackagePathInImageStore FO3211 -TimeoutSec 1800 +Copy-ServiceFabricApplicationPackage -ApplicationPackagePath $path -CompressPackage -ApplicationPackagePathInImageStore FO3213 -TimeoutSec 1800 #Register FO ApplicationType: -Register-ServiceFabricApplicationType -ApplicationPathInImageStore FO3211 +Register-ServiceFabricApplicationType -ApplicationPathInImageStore FO3213 #Create FO application (if not already deployed at lesser version): -New-ServiceFabricApplication -ApplicationName fabric:/FabricObserver -ApplicationTypeName FabricObserverType -ApplicationTypeVersion 3.2.12 +New-ServiceFabricApplication -ApplicationName fabric:/FabricObserver -ApplicationTypeName FabricObserverType -ApplicationTypeVersion 3.2.13 #Create the Service instances (-1 means all nodes, which is what is required for FO): @@ -143,7 +143,7 @@ New-ServiceFabricService -Stateless -PartitionSchemeSingleton -ApplicationName f #OR if updating existing version: -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.12 -Monitored -FailureAction rollback +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.13 -Monitored -FailureAction rollback ``` ## Observer Model diff --git a/SampleObserverPlugin/SampleObserverPlugin.csproj b/SampleObserverPlugin/SampleObserverPlugin.csproj index 9b87c8af..076ead5c 100644 --- a/SampleObserverPlugin/SampleObserverPlugin.csproj +++ b/SampleObserverPlugin/SampleObserverPlugin.csproj @@ -19,7 +19,7 @@ -