Win7/PowerShell7
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Added functionality to retry in Invoke-RestMethod and Invoke-WebRequest. (#5760)

Browse Source 
Added two parameters, RetryCount and RetryIntervalSec to enable retry functionality. When retrying a verbose message is sent out to inform the user.
This commit is contained in:
Aditya Patwardhan 2018-06-28 21:06:53 -07:00 committed by Ilya
parent 68ab1e09a6
commit 15f6abe944
10 changed files with 307 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.spelling
View File

@ -1061,10 +1061,13 @@ v6.0.
#region test/tools/WebListener/README.md Overrides
- test/tools/WebListener/README.md
Auth
failureCode
failureCount
NoResume
NTLM
NumberBytes
ResponseHeaders
sessionId
#endregion
includeide
interactivetesting

190
src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs
View File

@ -21,6 +21,7 @@ using System.Xml;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Linq;
using System.Threading.Tasks;
namespace Microsoft.PowerShell.Commands
{
@ -228,6 +229,20 @@ namespace Microsoft.PowerShell.Commands
}
private int _maximumRedirection = -1;
/// <summary>
/// Gets or sets the MaximumRetryCount property, which determines the number of retries of a failed web request.
/// </summary>
[Parameter]
[ValidateRange(0, Int32.MaxValue)]
public virtual int MaximumRetryCount { get; set; } = 0;
/// <summary>
/// Gets or sets the RetryIntervalSec property, which determines the number seconds between retries.
/// </summary>
[Parameter]
[ValidateRange(1, Int32.MaxValue)]
public virtual int RetryIntervalSec { get; set; } = 5;
#endregion
#region Method
@ -630,6 +645,14 @@ namespace Microsoft.PowerShell.Commands
WebSession.Headers[key] = Headers[key].ToString();
}
}
if (MaximumRetryCount > 0)
{
WebSession.MaximumRetryCount = MaximumRetryCount;
// only set retry interval if retry count is set.
WebSession.RetryIntervalInSeconds = RetryIntervalSec;
}
}
#endregion Virtual Methods
@ -662,7 +685,6 @@ namespace Microsoft.PowerShell.Commands
#endregion Helper Properties
#region Helper Methods
private Uri PrepareUri(Uri uri)
{
uri = CheckProtocol(uri);
@ -1268,80 +1290,128 @@ namespace Microsoft.PowerShell.Commands
);
}
private bool ShouldRetry(HttpStatusCode code)
{
int intCode = (int)code;
if (((intCode == 304) || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0)
{
return true;
}
return false;
}
internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool keepAuthorization)
{
if (client == null) { throw new ArgumentNullException("client"); }
if (request == null) { throw new ArgumentNullException("request"); }
// Track the current URI being used by various requests and re-requests.
var currentUri = request.RequestUri;
// Add 1 to account for the first request.
int totalRequests = WebSession.MaximumRetryCount + 1;
HttpRequestMessage req = request;
HttpResponseMessage response = null;
_cancelToken = new CancellationTokenSource();
HttpResponseMessage response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
if (keepAuthorization && IsRedirectCode(response.StatusCode) && response.Headers.Location != null)
do
{
_cancelToken.Cancel();
_cancelToken = null;
// Track the current URI being used by various requests and re-requests.
var currentUri = req.RequestUri;
// if explicit count was provided, reduce it for this redirection.
if (WebSession.MaximumRedirection > 0)
_cancelToken = new CancellationTokenSource();
response = client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
if (keepAuthorization && IsRedirectCode(response.StatusCode) && response.Headers.Location != null)
{
WebSession.MaximumRedirection--;
}
// For selected redirects that used POST, GET must be used with the
// redirected Location.
// Since GET is the default; POST only occurs when -Method POST is used.
if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode))
{
// See https://msdn.microsoft.com/library/system.net.httpstatuscode(v=vs.110).aspx
Method = WebRequestMethod.Get;
_cancelToken.Cancel();
_cancelToken = null;
// if explicit count was provided, reduce it for this redirection.
if (WebSession.MaximumRedirection > 0)
{
WebSession.MaximumRedirection--;
}
// For selected redirects that used POST, GET must be used with the
// redirected Location.
// Since GET is the default; POST only occurs when -Method POST is used.
if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode))
{
// See https://msdn.microsoft.com/library/system.net.httpstatuscode(v=vs.110).aspx
Method = WebRequestMethod.Get;
}
currentUri = new Uri(request.RequestUri, response.Headers.Location);
// Continue to handle redirection
using (client = GetHttpClient(handleRedirect: true))
using (HttpRequestMessage redirectRequest = GetRequest(currentUri))
{
response = GetResponse(client, redirectRequest, keepAuthorization);
}
}
currentUri = new Uri(request.RequestUri, response.Headers.Location);
// Continue to handle redirection
using (client = GetHttpClient(handleRedirect: true))
using (HttpRequestMessage redirectRequest = GetRequest(currentUri))
// Request again without the Range header because the server indicated the range was not satisfiable.
// This happens when the local file is larger than the remote file.
// If the size of the remote file is the same as the local file, there is nothing to resume.
if (Resume.IsPresent &&
response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable &&
(response.Content.Headers.ContentRange.HasLength &&
response.Content.Headers.ContentRange.Length != _resumeFileSize))
{
response = GetResponse(client, redirectRequest, keepAuthorization);
_cancelToken.Cancel();
WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg);
// Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream()
// are treated as a standard -OutFile request. This also disables appending local file.
Resume = new SwitchParameter(false);
using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri))
{
FillRequestStream(requestWithoutRange);
long requestContentLength = 0;
if (requestWithoutRange.Content != null)
{
requestContentLength = requestWithoutRange.Content.Headers.ContentLength.Value;
}
string reqVerboseMsg = String.Format(
CultureInfo.CurrentCulture,
WebCmdletStrings.WebMethodInvocationVerboseMsg,
requestWithoutRange.Method,
requestWithoutRange.RequestUri,
requestContentLength);
WriteVerbose(reqVerboseMsg);
return GetResponse(client, requestWithoutRange, keepAuthorization);
}
}
_resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent;
// When MaximumRetryCount is not specified, the totalRequests == 1.
if (totalRequests > 1 && ShouldRetry(response.StatusCode))
{
string retryMessage = string.Format(
CultureInfo.CurrentCulture,
WebCmdletStrings.RetryVerboseMsg,
RetryIntervalSec,
response.StatusCode);
WriteVerbose(retryMessage);
_cancelToken = new CancellationTokenSource();
Task.Delay(WebSession.RetryIntervalInSeconds * 1000, _cancelToken.Token).GetAwaiter().GetResult();
_cancelToken.Cancel();
_cancelToken = null;
req.Dispose();
req = GetRequest(currentUri);
FillRequestStream(req);
}
totalRequests--;
}
while (totalRequests > 0 && !response.IsSuccessStatusCode);
// Request again without the Range header because the server indicated the range was not satisfiable.
// This happens when the local file is larger than the remote file.
// If the size of the remote file is the same as the local file, there is nothing to resume.
if (Resume.IsPresent &&
response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable &&
(response.Content.Headers.ContentRange.HasLength &&
response.Content.Headers.ContentRange.Length != _resumeFileSize))
{
_cancelToken.Cancel();
WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg);
// Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream()
// are treated as a standard -OutFile request. This also disables appending local file.
Resume = new SwitchParameter(false);
using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri))
{
FillRequestStream(requestWithoutRange);
long requestContentLength = 0;
if (requestWithoutRange.Content != null)
requestContentLength = requestWithoutRange.Content.Headers.ContentLength.Value;
string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.WebMethodInvocationVerboseMsg,
requestWithoutRange.Method,
requestWithoutRange.RequestUri,
requestContentLength);
WriteVerbose(reqVerboseMsg);
return GetResponse(client, requestWithoutRange, keepAuthorization);
}
}
_resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent;
return response;
}

10
src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/WebRequestSession.cs
View File

@ -64,6 +64,16 @@ namespace Microsoft.PowerShell.Commands
/// </summary>
public int MaximumRedirection { get; set; }
/// <summary>
/// Gets or sets the count of retries for request failures.
/// </summary>
public int MaximumRetryCount { get; set; }
/// <summary>
/// Gets or sets the interval in seconds between retries.
/// </summary>
public int RetryIntervalInSeconds { get; set; }
/// <summary>
/// Construct a new instance of a WebRequestSession object.
/// </summary>

3
src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx
View File

@ -258,4 +258,7 @@
<data name="WebResponseVerboseMsg" xml:space="preserve">
<value>received {0}-byte response of content type {1}</value>
</data>
<data name="RetryVerboseMsg" xml:space="preserve">
<value>Retrying after interval of {0} seconds. Status code for previous attempt: {1}</value>
</data>
</root>

71
test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1
View File

@ -1723,6 +1723,49 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
$response.Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize"
}
}
Context "Invoke-WebRequest retry tests" {
It "Invoke-WebRequest can retry - <Name>" -TestCases @(
@{Name = "specified number of times - error 304"; failureCount = 2; failureCode = 304; retryCount = 2}
@{Name = "specified number of times - error 400"; failureCount = 3; failureCode = 400; retryCount = 3}
@{Name = "specified number of times - error 599"; failureCount = 1; failureCode = 599; retryCount = 2}
@{Name = "specified number of times - error 404"; failureCount = 2; failureCode = 404; retryCount = 2}
@{Name = "when retry count is higher than failure count"; failureCount = 2; failureCode = 404; retryCount = 4}
) {
param($failureCount, $retryCount, $failureCode)
$uri = Get-WebListenerUrl -Test 'Retry' -Query @{ sessionid = (New-Guid).Guid; failureCode = $failureCode; failureCount = $failureCount }
$commandStr = "Invoke-WebRequest -Uri '$uri' -MaximumRetryCount $retryCount -RetryIntervalSec 1"
$result = ExecuteWebCommand -command $commandStr
$result.output.StatusCode | Should -Be "200"
$jsonResult = $result.output.Content | ConvertFrom-Json
$jsonResult.failureResponsesSent | Should -Be $failureCount
}
It "Invoke-WebRequest should fail when failureCount is greater than MaximumRetryCount" {
$uri = Get-WebListenerUrl -Test 'Retry' -Query @{ sessionid = (New-Guid).Guid; failureCode = 400; failureCount = 4 }
$command = "Invoke-WebRequest -Uri '$uri' -MaximumRetryCount 1 -RetryIntervalSec 1"
$result = ExecuteWebCommand -command $command
$jsonError = $result.error | ConvertFrom-Json
$jsonError.error | Should -BeExactly 'Error: HTTP - 400 occurred.'
}
It "Invoke-WebRequest can retry with POST" {
$uri = Get-WebListenerUrl -Test 'Retry'
$sessionId = (New-Guid).Guid
$body = @{ sessionid = $sessionId; failureCode = 404; failureCount = 1 }
$commandStr = "Invoke-WebRequest -Uri '$uri' -MaximumRetryCount 2 -RetryIntervalSec 1 -Method POST -Body `$body"
$result = ExecuteWebCommand -command $commandStr
$result.output.StatusCode | Should -Be "200"
$jsonResult = $result.output.Content | ConvertFrom-Json
$jsonResult.SessionId | Should -BeExactly $sessionId
}
}
}
Describe "Invoke-RestMethod tests" -Tags "Feature" {
@ -2969,6 +3012,33 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" {
$Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize"
}
}
Context "Invoke-RestMethod retry tests" {
It "Invoke-RestMethod can retry - specified number of times - error 304" {
$uri = Get-WebListenerUrl -Test 'Retry'
$sessionId = (New-Guid).Guid
$body = @{ sessionid = $sessionId; failureCode = 304; failureCount = 2 }
$commandStr = "Invoke-RestMethod -Uri '$uri' -MaximumRetryCount 2 -RetryIntervalSec 1 -Method POST -Body `$body"
$result = ExecuteWebCommand -command $commandStr
$result.output.failureResponsesSent | Should -Be 2
$result.output.sessionId | Should -BeExactly $sessionId
}
It "Invoke-RestMethod can retry with POST" {
$uri = Get-WebListenerUrl -Test 'Retry'
$sessionId = (New-Guid).Guid
$body = @{ sessionid = $sessionId; failureCode = 404; failureCount = 1 }
$commandStr = "Invoke-RestMethod -Uri '$uri' -MaximumRetryCount 2 -RetryIntervalSec 1 -Method POST -Body `$body"
$result = ExecuteWebCommand -command $commandStr
$result.output.failureResponsesSent | Should -Be 1
$result.output.sessionId | Should -BeExactly $sessionId
}
}
}
Describe "Validate Invoke-WebRequest and Invoke-RestMethod -InFile" -Tags "Feature" {
@ -3077,3 +3147,4 @@ Describe "Web cmdlets tests using the cmdlet's aliases" -Tags "CI" {
$result.Hello | Should -Be "world"
}
}

1
test/tools/Modules/WebListener/WebListener.psm1
View File

@ -162,6 +162,7 @@ function Get-WebListenerUrl {
'Response',
'ResponseHeaders',
'Resume',
'Retry',
'/'
)]
[String]$Test,

65
test/tools/WebListener/Controllers/RetryController.cs Normal file
View File

@ -0,0 +1,65 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http.Extensions;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Primitives;
using mvc.Models;
namespace mvc.Controllers
{
public class RetryController : Controller
{
// Dictionary for sessionId as key and failureCode, failureCount and failureResponsesSent as the value.
private static Dictionary<string, Tuple<int, int, int>> retryInfo;
public JsonResult Retry(string sessionId, int failureCode, int failureCount)
{
if (retryInfo == null)
{
retryInfo = new Dictionary<string, Tuple<int, int, int>>();
}
if (retryInfo.TryGetValue(sessionId, out Tuple<int, int, int> retry))
{
// if failureResponsesSent is less than failureCount
if (retry.Item3 < retry.Item2)
{
Response.StatusCode = retry.Item1;
retryInfo[sessionId] = Tuple.Create(retry.Item1, retry.Item2, retry.Item3 + 1);
Hashtable error = new Hashtable { { "error", $"Error: HTTP - {retry.Item1} occurred." } };
return Json(error);
}
else
{
retryInfo.Remove(sessionId);
// echo back sessionId for POST test.
var resp = new Hashtable { { "failureResponsesSent", retry.Item3 }, { "sessionId", sessionId } };
return Json(resp);
}
}
else
{
// initialize the failureResponsesSent as 1.
var newRetryInfoItem = Tuple.Create(failureCode, failureCount, 1);
retryInfo.Add(sessionId, newRetryInfoItem);
Response.StatusCode = failureCode;
Hashtable error = new Hashtable { { "error", $"Error: HTTP - {failureCode} occurred." } };
return Json(error);
}
}
public IActionResult Error()
{
return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier });
}
}
}

20
test/tools/WebListener/README.md
View File

@ -28,8 +28,8 @@ The `WebListener.dll` takes 6 arguments:
Import-Module .\build.psm1
Publish-PSTestTools
$Listener = Start-WebListener -HttpPort 8083 -HttpsPort 8084 -Tls11Port 8085 -TlsPort 8086
```
```
## Tests
### / or /Home/
@ -682,3 +682,21 @@ X-WebListener-Has-Range: false
Content-Length: 20
Content-Type: application/octet-stream
```
### /Retry/{sessionId}/{failureCode}/{failureCount}
This endpoint causes the failure specified by `failureCode` for `failureCount` number of times.
After that a status 200 is returned with body containing the number of times the failure was caused.
```powershell
$response = Invoke-WebRequest -Uri 'http://127.0.0.1:8083/Retry?failureCode=599&failureCount=2&sessionid=100&' -MaximumRetryCount 2 -RetryIntervalSec 1
```
Response Body:
```json
{
"failureResponsesSent":2,
"sessionId":100
}
```

4
test/tools/WebListener/Startup.cs
View File

@ -79,6 +79,10 @@ namespace mvc
template: "Delete",
defaults: new {controller = "Get", action = "Index"},
constraints: new RouteValueDictionary(new { httpMethod = new HttpMethodRouteConstraint("DELETE") }));
routes.MapRoute(
name: "retry",
template: "Retry/{sessionId?}/{failureCode?}/{failureCount?}",
defaults: new { controller = "Retry", action = "Retry" });
});
}
}

1
test/tools/WebListener/Views/Home/Index.cshtml
View File

@ -19,4 +19,5 @@
<li><a href="/Redirect/">/Redirect/{count}</a> - 302 redirect <i>count</i> times.</li>
<li><a href="/Response/?statuscode=200&contenttype=application%2Fjson&body=%22Body%20text%22&headers=%7B%22x-header%22%3A%20%22Response%20Header%20Value%22%7D">/Response/?statuscode=&lt;StatusCode&gt;&amp;body=&lt;ResponseBody&gt;&amp;contenttype=&lt;ResponseContentType&gt;&amp;headers=&lt;JsonHeadersObject&gt;</a> - Returns the given response.</li>
<li><a href="/ResponseHeaders/?key=val">/ResponseHeaders/?key=val</a> - Returns given response headers.</li>
<li><a href="/Retry/?sessionid=100&failureCode=599&failureCount=2">/Retry/?sessionid=100&failureCode=599&failureCount=2</a> - Returns HTTP status code <i>failureCode</i>, <i>failureCount</i> number of times.</li>
</ul>