I try to download some web pages from a "program TV" site, it worked before but now something has changed and impossible to download a page from this site, the url is: https://www.programme-tv.net/programme/chaines-tv.html
This is my code:
//url = https://www.programme-tv.net/programme/chaines-tv.html
public async Task<PageResponse> GetPage(string url, int repeat = 0, string moreInfoForLog = null)
{
PageResponse pageResponse = new PageResponse();
Thread.Sleep(100);
ServicePointManager.Expect100Continue = false;
ServicePointManager.MaxServicePointIdleTime = 30000;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
ServicePointManager.ServerCertificateValidationCallback += TlsValidationCallback;
string result = null;
HttpWebRequest webReq = (HttpWebRequest)HttpWebRequest.Create(url);
try
{
webReq.CookieContainer = new CookieContainer();
webReq.AllowAutoRedirect = true;
webReq.KeepAlive = true;
webReq.MaximumAutomaticRedirections = 50;
webReq.ConnectionGroupName = Guid.NewGuid().ToString();
webReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
webReq.Method = "GET";
webReq.UserAgent = GetUserAgent(); //get random user Agent from another method
webReq.Accept = "ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
webReq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8,es-ES;q=0.5,en;q=0.3");
webReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate;q=0.8");
webReq.Headers.Add(HttpRequestHeader.CacheControl, "no-cache");
using (WebResponse response = await webReq.GetResponseAsync())
{
using (Stream stream = response.GetResponseStream())
{
StreamReader reader = new StreamReader(stream);
result = await reader.ReadToEndAsync();
pageResponse.Html = result;
}
}
}
catch (WebException ex)
{
HttpWebResponse errorResponse = ex.Response as HttpWebResponse;
if (errorResponse != null && errorResponse.StatusCode == HttpStatusCode.NotFound || ex.Status == WebExceptionStatus.ConnectionClosed)
{
pageResponse.isError = true;
pageResponse.Message = $"NOT FOUND {url}: {ex.Message}";
_log.Debug(pageResponse.Message);
}
else if (repeat < 5 && errorResponse != null && (errorResponse.StatusCode == HttpStatusCode.Redirect || errorResponse.StatusCode == HttpStatusCode.MovedPermanently))
{
url = errorResponse.Headers["Location"];
return await GetPage(url, repeat, moreInfoForLog);
}
else if (repeat < 5)
{
Thread.Sleep(5000);
repeat++;
return await GetPage(url, repeat, moreInfoForLog);
}
else
{
pageResponse.isError = true;
pageResponse.Message = $"WEB EXCEPTION ERROR {url} tried 5 times: {ex.Message}";
}
}
catch (Exception ex)
{
if (repeat < 5)
{
Thread.Sleep(5000);
repeat++;
return await GetPage(url, repeat, moreInfoForLog);
}
else
{
pageResponse.isError = true;
pageResponse.Message = $"GENERAL ERROR {url} tried 5 times: {ex.Message}";
}
}
return pageResponse;
}
private bool TlsValidationCallback(object sender, X509Certificate CACert, X509Chain CAChain, SslPolicyErrors sslPolicyErrors)
{
if ((sslPolicyErrors == SslPolicyErrors.None))
return true;
X509Certificate2 _Certificate = new X509Certificate2(CACert);
CAChain.Build(_Certificate);
foreach (X509ChainStatus CACStatus in CAChain.ChainStatus)
{
if ((CACStatus.Status != X509ChainStatusFlags.NoError) & (CACStatus.Status != X509ChainStatusFlags.UntrustedRoot))
return false;
}
return true;
}
The method GetPage return a simple object:
public class PageResponse
{
public string Html { get; set; }
public bool isError { get; set; }
public string Message { get; set; }
}
Now the error returning is always:
Too many automatic redirections were attempted.
I read all tickets here on stackoverflow about this error but no one helps me, I tried everything !!
Thanks for your help, I'm curious if you can find a solution ! :)