Given a URL, what would be the most efficient code to download the contents of that web page? I am only considering the HTML, not associated images, JS and CSS.
here is my answer ,a method that takes a URL and return a string
public static string downloadWebPage(string theURL)
{
//### download a web page to a string
WebClient client = new WebClient();
client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
Stream data = client.OpenRead(theURL);
StreamReader reader = new StreamReader(data);
string s = reader.ReadToEnd();
return s;
}