Results 1 to 9 of 9

Thread: Programmatically downloading file

  1. #1
    mitunit is offline C# Forum Newbie
    .NET Framework
    .NET 4.5
    Join Date
    Jan 2017
    Posts
    5
    Rep Power
    0

    Programmatically downloading file

    Hello,

    Using C# 4.5,I'm trying to programmatically log-on to a third-party website and then download a csv file from another page on that site. I tried using following approach.I dont get any errors but it seems to download the contents of their logon page itself and save it as a csv file instead of the actual csv file having data.

    Also,if I read the response stream after logging in, I see that I'm getting the log-on webpage back.
    What am I missing here please? Plus, I need to post the name of the form as well.How do I achieve that?Please advise.
    Thanks.

    Here's my C# code:

                string websiteURL = "https://testLoginPage";                        

    string poststring = "user=testUser&password=TestPwd";

    HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(websiteURL);


    httpRequest.Method = "POST";
    httpRequest.ContentType = "application/x-www-form-urlencoded";


    byte[] bytedata = Encoding.UTF8.GetBytes(poststring);
    httpRequest.ContentLength = bytedata.Length;


    //Log on to the site
    Stream requestStream = httpRequest.GetRequestStream();
    requestStream.Write(bytedata, 0, bytedata.Length);
    requestStream.Close();




    HttpWebResponse httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();
    Stream responseStream = httpWebResponse.GetResponseStream();


    string header1 = httpWebResponse.Headers.Get("Header1");
    string header2 = httpWebResponse.Headers.Get("Header2");
    string cookie = httpWebResponse.Headers.Get("Set-Cookie");



    //Read the response
    StringBuilder sb = new StringBuilder();
    using (StreamReader reader =
    new StreamReader(responseStream, System.Text.Encoding.UTF8))
    {
    string line;
    while ((line = reader.ReadLine()) != null)
    {
    sb.Append(line);
    }
    }
    string htmlResponse = sb.ToString();




    //Download the csv file
    //
    string localFileFormat = string.Format("{0}-{1}", "MyFile", DateTime.Now.ToString("yyyyMMdd"));
    string downloadedFilePath = string.Format(@"{0}\{1}.{2}", DownloadFileInputParams.OuputDirectory, localFileFormat, DownloadFileInputParams.FileType);
    string fileDownloadURL = "https://fileDownloadURL/data?fileType=csv";
    httpRequest = (HttpWebRequest)WebRequest.Create(fileDownloadURL);



    httpRequest.CookieContainer = new CookieContainer();
    httpRequest.Headers.Add("X-DB-NAR", header1);
    httpRequest.Headers.Add("DB-Nickname", header2);
    httpRequest.Headers.Add("Set-Cookie", cookie);


    httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();


    cookie = httpWebResponse.Headers.Get("Set-Cookie");

    httpRequest = (HttpWebRequest)WebRequest.Create(fileDownloadURL);
    httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();



    responseStream = httpWebResponse.GetResponseStream();
    FileStream localStream = File.Create(downloadedFilePath);



    byte[] buffer = new byte[1024];
    int bytesRead;
    int bytesProcessed = 0;
    // Simple do/while loop to read from stream until
    // no bytes are returned
    do
    {
    // Read data (up to 1k) from the stream
    bytesRead = responseStream.Read(buffer, 0, buffer.Length);


    // Write the data to the local file
    localStream.Write(buffer, 0, bytesRead);


    // Increment total bytes processed
    bytesProcessed += bytesRead;
    } while (bytesRead > 0);




    if (httpWebResponse != null) httpWebResponse.Close();
    if (responseStream != null) responseStream.Close();
    if (localStream != null) localStream.Close();



    return;
    Last edited by JohnH; 01-31-2017 at 4:04 PM. Reason: xcode box

  2. #2
    JohnH's Avatar
    JohnH is offline C# Forum Moderator
    .NET Framework
    .NET 4.5
    Join Date
    Apr 2011
    Location
    Norway
    Posts
    331
    Rep Power
    87
    Typically one would create and set a CookieContainer for the login, then assign the same CookieContainer to the next request.

  3. #3
    mitunit is offline C# Forum Newbie
    .NET Framework
    .NET 4.5
    Join Date
    Jan 2017
    Posts
    5
    Rep Power
    0
    Thanks JohnH.

    I was playing around with my code and modified my code yesterday as per following. And finally it did work.I was able to download about 0.5 MB of csv file from the website using this program.
    (I also saw a "Welcome page" response in the htmlResponse variable while debugging.)


    But when I ran the same code today multiple times and am getting an Internal Server error(500) at the following line.
    HttpWebResponse httpWebResponse1 = (HttpWebResponse)httpRequest1.GetResponse();


    Not sure why am I getting this error??


    Would you be able to advise on this please?


    Thanks for your help.
    --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
    string websiteURL = "https://testLoginPage";


    string poststring = "user=testUser&password=TestPwd";


    CookieContainer cookieContainer = new CookieContainer();


    //Create request for log in page
    HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(websiteURL);
    httpRequest.CookieContainer = cookieContainer;


    httpRequest.Method = "POST";
    httpRequest.ContentType = "application/x-www-form-urlencoded";


    byte[] bytedata = Encoding.UTF8.GetBytes(poststring);
    httpRequest.ContentLength = bytedata.Length;
    httpRequest.KeepAlive = true;


    using (Stream requestStream = httpRequest.GetRequestStream())
    {
    requestStream.Write(bytedata, 0, bytedata.Length);
    }


    HttpWebResponse httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();


    //Read the cookies from the request object
    httpRequest.CookieContainer.GetCookies(httpRequest.RequestUri);




    Stream responseStream = httpWebResponse.GetResponseStream();


    //Read the response
    StringBuilder sb = new StringBuilder();
    using (StreamReader reader = new StreamReader(responseStream, System.Text.Encoding.UTF8))
    {
    string line;
    while ((line = reader.ReadLine()) != null)
    {
    sb.Append(line);
    }


    //reader.Close();
    }
    string htmlResponse = sb.ToString();


    //Download the csv file
    string localFileFormat = string.Format("{0}-{1}", "MyFile", DateTime.Now.ToString("yyyyMMdd"));
    string downloadedFilePath = string.Format(@"{0}\{1}.{2}", DownloadFileInputParams.OuputDirectory, localFileFormat, DownloadFileInputParams.FileType);
    string fileDownloadURL = "https://fileDownloadURL/data?fileType=csv";




    HttpWebRequest httpRequest1 = (HttpWebRequest)WebRequest.Create(fileDownloadURL);
    httpRequest1.CookieContainer = cookieContainer;
    httpRequest1.Method = "GET";


    //cookie.Domain = "localhost";
    //httpRequest1.CookieContainer.Add(cookie);


    HttpWebResponse httpWebResponse1 = (HttpWebResponse)httpRequest1.GetResponse();
    responseStream = httpWebResponse1.GetResponseStream();


    FileStream localStream = File.Create(downloadedFilePath);


    // Allocate a 1k buffer
    byte[] buffer = new byte[1024];
    int bytesRead;
    int bytesProcessed = 0;
    // Simple do/while loop to read from stream until
    // no bytes are returned
    do
    {
    // Read data (up to 1k) from the stream
    bytesRead = responseStream.Read(buffer, 0, buffer.Length);


    // Write the data to the local file
    localStream.Write(buffer, 0, bytesRead);


    // Increment total bytes processed
    bytesProcessed += bytesRead;
    } while (bytesRead > 0);




    if (httpWebResponse != null) httpWebResponse.Close();
    if (responseStream != null) responseStream.Close();
    if (localStream != null) localStream.Close();


    //Log out


    // Create a request using a URL that can receive a post.
    httpRequest = (HttpWebRequest)HttpWebRequest.Create("https://logutPage");
    httpRequest.CookieContainer = cookieContainer;


    // Set the Method property of the request to POST.
    httpRequest.Method = "GET";


    HttpWebResponse response = (HttpWebResponse)httpRequest.GetResponse();
    response.Close();
    Last edited by JohnH; 01-31-2017 at 4:04 PM. Reason: xcode box

  4. #4
    JohnH's Avatar
    JohnH is offline C# Forum Moderator
    .NET Framework
    .NET 4.5
    Join Date
    Apr 2011
    Location
    Norway
    Posts
    331
    Rep Power
    87
    Internal Server error(500) means problem at their server, either because they just had problems with their code at that time, or perhaps they changed something so that your parameters are no longer valid. (ie "data?fileType=csv")

    See BB Code List - C# Developer Forums for how to format code when posting, the advanced posting editor also has nice buttons to insert these tags for you.

  5. #5
    mitunit is offline C# Forum Newbie
    .NET Framework
    .NET 4.5
    Join Date
    Jan 2017
    Posts
    5
    Rep Power
    0
    Hello JohnH,

    If I pass the same cookieContainer to my 2nd HttpWebRequest, then am getting that internal server error: 500.
    If I dont pass it in the request, I dont get an error but I dont the csv file in the response.

    Thanks.

  6. #6
    JohnH's Avatar
    JohnH is offline C# Forum Moderator
    .NET Framework
    .NET 4.5
    Join Date
    Apr 2011
    Location
    Norway
    Posts
    331
    Rep Power
    87
    As I said, maybe the server expects different data now. Do the same thing in a web browser with a web debugger, see if that uses same url/parameters and other data that your code is sending.

  7. #7
    mitunit is offline C# Forum Newbie
    .NET Framework
    .NET 4.5
    Join Date
    Jan 2017
    Posts
    5
    Rep Power
    0
    This issue got resolved when I added following line in my code for downloading the file:"
    downloadFileHttpRequest.UserAgent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.3) Gecko/20100423 Ubuntu/10.04 (lucid) Firefox/3.6.3";
    Thanks JohnH for your help!!!Much appreciated!

  8. #8
    JohnH's Avatar
    JohnH is offline C# Forum Moderator
    .NET Framework
    .NET 4.5
    Join Date
    Apr 2011
    Location
    Norway
    Posts
    331
    Rep Power
    87
    That is not unexpected, but hard to figure out. I've had to do the same several times. Just "Mozilla/5.0" will often do.

  9. #9
    mitunit is offline C# Forum Newbie
    .NET Framework
    .NET 4.5
    Join Date
    Jan 2017
    Posts
    5
    Rep Power
    0
    I see.Sure, will give it a try.Thanks again for your help.

Tags for this Thread

Bookmarks

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •