Getting HTML from response stream using Indy's IDTCPClient [closed]

Answer 1

The previous employee obviously did not know about the existence of Indy's TIdHTTP component, or how Indy's ReadStream() method actually works, or how HTTP actually works in general. You should re-write the function to use TIdHTTP and let it handle all of the details for you, eg:

function TMyConnector.GET(aRawHeader: String): String;
begin
  FHTTP.ProtocolVersion := ...; // pv1_0 or pv1_1

  // Indy has a TIdCookieManager component that can be attached to
  // the TIdHTTP.CookieManager property, which handles parsing,
  // tracking, and sending back cookies for you for each HTTP
  // request, so this should be re-written to utilize that
  // functionality...
  //
  FHTTP.Request.CustomHeaders.Values['Cookie'] := 'UserHPos=IOGLO00003000090000C000BS; '+
  'LOSID=qsBiy/wEDCq6tOXFzGbOlTD1lmo5AXdFnCkbzzPn6+qCeheYVyTcumRrjsqh+Hds4Fr2gZDazfDzGN1RA+nnHuQQeBy78ZUgctrZyyy9MnGl2qI/ulkV6EPxAfmmLg/lopRq99f5gAcG/dgtytAJjS+aD5DqtHGrAqjiqgtkwuA=; '+
  'LoginHPos=IOGLO00003000090000C000BS; '+
  'UIHPos=IOGLO00003000020000500003; '+
  'LOSG=61939308-7C83-47ED-B909-2D2D10AD7026; '+
  'fControllingBusiness=IOGLO000030000900001000050000200001';

  FHTTP.Request.Connection := 'Close';

  if ResponseStream = nil then
    ResponseStream := TMemoryStream.Create
  else
    ResponseStream.Size := 0;

  try
    try
      FHTTP.Get('http://' + FHost + '/' + aRawHeader, ResponseStream);
    finally
      FRawResponse := FHTTP.Response.RawHeaders.Text;
      Result := FRawResponse;
    end;
  except
    on E: EIdHTTPProtocolException do
    begin
      // HTTP error response. You can grab the error content if you need it...
      WriteStringToStream(ResponseStream, E.ErrorMessage);
    end;
    on E: Exception do
    begin
      // some other error, handle as needed...
    end;
  end;
end;

If a re-write to use TIdHTTP is not possible, then you must update the original code to actually implement rudimentary HTTP parsing. It is not enough to simply call ReadStream(), you have to know WHEN to call it, and WHAT parameters to pass to it. Try something more like this:

function TMyConnector.GET(aRawHeader: String): String;
var
  Headers: TIdHeaderList;
  Size: integer;

  function InternalReadLn: String;
  begin
    Result := FSock.IOHandler.ReadLn;
    if FSock.IOHandler.ReadLnTimedout then begin
      raise EIdReadTimeout.Create('');
    end;
  end;

  function ChunkSize: integer;
  var
    j: Integer;
    s: string;
  begin
    s := InternalReadLn;
    j := Pos(';', s); {do not localize}
    if j > 0 then begin
      s := Copy(s, 1, j - 1);
    end;
    Result := StrToInt('$' + Trim(s), 0);
  end;

begin
  if Not Connected then Connected := True;
  if Connected then
  begin
    FRawRequest :=  'GET /'+ aRawHeader + ' HTTP/'+HTTPVerText+#13#10+
                    'Host: '+FHost+#13#10+

                    'Cookie: UserHPos=IOGLO00003000090000C000BS; '+
                    'LOSID=qsBiy/wEDCq6tOXFzGbOlTD1lmo5AXdFnCkbzzPn6+qCeheYVyTcumRrjsqh+Hds4Fr2gZDazfDzGN1RA+nnHuQQeBy78ZUgctrZyyy9MnGl2qI/ulkV6EPxAfmmLg/lopRq99f5gAcG/dgtytAJjS+aD5DqtHGrAqjiqgtkwuA=; '+
                    'LoginHPos=IOGLO00003000090000C000BS; '+
                    'UIHPos=IOGLO00003000020000500003; '+
                    'LOSG=61939308-7C83-47ED-B909-2D2D10AD7026; '+
                    'fControllingBusiness=IOGLO000030000900001000050000200001'+#13#10+

                    'Connection: Close'+#13#10+
                    #13#10;

    FSock.IOHandler.Write(FRawRequest);
    FRawResponse := FSock.IOHandler.ReadLn(#13#10#13#10,nil);
    Result := FRawResponse;

    if ResponseStream = nil then ResponseStream := TMemoryStream.Create
    else ResponseStream.Size := 0;

    Headers := TIdHeaderList.Create(QuoteHTTP);
    try
      Headers.Text := FRawResponse;

      if Pos('chunked', LowerCase(Headers.Values['Transfer-Encoding']) > 0 then
      begin
        Size := ChunkSize;
        while Size <> 0 do begin
          FSock.IOHandler.ReadStream(ResponseStream, Size);
          InternalReadLn;
          Size := ChunkSize;
        end;
        repeat until InternalReadLn = '';
      end
      else if Headers.IndexOfName('Content-Length') <> -1 then
      begin
        FSock.IOHandler.ReadStream(ResponseStream, StrToInt64(Headers.Values['Content-Length']), False);
      end
      else
        FSock.IOHandler.ReadStream(ResponseStream, -1, True);
    finally
      Headers.Free;
    end;

    if Connected and (Not KeepAlive) then Connected := False;
  end;
end;