#pragma once #ifdef _MSC_VER #include #endif #include #include "httpagent.h" #include "httpparser.h" struct TFakeBackup { int Write(void* /*buf*/, size_t /*size*/) { return 0; } }; template struct TFakeAlloc { void Shrink(void* /*buf*/, size_t /*size*/) { } void* Grab(size_t /*min*/, size_t* real) { *real = bufsize; return buf; } char buf[bufsize]; }; template , typename TCheck = TFakeCheck<>, typename TWriter = TFakeBackup, typename TAgent = THttpAgent<>> class THttpFetcher: public THttpParser, public TAlloc, public TWriter, public TAgent { public: static const size_t TCP_MIN = 1500; static int TerminateNow; THttpFetcher() : THttpParser() , TAlloc() , TWriter() , TAgent() { } virtual ~THttpFetcher() { } int Fetch(THttpHeader* header, const char* path, const char* const* headers, int persistent, bool head_request = false) { int ret = 0; int fetcherr = 0; THttpParser::Init(header, head_request); const char* scheme = HttpUrlSchemeKindToString((THttpURL::TSchemeKind)TAgent::GetScheme()); size_t schemelen = strlen(scheme); if (*path == '/') { header->base = TStringBuf(scheme, schemelen); header->base += TStringBuf("://", 3); header->base += TStringBuf(TAgent::pHostBeg, TAgent::pHostEnd - TAgent::pHostBeg); header->base += path; } else { if (strlen(path) >= FETCHER_URL_MAX) { header->error = HTTP_URL_TOO_LARGE; return 0; } header->base = path; } if ((ret = TAgent::RequestGet(path, headers, persistent, head_request))) { header->error = (i16)ret; return 0; } bool inheader = 1; void *bufptr = nullptr, *buf = nullptr, *parsebuf = nullptr; ssize_t got; size_t buffree = 0, bufsize = 0, buflen = 0; size_t maxsize = TCheck::GetMaxHeaderSize(); do { if (buffree < TCP_MIN) { if (buf) { TAlloc::Shrink(buf, buflen - buffree); if (TWriter::Write(buf, buflen - buffree) < 0) { buf = nullptr; ret = EIO; break; } } if (!(buf = TAlloc::Grab(TCP_MIN, &buflen))) { ret = ENOMEM; break; } bufptr = buf; buffree = buflen; } if ((got = TAgent::read(bufptr, buffree)) < 0) { fetcherr = errno; if (errno == EINTR) header->error = HTTP_INTERRUPTED; else if (errno == ETIMEDOUT) header->error = HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING; else header->error = HTTP_CONNECTION_LOST; break; } parsebuf = bufptr; bufptr = (char*)bufptr + got; bufsize += got; buffree -= got; THttpParser::Parse(parsebuf, got); if (header->error) break; //if ANY error ocurred we will stop download that file or will have unprognosed stream position until MAX size reached if (inheader && THttpParser::GetState() != THttpParser::hp_in_header) { inheader = 0; if (TCheck::Check(header)) break; if (header->header_size > (long)maxsize) { header->error = HTTP_HEADER_TOO_LARGE; break; } } if (!inheader) { maxsize = TCheck::GetMaxBodySize(header); } if (header->http_status >= HTTP_EXTENDED) break; if (bufsize > maxsize) { header->error = inheader ? HTTP_HEADER_TOO_LARGE : HTTP_BODY_TOO_LARGE; break; } if (TerminateNow) { header->error = HTTP_INTERRUPTED; break; } } while (THttpParser::GetState() > THttpParser::hp_eof); i64 Adjustment = 0; if (!header->error) { if (header->transfer_chunked) { Adjustment = header->header_size + header->entity_size - bufsize - 1; } else if (header->content_length >= 0) { Adjustment = header->header_size + header->content_length - bufsize; } if (Adjustment > 0) Adjustment = 0; } if (buf) { TAlloc::Shrink(buf, buflen - buffree + Adjustment); if (TWriter::Write(buf, buflen - buffree) < 0) ret = EIO; } TCheck::CheckEndDoc(header); if (ret || header->error || header->http_status >= HTTP_EXTENDED || header->connection_closed) { TAgent::Disconnect(); if (!fetcherr) fetcherr = errno; } errno = fetcherr; return ret; } }; template int THttpFetcher::TerminateNow = 0;