ydb/library/cpp/http/fetch/httpload.cpp

#include "httpload.h"

/************************************************************/
/************************************************************/
httpAgentReader::httpAgentReader(httpSpecialAgent& agent,
                                 const char* baseUrl,
                                 bool assumeConnectionClosed,
                                 bool use_auth,
                                 int bufSize)
    : Header_()
    , Agent_(agent)
    , Buffer_(new char[bufSize])
    , BufPtr_(Buffer_)
    , BufSize_(bufSize)
    , BufRest_(0)
{
    HeadRequest = false;
    Header = &Header_;
    if (use_auth)
        HeaderParser.Init(&Header_);
    else
        HeaderParser.Init(Header);
    setAssumeConnectionClosed(assumeConnectionClosed ? 1 : 0);
    Header_.SetBase(baseUrl);

    if (Header_.error)
        State = hp_error;
    else
        State = hp_in_header;
}

/************************************************************/
httpAgentReader::~httpAgentReader() {
    delete[] Buffer_;
}

/************************************************************/
void httpAgentReader::readBuf() {
    assert(BufRest_ == 0);
    if (!BufPtr_) {
        BufRest_ = -1;
        return;
    }

    BufRest_ = Agent_.read(Buffer_, BufSize_);
    if (BufRest_ <= 0) {
        BufRest_ = -1;
        BufPtr_ = nullptr;
    } else {
        BufPtr_ = Buffer_;

        //cout << "BUF: " << mBuffer << endl << endl;
    }
}

/************************************************************/
const THttpHeader* httpAgentReader::readHeader() {
    while (State == hp_in_header) {
        if (!step()) {
            Header_.error = HTTP_CONNECTION_LOST;
            return nullptr;
        }
        ParseGeneric(BufPtr_, BufRest_);
    }
    if (State == hp_eof || State == hp_error) {
        BufPtr_ = nullptr;
        BufRest_ = -1;
    }
    if (State == hp_error || Header_.error)
        return nullptr;
    return &Header_;
}

/************************************************************/
long httpAgentReader::readPortion(void*& buf) {
    assert(State != hp_in_header);

    long Chunk = 0;
    do {
        if (BufSize_ == 0 && !BufPtr_)
            return 0;

        if (!step())
            return 0;

        Chunk = ParseGeneric(BufPtr_, BufRest_);
        buf = BufPtr_;

        if (State == hp_error && Header_.entity_size > Header_.content_length) {
            Chunk -= (Header_.entity_size - Header_.content_length);
            BufPtr_ = (char*)BufPtr_ + Chunk;
            BufRest_ = 0;
            State = hp_eof;
            Header_.error = 0;
            break;
        }

        BufPtr_ = (char*)BufPtr_ + Chunk;
        BufRest_ -= Chunk;

        if (State == hp_eof || State == hp_error) {
            BufRest_ = -1;
            BufPtr_ = nullptr;
        }
    } while (!Chunk);
    return Chunk;
}

/************************************************************/
bool httpAgentReader::skipTheRest() {
    void* b;
    while (!eof())
        readPortion(b);
    return (State == hp_eof);
}

/************************************************************/
/************************************************************/
httpLoadAgent::httpLoadAgent(bool handleAuthorization,
                             socketHandlerFactory& factory)
    : Factory_(factory)
    , HandleAuthorization_(handleAuthorization)
    , URL_()
    , PersistentConn_(false)
    , Reader_(nullptr)
    , Headers_()
    , ErrCode_(0)
    , RealHost_(nullptr)
{
}

/************************************************************/
httpLoadAgent::~httpLoadAgent() {
    delete Reader_;
    free(RealHost_);
}

/************************************************************/
void httpLoadAgent::clearReader() {
    if (Reader_) {
        bool opened = false;
        if (PersistentConn_) {
            const THttpHeader* H = Reader_->readHeader();
            if (H && !H->connection_closed) {
                Reader_->skipTheRest();
                opened = true;
            }
        }
        if (!opened)
            Disconnect();
        delete Reader_;
        Reader_ = nullptr;
    }
    ErrCode_ = 0;
}
/************************************************************/
void httpLoadAgent::setRealHost(const char* hostname) {
    free(RealHost_);
    if (hostname)
        RealHost_ = strdup(hostname);
    else
        RealHost_ = nullptr;
    ErrCode_ = 0;
}

/************************************************************/
void httpLoadAgent::setIMS(const char* ifModifiedSince) {
    char ims_buf[100];
    snprintf(ims_buf, 100, "If-Modified-Since: %s\r\n",
             ifModifiedSince);
    Headers_.push_back(ims_buf);
}

/************************************************************/
void httpLoadAgent::addHeaderInstruction(const char* instr) {
    Headers_.push_back(instr);
}

/************************************************************/
void httpLoadAgent::dropHeaderInstructions() {
    Headers_.clear();
}

/************************************************************/
bool httpLoadAgent::startRequest(const THttpURL& url,
                                 bool persistent,
                                 const TAddrList& addrs)

{
    clearReader();
    ErrCode_ = 0;

    URL_.Clear();
    URL_ = url;
    PersistentConn_ = persistent;
    if (!URL_.IsValidAbs())
        return false;
    if (!HandleAuthorization_ && !URL_.IsNull(THttpURL::FlagAuth))
        return false;

    return doSetHost(addrs) && doStartRequest();
}

/************************************************************/
bool httpLoadAgent::startRequest(const char* url,
                                 const char* url_to_merge,
                                 bool persistent,
                                 const TAddrList& addrs) {
    clearReader();

    URL_.Clear();
    PersistentConn_ = persistent;

    ui64 flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet;
    if (HandleAuthorization_)
        flags |= THttpURL::FeatureAuthSupported;

    if (URL_.Parse(url, flags, url_to_merge) || !URL_.IsValidGlobal())
        return false;

    return doSetHost(addrs) && doStartRequest();
}

/************************************************************/
bool httpLoadAgent::startRequest(const char* url,
                                 const char* url_to_merge,
                                 bool persistent,
                                 ui32 ip) {
    clearReader();

    URL_.Clear();
    PersistentConn_ = persistent;

    ui64 flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet;
    if (HandleAuthorization_)
        flags |= THttpURL::FeatureAuthSupported;

    if (URL_.Parse(url, flags, url_to_merge) || !URL_.IsValidGlobal())
        return false;

    return doSetHost(TAddrList::MakeV4Addr(ip, URL_.GetPort())) && doStartRequest();
}

/************************************************************/
bool httpLoadAgent::doSetHost(const TAddrList& addrs) {
    socketAbstractHandler* h = Factory_.chooseHandler(URL_);
    if (!h)
        return false;
    Socket.setHandler(h);

    if (addrs.size()) {
        ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost),
                           URL_.GetPort(), addrs);
    } else {
        ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost),
                           URL_.GetPort());
    }
    if (ErrCode_)
        return false;

    if (RealHost_) {
        size_t reqHostheaderLen = strlen(RealHost_) + 20;
        free(Hostheader);
        Hostheader = (char*)malloc((HostheaderLen = reqHostheaderLen));
        snprintf(Hostheader, HostheaderLen, "Host: %s\r\n", RealHost_);
    }

    if (!URL_.IsNull(THttpURL::FlagAuth)) {
        if (!HandleAuthorization_) {
            ErrCode_ = HTTP_UNAUTHORIZED;
            return false;
        }

        Digest_.setAuthorization(URL_.Get(THttpURL::FieldUsername),
                                 URL_.Get(THttpURL::FieldPassword));
    }

    return true;
}

/************************************************************/
bool httpLoadAgent::setHost(const char* host_url,
                            const TAddrList& addrs) {
    clearReader();

    URL_.Clear();
    PersistentConn_ = true;

    ui64 flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet;
    if (HandleAuthorization_)
        flags |= THttpURL::FeatureAuthSupported;

    if (URL_.Parse(host_url, flags) || !URL_.IsValidGlobal())
        return false;

    return doSetHost(addrs);
}

/************************************************************/
bool httpLoadAgent::startOneRequest(const char* local_url) {
    clearReader();

    THttpURL lURL;
    if (lURL.Parse(local_url, THttpURL::FeaturesNormalizeSet) || lURL.IsValidGlobal())
        return false;

    URL_.SetInMemory(THttpURL::FieldPath, lURL.Get(THttpURL::FieldPath));
    URL_.SetInMemory(THttpURL::FieldQuery, lURL.Get(THttpURL::FieldQuery));
    URL_.Rewrite();

    return doStartRequest();
}

/************************************************************/
bool httpLoadAgent::doStartRequest() {
    TString urlStr = URL_.PrintS(THttpURL::FlagPath | THttpURL::FlagQuery);
    if (!urlStr)
        urlStr = "/";

    for (int step = 0; step < 10; step++) {
        const char* digestHeader = Digest_.getHeaderInstruction();

        unsigned i = (digestHeader) ? 2 : 1;
        const char** headers =
            (const char**)(alloca((i + Headers_.size()) * sizeof(char*)));

        for (i = 0; i < Headers_.size(); i++)
            headers[i] = Headers_[i].c_str();
        if (digestHeader)
            headers[i++] = digestHeader;
        headers[i] = nullptr;

        ErrCode_ = RequestGet(urlStr.c_str(), headers, PersistentConn_);

        if (ErrCode_) {
            Disconnect();
            return false;
        }

        TString urlBaseStr = URL_.PrintS(THttpURL::FlagNoFrag);

        clearReader();
        Reader_ = new httpAgentReader(*this, urlBaseStr.c_str(),
                                      !PersistentConn_, !Digest_.empty());

        if (Reader_->readHeader()) {
            //mReader->getHeader()->Print();
            if (getHeader()->http_status == HTTP_UNAUTHORIZED &&
                step < 1 &&
                Digest_.processHeader(getAuthHeader(),
                                      urlStr.c_str(),
                                      "GET")) {
                //mReader->skipTheRest();
                delete Reader_;
                Reader_ = nullptr;
                ErrCode_ = 0;
                Disconnect();
                continue;
            }

            return true;
        }
        Disconnect();
        clearReader();

        return false;
    }

    ErrCode_ = HTTP_UNAUTHORIZED;
    return false;
}

/************************************************************/
/************************************************************/