ydb/library/cpp/uri/assign.cpp

#include "uri.h"
#include "parse.h"

#include <idna.h>

#include <library/cpp/charset/recyr.hh>
#include <util/charset/wide.h>
#include <util/memory/tempbuf.h>
#include <util/string/cast.h>
#include <util/system/yassert.h>
#include <util/system/sys_alloc.h>

namespace NUri {
    TMallocPtr<char> TUri::IDNToAscii(const wchar32* idna) {
        // XXX: don't use punycode_encode directly as it doesn't include
        // proper stringprep and splitting on dot-equivalent characters
        char* buf;
        static_assert(sizeof(*idna) == sizeof(ui32), "fixme");
        if (IDNA_SUCCESS != idna_to_ascii_4z((const uint32_t*)idna, &buf, 0))
            buf = nullptr;
        return buf;
    }

    TMallocPtr<char> TUri::IDNToAscii(const TStringBuf& host, ECharset enc) {
        TTempBuf buf(sizeof(wchar32) * (1 + host.length()));
        wchar32* wbuf = reinterpret_cast<wchar32*>(buf.Data());

        const size_t written = NDetail::NBaseOps::Recode(host, wbuf, enc).length();
        wbuf[written] = 0;

        return IDNToAscii(wbuf);
    }

    TStringBuf TUri::HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc) {
        TStringBuf outhost; // store the result here before returning it, to get RVO

        size_t buflen = 0;

        if (hasExtended && !allowIDN)
            return outhost; // definitely can't convert

        // charset-recode: RFC 3986, 3.2.2, requires percent-encoded non-ASCII
        // chars in reg-name to be UTF-8 so convert to UTF-8 prior to decoding
        const bool recoding = CODES_UTF8 != enc && hasExtended;
        if (recoding) {
            size_t nrd, nwr;
            buflen = host.length() * 4;
            buf.Reset(static_cast<char*>(y_allocate(buflen)));
            if (RECODE_OK != Recode(enc, CODES_UTF8, host.data(), buf.Get(), host.length(), buflen, nrd, nwr))
                return outhost;
            host = TStringBuf(buf.Get(), nwr);
        }

        // percent-decode
        if (0 == buflen) {
            buflen = host.length();
            buf.Reset(static_cast<char*>(y_allocate(buflen)));
        }
        // decoding shortens so writing over host in buf is OK
        TMemoryWriteBuffer out(buf.Get(), buflen);
        TEncoder decoder(out, FeatureDecodeANY | FeatureToLower);
        const long outFlags = decoder.ReEncode(host);
        hasExtended = 0 != (outFlags & FeatureEncodeExtendedASCII);

        // check again
        if (hasExtended && !allowIDN)
            return outhost;

        host = out.Str();

        // convert to punycode if needed
        if (!hasExtended) {
            outhost = host;
            return outhost;
        }

        TMallocPtr<char> puny;
        try {
            puny = IDNToAscii(host);
        } catch (const yexception& /* exc */) {
        }

        if (!puny) {
            // XXX: try user charset unless UTF8 or converted to it
            if (CODES_UTF8 == enc || recoding)
                return outhost;
            try {
                puny = IDNToAscii(host, enc);
            } catch (const yexception& /* exc */) {
                return outhost;
            }
            if (!puny)
                return outhost;
        }

        buf = puny;
        outhost = buf.Get();

        return outhost;
    }

    TStringBuf TUri::HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc) {
        // find what we have
        long haveFlags = 0;
        for (size_t i = 0; i != host.length(); ++i)
            haveFlags |= TEncoder::GetFlags(host[i]).FeatFlags;

        // interested in encoded characters or (if IDN is allowed) extended ascii
        TStringBuf outhost;
        const bool haveExtended = haveFlags & FeatureEncodeExtendedASCII;

        if (!haveExtended || allowIDN) {
            if (!haveExtended && 0 == (haveFlags & FeatureDecodeANY))
                outhost = host;
            else
                outhost = HostToAscii(host, buf, haveExtended, allowIDN, enc);
        }

        return outhost;
    }

    static inline bool AppendField(TMemoryWriteBuffer& out, TField::EField fld, const TStringBuf& val, long flags) {
        if (val.empty())
            return false;
        if (flags & TFeature::FeaturesAllEncoder)
            TUri::ReEncodeField(out, val, fld, flags);
        else
            out << val;
        return true;
    }

    TState::EParsed TUri::AssignImpl(const TParser& parser, TScheme::EKind defscheme) {
        Clear();

        TState::EParsed ret = parser.State;
        if (ParsedBadFormat <= ret)
            return ret;

        const TSection& scheme = parser.Get(FieldScheme);
        const TSchemeInfo& schemeInfo = SetSchemeImpl(parser.Scheme);

        // set the scheme always if available
        if (schemeInfo.Str.empty() && scheme.IsSet())
            FldSet(FieldScheme, scheme.Get());

        if (ParsedOK != ret)
            return ret;

        size_t buflen = 0;

        // special processing for fields

        const bool convertIDN = parser.Flags & FeatureConvertHostIDN;
        long flags = parser.Flags.Allow;
        if (convertIDN)
            flags |= FeatureAllowHostIDN | FeatureCheckHost;

        // process non-ASCII host for punycode

        TMallocPtr<char> hostptr;
        TStringBuf hostascii;       // empty: use host field; non-empty: ascii
        bool hostConverted = false; // hostascii is empty or the original
        const TSection& host = parser.Get(FieldHost);
        if (host.IsSet() && !FldIsSet(FieldHost)) {
            const bool allowIDN = (flags & FeatureAllowHostIDN);
            const TStringBuf hostbuf = host.Get();

            // if we know we have and allow extended-ASCII chars, no need to check further
            if (allowIDN && (host.GetFlagsAllPlaintext() & FeatureEncodeExtendedASCII))
                hostascii = HostToAscii(hostbuf, hostptr, true, true, parser.Enc);
            else
                hostascii = HostToAscii(hostbuf, hostptr, allowIDN, parser.Enc);

            if (hostascii.empty())
                ret = ParsedBadHost; // exists but cannot be converted
            else if (hostbuf.data() != hostascii.data()) {
                hostConverted = true;
                buflen += 1 + hostascii.length();
                if (convertIDN)
                    FldMarkSet(FieldHost); // so that we don't process host below
            }
        }

        // add unprocessed fields

        for (int idx = 0; idx < FieldUrlMAX; ++idx) {
            const EField fld = EField(idx);
            const TSection& section = parser.Get(fld);
            if (section.IsSet() && !FldIsSet(fld))
                buflen += 1 + section.EncodedLen(); // includes null
        }
        if (0 == buflen) // no more sections set?
            return ret;

        // process #! fragments
        // https://developers.google.com/webmasters/ajax-crawling/docs/specification

        static const TStringBuf escFragPrefix(TStringBuf("_escaped_fragment_="));

        bool encHashBangFrag = false;
        TStringBuf qryBeforeEscapedFragment;
        TStringBuf qryEscapedFragment;
        do {
            if (FldIsSet(FieldFrag) || FldIsSet(FieldQuery))
                break;

            const TSection& frag = parser.Get(FieldFrag);
            if (frag.IsSet()) {
                if (0 == (parser.Flags & FeatureHashBangToEscapedFragment))
                    break;
                const TStringBuf fragbuf = frag.Get();
                if (fragbuf.empty() || '!' != fragbuf[0])
                    break;
                encHashBangFrag = true;
                // '!' will make space for '&' or '\0' if needed
                buflen += escFragPrefix.length();
                buflen += 2 * fragbuf.length(); // we don't know how many will be encoded
            } else {
                const TSection& qry = parser.Get(FieldQuery);
                if (!qry.IsSet())
                    break;
                // FeatureHashBangToEscapedFragment has preference
                if (FeatureEscapedToHashBangFragment != (parser.Flags & FeaturesEscapedFragment))
                    break;
                qry.Get().RSplit('&', qryBeforeEscapedFragment, qryEscapedFragment);
                if (!qryEscapedFragment.StartsWith(escFragPrefix)) {
                    qryEscapedFragment.Clear();
                    break;
                }
                qryEscapedFragment.Skip(escFragPrefix.length());
                buflen += 2; // for '!' and '\0' in fragment
                buflen -= escFragPrefix.length();
            }
        } while (false);

        // now set all fields prior to validating

        Alloc(buflen);

        TMemoryWriteBuffer out(Buffer.data(), Buffer.size());
        for (int idx = 0; idx < FieldUrlMAX; ++idx) {
            const EField fld = EField(idx);

            const TSection& section = parser.Get(fld);
            if (!section.IsSet() || FldIsSet(fld))
                continue;

            if (FieldQuery == fld && encHashBangFrag)
                continue;

            if (FieldFrag == fld && qryEscapedFragment.IsInited())
                continue;

            char* beg = out.Buf();
            TStringBuf val = section.Get();
            long careFlags = section.GetFlagsEncode();

            switch (fld) {
                default:
                    break;

                case FieldQuery:
                    if (qryEscapedFragment.IsInited()) {
                        const EField dstfld = FieldFrag; // that's where we will store
                        out << '!';
                        if (!qryEscapedFragment.empty())
                            ReEncodeToField(out, qryEscapedFragment, fld, FeatureDecodeANY | careFlags, dstfld, FeatureDecodeANY | parser.GetFieldFlags(dstfld));
                        FldSetNoDirty(dstfld, TStringBuf(beg, out.Buf()));
                        if (qryBeforeEscapedFragment.empty())
                            continue;
                        out << '\0';
                        beg = out.Buf();
                        val = qryBeforeEscapedFragment;
                    }
                    break;

                case FieldFrag:
                    if (encHashBangFrag) {
                        const EField dstfld = FieldQuery; // that's where we will store
                        const TSection& qry = parser.Get(dstfld);
                        if (qry.IsSet())
                            if (AppendField(out, dstfld, qry.Get(), qry.GetFlagsEncode()))
                                out << '&';
                        out << escFragPrefix;
                        val.Skip(1); // skip '!'
                        ReEncodeToField(out, val, fld, careFlags, dstfld, parser.GetFieldFlags(dstfld));
                        FldSetNoDirty(dstfld, TStringBuf(beg, out.Buf()));
                        continue;
                    }
                    break;
            }

            AppendField(out, fld, val, careFlags);
            char* end = out.Buf();

            if (careFlags & FeaturePathOperation) {
                if (!PathOperation(beg, end, PathOperationFlag(parser.Flags)))
                    return ParsedBadPath;

                Y_ASSERT(beg >= out.Beg());
                out.SetPos(end);
            }

            FldSetNoDirty(fld, TStringBuf(beg, end));

            // special character case
            const long checkChars = section.GetFlagsAllPlaintext() & FeaturesCheckSpecialChar;
            if (0 != checkChars) { // has unencoded special chars: check permission
                const long allowChars = parser.GetFieldFlags(fld) & checkChars;
                if (checkChars != allowChars)
                    ret = ParsedBadFormat;
            }

            out << '\0';
        }

        if (hostConverted) {
            char* beg = out.Buf();
            out << hostascii;
            char* end = out.Buf();
            const EField fld = convertIDN ? FieldHost : FieldHostAscii;
            FldSetNoDirty(fld, TStringBuf(beg, end));
            out << '\0';
        }

        Buffer.Resize(out.Len());

        if (GetScheme() == SchemeEmpty && SchemeEmpty != defscheme) {
            if (SchemeUnknown == defscheme)
                ret = ParsedBadScheme;
            else
                SetSchemeImpl(defscheme);
        }

        if (0 == (parser.Flags & FeatureAllowEmptyPath))
            CheckMissingFields();

        const TStringBuf& port = GetField(FieldPort);
        if (!port.empty()) {
            if (!TryFromString<ui16>(port, Port))
                ret = ParsedBadPort;
        }

        if (ParsedOK != ret)
            return ret;

        // run validity checks now that all fields are set

        // check the host for DNS compliance
        do {
            if (0 == (flags & FeatureCheckHost))
                break;
            if (hostascii.empty())
                hostascii = GetField(FieldHost);
            if (hostascii.empty())
                break;
            // IP literal
            if ('[' == hostascii[0] && ']' == hostascii.back())
                break;
            ret = CheckHost(hostascii);
            if (ParsedOK != ret)
                return ret;
        } while (false);

        return ret;
    }

    TState::EParsed TUri::ParseImpl(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) {
        Clear();

        if (url.empty())
            return ParsedEmpty;

        if (maxlen > 0 && url.length() > maxlen)
            return ParsedTooLong;

        const TParser parser(flags, url, enc);

        return AssignImpl(parser, defscheme);
    }

    TState::EParsed TUri::Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& url_base, ui32 maxlen, ECharset enc) {
        const TParseFlags flags1 = flags.Exclude(FeatureNoRelPath);
        TState::EParsed ret = ParseImpl(url, url_base.empty() ? flags : flags1, maxlen, SchemeEmpty, enc);
        if (ParsedOK != ret)
            return ret;

        if (!url_base.empty() && !IsValidAbs()) {
            TUri base;
            ret = base.ParseImpl(url_base, flags, maxlen, SchemeEmpty, enc);
            if (ParsedOK != ret)
                return ret;
            Merge(base, PathOperationFlag(flags));
        }

        Rewrite();
        return ret;
    }

    TState::EParsed TUri::Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags, ui32 maxlen, ECharset enc) {
        const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc);
        if (ParsedOK != ret)
            return ret;

        if (!IsValidAbs())
            Merge(base, PathOperationFlag(flags));

        Rewrite();
        return ret;
    }

    TState::EParsed TUri::ParseAbsUri(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) {
        const TState::EParsed ret = ParseImpl(
            url, flags | FeatureNoRelPath, maxlen, defscheme, enc);
        if (ParsedOK != ret)
            return ret;

        if (IsNull(FlagHost))
            return ParsedBadHost;

        Rewrite();
        return ParsedOK;
    }

}