ydb/contrib/python/zstandard/py3/c-ext/bufferutil.c

/**
 * Copyright (c) 2017-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */

#include "python-zstandard.h"

extern PyObject *ZstdError;

static void BufferWithSegments_dealloc(ZstdBufferWithSegments *self) {
    /* Backing memory is either canonically owned by a Py_buffer or by us. */
    if (self->parent.buf) {
        PyBuffer_Release(&self->parent);
    }
    else if (self->useFree) {
        free(self->data);
    }
    else {
        PyMem_Free(self->data);
    }

    self->data = NULL;

    if (self->useFree) {
        free(self->segments);
    }
    else {
        PyMem_Free(self->segments);
    }

    self->segments = NULL;

    PyObject_Del(self);
}

static int BufferWithSegments_init(ZstdBufferWithSegments *self, PyObject *args,
                                   PyObject *kwargs) {
    static char *kwlist[] = {"data", "segments", NULL};

    Py_buffer segments;
    Py_ssize_t segmentCount;
    Py_ssize_t i;

    memset(&self->parent, 0, sizeof(self->parent));

    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*y*:BufferWithSegments",
                                     kwlist, &self->parent, &segments)) {
        return -1;
    }

    if (segments.len % sizeof(BufferSegment)) {
        PyErr_Format(PyExc_ValueError,
                     "segments array size is not a multiple of %zu",
                     sizeof(BufferSegment));
        goto except;
    }

    segmentCount = segments.len / sizeof(BufferSegment);

    /* Validate segments data, as blindly trusting it could lead to arbitrary
    memory access. */
    for (i = 0; i < segmentCount; i++) {
        BufferSegment *segment = &((BufferSegment *)(segments.buf))[i];

        if (segment->offset + segment->length >
            (unsigned long long)self->parent.len) {
            PyErr_SetString(PyExc_ValueError,
                            "offset within segments array references memory "
                            "outside buffer");
            goto except;
            return -1;
        }
    }

    /* Make a copy of the segments data. It is cheap to do so and is a guard
       against caller changing offsets, which has security implications. */
    self->segments = PyMem_Malloc(segments.len);
    if (!self->segments) {
        PyErr_NoMemory();
        goto except;
    }

    memcpy(self->segments, segments.buf, segments.len);
    PyBuffer_Release(&segments);

    self->data = self->parent.buf;
    self->dataSize = self->parent.len;
    self->segmentCount = segmentCount;

    return 0;

except:
    PyBuffer_Release(&self->parent);
    PyBuffer_Release(&segments);
    return -1;
}

/**
 * Construct a BufferWithSegments from existing memory and offsets.
 *
 * Ownership of the backing memory and BufferSegments will be transferred to
 * the created object and freed when the BufferWithSegments is destroyed.
 */
ZstdBufferWithSegments *
BufferWithSegments_FromMemory(void *data, unsigned long long dataSize,
                              BufferSegment *segments,
                              Py_ssize_t segmentsSize) {
    ZstdBufferWithSegments *result = NULL;
    Py_ssize_t i;

    if (NULL == data) {
        PyErr_SetString(PyExc_ValueError, "data is NULL");
        return NULL;
    }

    if (NULL == segments) {
        PyErr_SetString(PyExc_ValueError, "segments is NULL");
        return NULL;
    }

    for (i = 0; i < segmentsSize; i++) {
        BufferSegment *segment = &segments[i];

        if (segment->offset + segment->length > dataSize) {
            PyErr_SetString(PyExc_ValueError,
                            "offset in segments overflows buffer size");
            return NULL;
        }
    }

    result = PyObject_New(ZstdBufferWithSegments, ZstdBufferWithSegmentsType);
    if (NULL == result) {
        return NULL;
    }

    result->useFree = 0;

    memset(&result->parent, 0, sizeof(result->parent));
    result->data = data;
    result->dataSize = dataSize;
    result->segments = segments;
    result->segmentCount = segmentsSize;

    return result;
}

static Py_ssize_t BufferWithSegments_length(ZstdBufferWithSegments *self) {
    return self->segmentCount;
}

static ZstdBufferSegment *BufferWithSegments_item(ZstdBufferWithSegments *self,
                                                  Py_ssize_t i) {
    ZstdBufferSegment *result = NULL;

    if (i < 0) {
        PyErr_SetString(PyExc_IndexError, "offset must be non-negative");
        return NULL;
    }

    if (i >= self->segmentCount) {
        PyErr_Format(PyExc_IndexError, "offset must be less than %zd",
                     self->segmentCount);
        return NULL;
    }

    if (self->segments[i].length > PY_SSIZE_T_MAX) {
        PyErr_Format(PyExc_ValueError,
                     "item at offset %zd is too large for this platform", i);
        return NULL;
    }

    result = (ZstdBufferSegment *)PyObject_CallObject(
        (PyObject *)ZstdBufferSegmentType, NULL);
    if (NULL == result) {
        return NULL;
    }

    result->parent = (PyObject *)self;
    Py_INCREF(self);

    result->data = (char *)self->data + self->segments[i].offset;
    result->dataSize = (Py_ssize_t)self->segments[i].length;
    result->offset = self->segments[i].offset;

    return result;
}

static int BufferWithSegments_getbuffer(ZstdBufferWithSegments *self,
                                        Py_buffer *view, int flags) {
    if (self->dataSize > PY_SSIZE_T_MAX) {
        view->obj = NULL;
        PyErr_SetString(PyExc_BufferError,
                        "buffer is too large for this platform");
        return -1;
    }

    return PyBuffer_FillInfo(view, (PyObject *)self, self->data,
                             (Py_ssize_t)self->dataSize, 1, flags);
}

static PyObject *BufferWithSegments_tobytes(ZstdBufferWithSegments *self) {
    if (self->dataSize > PY_SSIZE_T_MAX) {
        PyErr_SetString(PyExc_ValueError,
                        "buffer is too large for this platform");
        return NULL;
    }

    return PyBytes_FromStringAndSize(self->data, (Py_ssize_t)self->dataSize);
}

static ZstdBufferSegments *
BufferWithSegments_segments(ZstdBufferWithSegments *self) {
    ZstdBufferSegments *result = (ZstdBufferSegments *)PyObject_CallObject(
        (PyObject *)ZstdBufferSegmentsType, NULL);
    if (NULL == result) {
        return NULL;
    }

    result->parent = (PyObject *)self;
    Py_INCREF(self);
    result->segments = self->segments;
    result->segmentCount = self->segmentCount;

    return result;
}

#if PY_VERSION_HEX < 0x03090000
static PyBufferProcs BufferWithSegments_as_buffer = {
    (getbufferproc)BufferWithSegments_getbuffer, /* bf_getbuffer */
    0                                            /* bf_releasebuffer */
};
#endif

static PyMethodDef BufferWithSegments_methods[] = {
    {"segments", (PyCFunction)BufferWithSegments_segments, METH_NOARGS, NULL},
    {"tobytes", (PyCFunction)BufferWithSegments_tobytes, METH_NOARGS, NULL},
    {NULL, NULL}};

static PyMemberDef BufferWithSegments_members[] = {
    {"size", T_ULONGLONG, offsetof(ZstdBufferWithSegments, dataSize), READONLY,
     "total size of the buffer in bytes"},
    {NULL}};

PyType_Slot ZstdBufferWithSegmentsSlots[] = {
    {Py_tp_dealloc, BufferWithSegments_dealloc},
    {Py_sq_length, BufferWithSegments_length},
    {Py_sq_item, BufferWithSegments_item},
#if PY_VERSION_HEX >= 0x03090000
    {Py_bf_getbuffer, BufferWithSegments_getbuffer},
#endif
    {Py_tp_methods, BufferWithSegments_methods},
    {Py_tp_members, BufferWithSegments_members},
    {Py_tp_init, BufferWithSegments_init},
    {Py_tp_new, PyType_GenericNew},
    {0, NULL},
};

PyType_Spec ZstdBufferWithSegmentsSpec = {
    "zstd.BufferWithSegments",
    sizeof(ZstdBufferWithSegments),
    0,
    Py_TPFLAGS_DEFAULT,
    ZstdBufferWithSegmentsSlots,
};

PyTypeObject *ZstdBufferWithSegmentsType;

static void BufferSegments_dealloc(ZstdBufferSegments *self) {
    Py_CLEAR(self->parent);
    PyObject_Del(self);
}

static int BufferSegments_getbuffer(ZstdBufferSegments *self, Py_buffer *view,
                                    int flags) {
    return PyBuffer_FillInfo(view, (PyObject *)self, (void *)self->segments,
                             self->segmentCount * sizeof(BufferSegment), 1,
                             flags);
}

PyType_Slot ZstdBufferSegmentsSlots[] = {
    {Py_tp_dealloc, BufferSegments_dealloc},
#if PY_VERSION_HEX >= 0x03090000
    {Py_bf_getbuffer, BufferSegments_getbuffer},
#endif
    {Py_tp_new, PyType_GenericNew},
    {0, NULL},
};

PyType_Spec ZstdBufferSegmentsSpec = {
    "zstd.BufferSegments",
    sizeof(ZstdBufferSegments),
    0,
    Py_TPFLAGS_DEFAULT,
    ZstdBufferSegmentsSlots,
};

#if PY_VERSION_HEX < 0x03090000
static PyBufferProcs BufferSegments_as_buffer = {
    (getbufferproc)BufferSegments_getbuffer, 0};
#endif

PyTypeObject *ZstdBufferSegmentsType;

static void BufferSegment_dealloc(ZstdBufferSegment *self) {
    Py_CLEAR(self->parent);
    PyObject_Del(self);
}

static Py_ssize_t BufferSegment_length(ZstdBufferSegment *self) {
    return self->dataSize;
}

static int BufferSegment_getbuffer(ZstdBufferSegment *self, Py_buffer *view,
                                   int flags) {
    return PyBuffer_FillInfo(view, (PyObject *)self, self->data, self->dataSize,
                             1, flags);
}

static PyObject *BufferSegment_tobytes(ZstdBufferSegment *self) {
    return PyBytes_FromStringAndSize(self->data, self->dataSize);
}

#if PY_VERSION_HEX < 0x03090000
static PyBufferProcs BufferSegment_as_buffer = {
    (getbufferproc)BufferSegment_getbuffer, 0};
#endif

static PyMethodDef BufferSegment_methods[] = {
    {"tobytes", (PyCFunction)BufferSegment_tobytes, METH_NOARGS, NULL},
    {NULL, NULL}};

static PyMemberDef BufferSegment_members[] = {
    {"offset", T_ULONGLONG, offsetof(ZstdBufferSegment, offset), READONLY,
     "offset of segment within parent buffer"},
    {NULL}};

PyType_Slot ZstdBufferSegmentSlots[] = {
    {Py_tp_dealloc, BufferSegment_dealloc},
    {Py_sq_length, BufferSegment_length},
#if PY_VERSION_HEX >= 0x03090000
    {Py_bf_getbuffer, BufferSegment_getbuffer},
#endif
    {Py_tp_methods, BufferSegment_methods},
    {Py_tp_members, BufferSegment_members},
    {Py_tp_new, PyType_GenericNew},
    {0, NULL},
};

PyType_Spec ZstdBufferSegmentSpec = {
    "zstd.BufferSegment",
    sizeof(ZstdBufferSegment),
    0,
    Py_TPFLAGS_DEFAULT,
    ZstdBufferSegmentSlots,
};

PyTypeObject *ZstdBufferSegmentType;

static void
BufferWithSegmentsCollection_dealloc(ZstdBufferWithSegmentsCollection *self) {
    Py_ssize_t i;

    if (self->firstElements) {
        PyMem_Free(self->firstElements);
        self->firstElements = NULL;
    }

    if (self->buffers) {
        for (i = 0; i < self->bufferCount; i++) {
            Py_CLEAR(self->buffers[i]);
        }

        PyMem_Free(self->buffers);
        self->buffers = NULL;
    }

    PyObject_Del(self);
}

static int
BufferWithSegmentsCollection_init(ZstdBufferWithSegmentsCollection *self,
                                  PyObject *args) {
    Py_ssize_t size;
    Py_ssize_t i;
    Py_ssize_t offset = 0;

    size = PyTuple_Size(args);
    if (-1 == size) {
        return -1;
    }

    if (0 == size) {
        PyErr_SetString(PyExc_ValueError, "must pass at least 1 argument");
        return -1;
    }

    for (i = 0; i < size; i++) {
        PyObject *item = PyTuple_GET_ITEM(args, i);
        if (!PyObject_TypeCheck(item, ZstdBufferWithSegmentsType)) {
            PyErr_SetString(PyExc_TypeError,
                            "arguments must be BufferWithSegments instances");
            return -1;
        }

        if (0 == ((ZstdBufferWithSegments *)item)->segmentCount ||
            0 == ((ZstdBufferWithSegments *)item)->dataSize) {
            PyErr_SetString(PyExc_ValueError,
                            "ZstdBufferWithSegments cannot be empty");
            return -1;
        }
    }

    self->buffers = PyMem_Malloc(size * sizeof(ZstdBufferWithSegments *));
    if (NULL == self->buffers) {
        PyErr_NoMemory();
        return -1;
    }

    self->firstElements = PyMem_Malloc(size * sizeof(Py_ssize_t));
    if (NULL == self->firstElements) {
        PyMem_Free(self->buffers);
        self->buffers = NULL;
        PyErr_NoMemory();
        return -1;
    }

    self->bufferCount = size;

    for (i = 0; i < size; i++) {
        ZstdBufferWithSegments *item =
            (ZstdBufferWithSegments *)PyTuple_GET_ITEM(args, i);

        self->buffers[i] = item;
        Py_INCREF(item);

        if (i > 0) {
            self->firstElements[i - 1] = offset;
        }

        offset += item->segmentCount;
    }

    self->firstElements[size - 1] = offset;

    return 0;
}

static PyObject *
BufferWithSegmentsCollection_size(ZstdBufferWithSegmentsCollection *self) {
    Py_ssize_t i;
    Py_ssize_t j;
    unsigned long long size = 0;

    for (i = 0; i < self->bufferCount; i++) {
        for (j = 0; j < self->buffers[i]->segmentCount; j++) {
            size += self->buffers[i]->segments[j].length;
        }
    }

    return PyLong_FromUnsignedLongLong(size);
}

Py_ssize_t
BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection *self) {
    return self->firstElements[self->bufferCount - 1];
}

static ZstdBufferSegment *
BufferWithSegmentsCollection_item(ZstdBufferWithSegmentsCollection *self,
                                  Py_ssize_t i) {
    Py_ssize_t bufferOffset;

    if (i < 0) {
        PyErr_SetString(PyExc_IndexError, "offset must be non-negative");
        return NULL;
    }

    if (i >= BufferWithSegmentsCollection_length(self)) {
        PyErr_Format(PyExc_IndexError, "offset must be less than %zd",
                     BufferWithSegmentsCollection_length(self));
        return NULL;
    }

    for (bufferOffset = 0; bufferOffset < self->bufferCount; bufferOffset++) {
        Py_ssize_t offset = 0;

        if (i < self->firstElements[bufferOffset]) {
            if (bufferOffset > 0) {
                offset = self->firstElements[bufferOffset - 1];
            }

            return BufferWithSegments_item(self->buffers[bufferOffset],
                                           i - offset);
        }
    }

    PyErr_SetString(ZstdError,
                    "error resolving segment; this should not happen");
    return NULL;
}

static PyMethodDef BufferWithSegmentsCollection_methods[] = {
    {"size", (PyCFunction)BufferWithSegmentsCollection_size, METH_NOARGS,
     PyDoc_STR("total size in bytes of all segments")},
    {NULL, NULL}};

PyType_Slot ZstdBufferWithSegmentsCollectionSlots[] = {
    {Py_tp_dealloc, BufferWithSegmentsCollection_dealloc},
    {Py_sq_length, BufferWithSegmentsCollection_length},
    {Py_sq_item, BufferWithSegmentsCollection_item},
    {Py_tp_methods, BufferWithSegmentsCollection_methods},
    {Py_tp_init, BufferWithSegmentsCollection_init},
    {Py_tp_new, PyType_GenericNew},
    {0, NULL},
};

PyType_Spec ZstdBufferWithSegmentsCollectionSpec = {
    "zstd.BufferWithSegmentsCollection",
    sizeof(ZstdBufferWithSegmentsCollection),
    0,
    Py_TPFLAGS_DEFAULT,
    ZstdBufferWithSegmentsCollectionSlots,
};

PyTypeObject *ZstdBufferWithSegmentsCollectionType;

void bufferutil_module_init(PyObject *mod) {
    ZstdBufferWithSegmentsType =
        (PyTypeObject *)PyType_FromSpec(&ZstdBufferWithSegmentsSpec);
#if PY_VERSION_HEX < 0x03090000
    ZstdBufferWithSegmentsType->tp_as_buffer = &BufferWithSegments_as_buffer;
#endif
    if (PyType_Ready(ZstdBufferWithSegmentsType) < 0) {
        return;
    }

    Py_INCREF(ZstdBufferWithSegmentsType);
    PyModule_AddObject(mod, "BufferWithSegments",
                       (PyObject *)ZstdBufferWithSegmentsType);

    ZstdBufferSegmentsType =
        (PyTypeObject *)PyType_FromSpec(&ZstdBufferSegmentsSpec);
#if PY_VERSION_HEX < 0x03090000
    ZstdBufferSegmentsType->tp_as_buffer = &BufferSegments_as_buffer;
#endif
    if (PyType_Ready(ZstdBufferSegmentsType) < 0) {
        return;
    }

    Py_INCREF(ZstdBufferSegmentsType);
    PyModule_AddObject(mod, "BufferSegments",
                       (PyObject *)ZstdBufferSegmentsType);

    ZstdBufferSegmentType =
        (PyTypeObject *)PyType_FromSpec(&ZstdBufferSegmentSpec);
#if PY_VERSION_HEX < 0x03090000
    ZstdBufferSegmentType->tp_as_buffer = &BufferSegment_as_buffer;
#endif
    if (PyType_Ready(ZstdBufferSegmentType) < 0) {
        return;
    }

    Py_INCREF(ZstdBufferSegmentType);
    PyModule_AddObject(mod, "BufferSegment", (PyObject *)ZstdBufferSegmentType);

    ZstdBufferWithSegmentsCollectionType =
        (PyTypeObject *)PyType_FromSpec(&ZstdBufferWithSegmentsCollectionSpec);
    if (PyType_Ready(ZstdBufferWithSegmentsCollectionType) < 0) {
        return;
    }

    Py_INCREF(ZstdBufferWithSegmentsCollectionType);
    PyModule_AddObject(mod, "BufferWithSegmentsCollection",
                       (PyObject *)ZstdBufferWithSegmentsCollectionType);
}