diff options
Diffstat (limited to 'circuitpython/py/objstrunicode.c')
| -rw-r--r-- | circuitpython/py/objstrunicode.c | 350 |
1 files changed, 350 insertions, 0 deletions
diff --git a/circuitpython/py/objstrunicode.c b/circuitpython/py/objstrunicode.c new file mode 100644 index 0000000..eb79d54 --- /dev/null +++ b/circuitpython/py/objstrunicode.c @@ -0,0 +1,350 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * SPDX-FileCopyrightText: Copyright (c) 2014-2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include <string.h> +#include <assert.h> + +#include "py/objstr.h" +#include "py/objlist.h" +#include "py/runtime.h" + +#include "supervisor/shared/translate.h" + +#if MICROPY_PY_BUILTINS_STR_UNICODE + +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf); + +/******************************************************************************/ +/* str */ + + +// These settings approximate CPython's printability. It is not +// exhaustive and may print "unprintable" characters. All ASCII control codes +// are escaped along with variable space widths and paragraph designators. +// Unlike CPython, we do not escape private use codes or reserved characters. +// We assume that the unicode is well formed. +// CPython policy is documented here: https://github.com/python/cpython/blob/bb3e0c240bc60fe08d332ff5955d54197f79751c/Objects/unicodectype.c#L147-L159 +STATIC void uni_print_quoted(const mp_print_t *print, const byte *str_data, uint str_len) { + // this escapes characters, but it will be very slow to print (calling print many times) + bool has_single_quote = false; + bool has_double_quote = false; + for (const byte *s = str_data, *top = str_data + str_len; !has_double_quote && s < top; s++) { + if (*s == '\'') { + has_single_quote = true; + } else if (*s == '"') { + has_double_quote = true; + } + } + unichar quote_char = '\''; + if (has_single_quote && !has_double_quote) { + quote_char = '"'; + } + mp_printf(print, "%c", quote_char); + const byte *s = str_data, *top = str_data + str_len; + while (s < top) { + unichar ch; + ch = utf8_get_char(s); + const byte *start = s; + s = utf8_next_char(s); + if (ch == quote_char) { + mp_printf(print, "\\%c", quote_char); + } else if (ch == '\\') { + mp_print_str(print, "\\\\"); + } else if (ch == '\n') { + mp_print_str(print, "\\n"); + } else if (ch == '\r') { + mp_print_str(print, "\\r"); + } else if (ch == '\t') { + mp_print_str(print, "\\t"); + } else if (ch <= 0x1f || (0x7f <= ch && ch <= 0xa0) || ch == 0xad) { + mp_printf(print, "\\x%02x", ch); + } else if ((0x2000 <= ch && ch <= 0x200f) || ch == 0x2028 || ch == 0x2029) { + mp_printf(print, "\\u%04x", ch); + } else { + // Print the full character out. + int width = s - start; + mp_print_strn(print, (const char *)start, width, 0, ' ', width); + } + } + mp_printf(print, "%c", quote_char); +} + +STATIC void uni_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { + GET_STR_DATA_LEN(self_in, str_data, str_len); + #if MICROPY_PY_UJSON + if (kind == PRINT_JSON) { + mp_str_print_json(print, str_data, str_len); + return; + } + #endif + if (kind == PRINT_STR) { + print->print_strn(print->data, (const char *)str_data, str_len); + } else { + uni_print_quoted(print, str_data, str_len); + } +} + +STATIC mp_obj_t uni_unary_op(mp_unary_op_t op, mp_obj_t self_in) { + GET_STR_DATA_LEN(self_in, str_data, str_len); + switch (op) { + case MP_UNARY_OP_BOOL: + return mp_obj_new_bool(str_len != 0); + case MP_UNARY_OP_LEN: + return MP_OBJ_NEW_SMALL_INT(utf8_charlen(str_data, str_len)); + default: + return MP_OBJ_NULL; // op not supported + } +} + +size_t str_offset_to_index(const mp_obj_type_t *type, const byte *self_data, size_t self_len, + size_t offset) { + if (offset > self_len) { + mp_raise_ValueError(MP_ERROR_TEXT("offset out of bounds")); + } + + if (type == &mp_type_bytes) { + return offset; + } + + size_t index_val = 0; + const byte *s = self_data; + for (size_t i = 0; i < offset; i++, s++) { + if (!UTF8_IS_CONT(*s)) { + ++index_val; + } + } + return index_val; +} + +// Convert an index into a pointer to its lead byte. Out of bounds indexing will raise IndexError or +// be capped to the first/last character of the string, depending on is_slice. +const byte *str_index_to_ptr(const mp_obj_type_t *type, const byte *self_data, size_t self_len, + mp_obj_t index, bool is_slice) { + // All str functions also handle bytes objects, and they call str_index_to_ptr(), + // so it must handle bytes. + if (type == &mp_type_bytes) { + // Taken from objstr.c:str_index_to_ptr() + size_t index_val = mp_get_index(type, self_len, index, is_slice); + return self_data + index_val; + } + + mp_int_t i; + // Copied from mp_get_index; I don't want bounds checking, just give me + // the integer as-is. (I can't bounds-check without scanning the whole + // string; an out-of-bounds index will be caught in the loops below.) + if (mp_obj_is_small_int(index)) { + i = MP_OBJ_SMALL_INT_VALUE(index); + } else if (!mp_obj_get_int_maybe(index, &i)) { + mp_raise_TypeError_varg(MP_ERROR_TEXT("string indices must be integers, not %q"), mp_obj_get_type_qstr(index)); + } + const byte *s, *top = self_data + self_len; + if (i < 0) { + // Negative indexing is performed by counting from the end of the string. + for (s = top - 1; i; --s) { + if (s < self_data) { + if (is_slice) { + return self_data; + } + mp_raise_IndexError_varg(MP_ERROR_TEXT("%q index out of range"), MP_QSTR_str); + } + if (!UTF8_IS_CONT(*s)) { + ++i; + } + } + ++s; + } else { + // Positive indexing, correspondingly, counts from the start of the string. + // It's assumed that negative indexing will generally be used with small + // absolute values (eg str[-1], not str[-1000000]), which means it'll be + // more efficient this way. + s = self_data; + while (1) { + // First check out-of-bounds + if (s >= top) { + if (is_slice) { + return top; + } + mp_raise_IndexError_varg(MP_ERROR_TEXT("%q index out of range"), MP_QSTR_str); + } + // Then check completion + if (i-- == 0) { + break; + } + // Then skip UTF-8 char + ++s; + while (UTF8_IS_CONT(*s)) { + ++s; + } + } + } + return s; +} + +STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { + const mp_obj_type_t *type = mp_obj_get_type(self_in); + assert(type == &mp_type_str); + GET_STR_DATA_LEN(self_in, self_data, self_len); + if (value == MP_OBJ_SENTINEL) { + // load + #if MICROPY_PY_BUILTINS_SLICE + if (mp_obj_is_type(index, &mp_type_slice)) { + mp_obj_t ostart, ostop, ostep; + mp_obj_slice_t *slice = MP_OBJ_TO_PTR(index); + ostart = slice->start; + ostop = slice->stop; + ostep = slice->step; + + if (ostep != mp_const_none && ostep != MP_OBJ_NEW_SMALL_INT(1)) { + mp_raise_NotImplementedError(MP_ERROR_TEXT("only slices with step=1 (aka None) are supported")); + } + + const byte *pstart, *pstop; + if (ostart != mp_const_none) { + pstart = str_index_to_ptr(type, self_data, self_len, ostart, true); + } else { + pstart = self_data; + } + if (ostop != mp_const_none) { + // pstop will point just after the stop character. This depends on + // the \0 at the end of the string. + pstop = str_index_to_ptr(type, self_data, self_len, ostop, true); + } else { + pstop = self_data + self_len; + } + if (pstop < pstart) { + return MP_OBJ_NEW_QSTR(MP_QSTR_); + } + return mp_obj_new_str_of_type(type, (const byte *)pstart, pstop - pstart); + } + #endif + const byte *s = str_index_to_ptr(type, self_data, self_len, index, false); + int len = 1; + if (UTF8_IS_NONASCII(*s)) { + // Count the number of 1 bits (after the first) + for (char mask = 0x40; *s & mask; mask >>= 1) { + ++len; + } + } + return mp_obj_new_str_via_qstr((const char *)s, len); // This will create a one-character string + } else { + return MP_OBJ_NULL; // op not supported + } +} + +STATIC const mp_rom_map_elem_t struni_locals_dict_table[] = { + #if MICROPY_CPYTHON_COMPAT + { MP_ROM_QSTR(MP_QSTR_encode), MP_ROM_PTR(&str_encode_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_find), MP_ROM_PTR(&str_find_obj) }, + { MP_ROM_QSTR(MP_QSTR_rfind), MP_ROM_PTR(&str_rfind_obj) }, + { MP_ROM_QSTR(MP_QSTR_index), MP_ROM_PTR(&str_index_obj) }, + { MP_ROM_QSTR(MP_QSTR_rindex), MP_ROM_PTR(&str_rindex_obj) }, + { MP_ROM_QSTR(MP_QSTR_join), MP_ROM_PTR(&str_join_obj) }, + { MP_ROM_QSTR(MP_QSTR_split), MP_ROM_PTR(&str_split_obj) }, + #if MICROPY_PY_BUILTINS_STR_SPLITLINES + { MP_ROM_QSTR(MP_QSTR_splitlines), MP_ROM_PTR(&str_splitlines_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_rsplit), MP_ROM_PTR(&str_rsplit_obj) }, + { MP_ROM_QSTR(MP_QSTR_startswith), MP_ROM_PTR(&str_startswith_obj) }, + { MP_ROM_QSTR(MP_QSTR_endswith), MP_ROM_PTR(&str_endswith_obj) }, + { MP_ROM_QSTR(MP_QSTR_strip), MP_ROM_PTR(&str_strip_obj) }, + { MP_ROM_QSTR(MP_QSTR_lstrip), MP_ROM_PTR(&str_lstrip_obj) }, + { MP_ROM_QSTR(MP_QSTR_rstrip), MP_ROM_PTR(&str_rstrip_obj) }, + { MP_ROM_QSTR(MP_QSTR_format), MP_ROM_PTR(&str_format_obj) }, + { MP_ROM_QSTR(MP_QSTR_replace), MP_ROM_PTR(&str_replace_obj) }, + #if MICROPY_PY_BUILTINS_STR_COUNT + { MP_ROM_QSTR(MP_QSTR_count), MP_ROM_PTR(&str_count_obj) }, + #endif + #if MICROPY_PY_BUILTINS_STR_PARTITION + { MP_ROM_QSTR(MP_QSTR_partition), MP_ROM_PTR(&str_partition_obj) }, + { MP_ROM_QSTR(MP_QSTR_rpartition), MP_ROM_PTR(&str_rpartition_obj) }, + #endif + #if MICROPY_PY_BUILTINS_STR_CENTER + { MP_ROM_QSTR(MP_QSTR_center), MP_ROM_PTR(&str_center_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_lower), MP_ROM_PTR(&str_lower_obj) }, + { MP_ROM_QSTR(MP_QSTR_upper), MP_ROM_PTR(&str_upper_obj) }, + { MP_ROM_QSTR(MP_QSTR_isspace), MP_ROM_PTR(&str_isspace_obj) }, + { MP_ROM_QSTR(MP_QSTR_isalpha), MP_ROM_PTR(&str_isalpha_obj) }, + { MP_ROM_QSTR(MP_QSTR_isdigit), MP_ROM_PTR(&str_isdigit_obj) }, + { MP_ROM_QSTR(MP_QSTR_isupper), MP_ROM_PTR(&str_isupper_obj) }, + { MP_ROM_QSTR(MP_QSTR_islower), MP_ROM_PTR(&str_islower_obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(struni_locals_dict, struni_locals_dict_table); + +const mp_obj_type_t mp_type_str = { + { &mp_type_type }, + .flags = MP_TYPE_FLAG_EXTENDED, + .name = MP_QSTR_str, + .print = uni_print, + .make_new = mp_obj_str_make_new, + .locals_dict = (mp_obj_dict_t *)&struni_locals_dict, + MP_TYPE_EXTENDED_FIELDS( + .unary_op = uni_unary_op, + .binary_op = mp_obj_str_binary_op, + .subscr = str_subscr, + .getiter = mp_obj_new_str_iterator, + .buffer_p = { .get_buffer = mp_obj_str_get_buffer }, + ), +}; + +/******************************************************************************/ +/* str iterator */ + +typedef struct _mp_obj_str_it_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + mp_obj_t str; + size_t cur; +} mp_obj_str_it_t; + +STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { + mp_obj_str_it_t *self = MP_OBJ_TO_PTR(self_in); + GET_STR_DATA_LEN(self->str, str, len); + if (self->cur < len) { + const byte *cur = str + self->cur; + const byte *end = utf8_next_char(str + self->cur); + mp_obj_t o_out = mp_obj_new_str_via_qstr((const char *)cur, end - cur); + self->cur += end - cur; + return o_out; + } else { + return MP_OBJ_STOP_ITERATION; + } +} + +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_str_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_str_it_t *o = (mp_obj_str_it_t *)iter_buf; + o->base.type = &mp_type_polymorph_iter; + o->iternext = str_it_iternext; + o->str = str; + o->cur = 0; + return MP_OBJ_FROM_PTR(o); +} + +#endif // MICROPY_PY_BUILTINS_STR_UNICODE |