libgrapheme

unicode string library
git clone git://git.suckless.org/libgrapheme
Log | Files | Refs | README | LICENSE

util.c (20333B)


      1 /* See LICENSE file for copyright and license details. */
      2 #include <ctype.h>
      3 #include <errno.h>
      4 #include <inttypes.h>
      5 #include <stdbool.h>
      6 #include <stddef.h>
      7 #include <stdint.h>
      8 #include <stdio.h>
      9 #include <stdlib.h>
     10 #include <string.h>
     11 
     12 #include "util.h"
     13 
     14 struct range {
     15 	uint_least32_t lower;
     16 	uint_least32_t upper;
     17 };
     18 
     19 struct properties_payload {
     20 	struct properties *prop;
     21 	const struct property_spec *spec;
     22 	uint_least8_t speclen;
     23 	int (*set_value)(struct properties_payload *, uint_least32_t,
     24 	                 int_least64_t);
     25 	uint_least8_t (*handle_conflict)(uint_least32_t, uint_least8_t,
     26 	                                 uint_least8_t);
     27 };
     28 
     29 struct break_test_payload {
     30 	struct break_test **test;
     31 	size_t *testlen;
     32 };
     33 
     34 static void *
     35 reallocate_array(void *p, size_t len, size_t size)
     36 {
     37 	if (len > 0 && size > SIZE_MAX / len) {
     38 		errno = ENOMEM;
     39 		return NULL;
     40 	}
     41 
     42 	return realloc(p, len * size);
     43 }
     44 
     45 int
     46 hextocp(const char *str, size_t len, uint_least32_t *cp)
     47 {
     48 	size_t i;
     49 	int off;
     50 	char relative;
     51 
     52 	/* the maximum valid codepoint is 0x10FFFF */
     53 	if (len > 6) {
     54 		fprintf(stderr, "hextocp: '%.*s' is too long.\n", (int)len,
     55 		        str);
     56 		return 1;
     57 	}
     58 
     59 	for (i = 0, *cp = 0; i < len; i++) {
     60 		if (str[i] >= '0' && str[i] <= '9') {
     61 			relative = '0';
     62 			off = 0;
     63 		} else if (str[i] >= 'a' && str[i] <= 'f') {
     64 			relative = 'a';
     65 			off = 10;
     66 		} else if (str[i] >= 'A' && str[i] <= 'F') {
     67 			relative = 'A';
     68 			off = 10;
     69 		} else {
     70 			fprintf(stderr, "hextocp: '%.*s' is not hexadecimal.\n",
     71 			        (int)len, str);
     72 			return 1;
     73 		}
     74 
     75 		*cp += ((uint_least32_t)1 << (4 * (len - i - 1))) *
     76 		       (uint_least32_t)(str[i] - relative + off);
     77 	}
     78 
     79 	if (*cp > UINT32_C(0x10FFFF)) {
     80 		fprintf(stderr, "hextocp: '%.*s' is too large.\n", (int)len,
     81 		        str);
     82 		return 1;
     83 	}
     84 
     85 	return 0;
     86 }
     87 
     88 int
     89 parse_cp_list(const char *str, uint_least32_t **cp, size_t *cplen)
     90 {
     91 	size_t count, i;
     92 	const char *tmp1 = NULL, *tmp2 = NULL;
     93 
     94 	if (strlen(str) == 0) {
     95 		*cp = NULL;
     96 		*cplen = 0;
     97 		return 0;
     98 	}
     99 
    100 	/* count the number of spaces in the string and infer list length */
    101 	for (count = 1, tmp1 = str; (tmp2 = strchr(tmp1, ' ')) != NULL;
    102 	     count++, tmp1 = tmp2 + 1) {
    103 		;
    104 	}
    105 
    106 	/* allocate resources */
    107 	if (!(*cp = calloc((*cplen = count), sizeof(**cp)))) {
    108 		fprintf(stderr, "calloc: %s\n", strerror(errno));
    109 		exit(1);
    110 	}
    111 
    112 	/* go through the string again, parsing the numbers */
    113 	for (i = 0, tmp1 = tmp2 = str; tmp2 != NULL; i++) {
    114 		tmp2 = strchr(tmp1, ' ');
    115 		if (hextocp(tmp1, tmp2 ? (size_t)(tmp2 - tmp1) : strlen(tmp1),
    116 		            &((*cp)[i]))) {
    117 			return 1;
    118 		}
    119 		if (tmp2 != NULL) {
    120 			tmp1 = tmp2 + 1;
    121 		}
    122 	}
    123 
    124 	return 0;
    125 }
    126 
    127 static int
    128 range_parse(const char *str, struct range *range)
    129 {
    130 	char *p;
    131 
    132 	if ((p = strstr(str, "..")) == NULL) {
    133 		/* input has the form "XXXXXX" */
    134 		if (hextocp(str, strlen(str), &range->lower)) {
    135 			return 1;
    136 		}
    137 		range->upper = range->lower;
    138 	} else {
    139 		/* input has the form "XXXXXX..XXXXXX" */
    140 		if (hextocp(str, (size_t)(p - str), &range->lower) ||
    141 		    hextocp(p + 2, strlen(p + 2), &range->upper)) {
    142 			return 1;
    143 		}
    144 	}
    145 
    146 	return 0;
    147 }
    148 
    149 void
    150 parse_file_with_callback(const char *fname,
    151                          int (*callback)(const char *, char **, size_t, char *,
    152                                          void *),
    153                          void *payload)
    154 {
    155 	FILE *fp;
    156 	char *line = NULL, **field = NULL, *comment;
    157 	size_t linebufsize = 0, i, fieldbufsize = 0, j, nfields;
    158 	ssize_t len;
    159 
    160 	/* open file */
    161 	if (!(fp = fopen(fname, "r"))) {
    162 		fprintf(stderr, "parse_file_with_callback: fopen '%s': %s.\n",
    163 		        fname, strerror(errno));
    164 		exit(1);
    165 	}
    166 
    167 	while ((len = getline(&line, &linebufsize, fp)) >= 0) {
    168 		/* remove trailing newline */
    169 		if (len > 0 && line[len - 1] == '\n') {
    170 			line[len - 1] = '\0';
    171 			len--;
    172 		}
    173 
    174 		/* skip empty lines and comment lines */
    175 		if (len == 0 || line[0] == '#') {
    176 			continue;
    177 		}
    178 
    179 		/* tokenize line into fields */
    180 		for (i = 0, nfields = 0, comment = NULL; i < (size_t)len; i++) {
    181 			/* skip leading whitespace */
    182 			while (line[i] == ' ') {
    183 				i++;
    184 			}
    185 
    186 			/* check if we crashed into the comment */
    187 			if (line[i] != '#') {
    188 				/* extend field buffer, if necessary */
    189 				if (++nfields > fieldbufsize) {
    190 					if ((field = realloc(
    191 						     field,
    192 						     nfields *
    193 							     sizeof(*field))) ==
    194 					    NULL) {
    195 						fprintf(stderr,
    196 						        "parse_file_with_"
    197 						        "callback: realloc: "
    198 						        "%s.\n",
    199 						        strerror(errno));
    200 						exit(1);
    201 					}
    202 					fieldbufsize = nfields;
    203 				}
    204 
    205 				/* set current position as field start */
    206 				field[nfields - 1] = &line[i];
    207 
    208 				/* continue until we reach ';' or '#' or end */
    209 				while (line[i] != ';' && line[i] != '#' &&
    210 				       line[i] != '\0') {
    211 					i++;
    212 				}
    213 			}
    214 
    215 			if (line[i] == '#') {
    216 				/* set comment-variable for later */
    217 				comment = &line[i + 1];
    218 			}
    219 
    220 			/* go back whitespace and terminate field there */
    221 			if (i > 0) {
    222 				for (j = i - 1; line[j] == ' '; j--) {
    223 					;
    224 				}
    225 				line[j + 1] = '\0';
    226 			} else {
    227 				line[i] = '\0';
    228 			}
    229 
    230 			/* if comment is set, we are done */
    231 			if (comment != NULL) {
    232 				break;
    233 			}
    234 		}
    235 
    236 		/* skip leading whitespace in comment */
    237 		while (comment != NULL && comment[0] == ' ') {
    238 			comment++;
    239 		}
    240 
    241 		/* call callback function */
    242 		if (callback(fname, field, nfields, comment, payload)) {
    243 			fprintf(stderr, "parse_file_with_callback: "
    244 			                "Malformed input.\n");
    245 			exit(1);
    246 		}
    247 	}
    248 
    249 	free(line);
    250 	free(field);
    251 }
    252 
    253 static int
    254 properties_callback(const char *file, char **field, size_t nfields,
    255                     char *comment, void *payload)
    256 {
    257 	/* prop always has the length 0x110000 */
    258 	struct properties_payload *p = (struct properties_payload *)payload;
    259 	struct range r;
    260 	uint_least8_t i;
    261 	uint_least32_t cp;
    262 
    263 	(void)comment;
    264 
    265 	if (nfields < 2) {
    266 		return 1;
    267 	}
    268 
    269 	for (i = 0; i < p->speclen; i++) {
    270 		/* identify fitting file and identifier */
    271 		if (p->spec[i].file && !strcmp(p->spec[i].file, file) &&
    272 		    (!strcmp(p->spec[i].ucdname, field[1]) ||
    273 		     (comment != NULL &&
    274 		      !strncmp(p->spec[i].ucdname, comment,
    275 		               strlen(p->spec[i].ucdname)) &&
    276 		      comment[strlen(p->spec[i].ucdname)] == ' '))) {
    277 			/* parse range in first field */
    278 			if (range_parse(field[0], &r)) {
    279 				return 1;
    280 			}
    281 
    282 			/* apply to all codepoints in the range */
    283 			for (cp = r.lower; cp <= r.upper; cp++) {
    284 				if (p->set_value(payload, cp, i)) {
    285 					exit(1);
    286 				}
    287 			}
    288 			break;
    289 		}
    290 	}
    291 
    292 	return 0;
    293 }
    294 
    295 void
    296 properties_compress(const struct properties *prop,
    297                     struct properties_compressed *comp)
    298 {
    299 	uint_least32_t cp, i;
    300 
    301 	/* initialization */
    302 	if (!(comp->offset = malloc((size_t)UINT32_C(0x110000) *
    303 	                            sizeof(*(comp->offset))))) {
    304 		fprintf(stderr, "malloc: %s\n", strerror(errno));
    305 		exit(1);
    306 	}
    307 	comp->data = NULL;
    308 	comp->datalen = 0;
    309 
    310 	for (cp = 0; cp < UINT32_C(0x110000); cp++) {
    311 		for (i = 0; i < comp->datalen; i++) {
    312 			if (!memcmp(&(prop[cp]), &(comp->data[i]),
    313 			            sizeof(*prop))) {
    314 				/* found a match! */
    315 				comp->offset[cp] = i;
    316 				break;
    317 			}
    318 		}
    319 		if (i == comp->datalen) {
    320 			/*
    321 			 * found no matching properties-struct, so
    322 			 * add current properties to data and add the
    323 			 * offset in the offset-table
    324 			 */
    325 			if (!(comp->data = reallocate_array(
    326 				      comp->data, ++(comp->datalen),
    327 				      sizeof(*(comp->data))))) {
    328 				fprintf(stderr, "reallocate_array: %s\n",
    329 				        strerror(errno));
    330 				exit(1);
    331 			}
    332 			memcpy(&(comp->data[comp->datalen - 1]), &(prop[cp]),
    333 			       sizeof(*prop));
    334 			comp->offset[cp] = comp->datalen - 1;
    335 		}
    336 	}
    337 }
    338 
    339 double
    340 properties_get_major_minor(const struct properties_compressed *comp,
    341                            struct properties_major_minor *mm)
    342 {
    343 	size_t i, j, compression_count = 0;
    344 
    345 	/*
    346 	 * we currently have an array comp->offset which maps the
    347 	 * codepoints 0..0x110000 to offsets into comp->data.
    348 	 * To improve cache-locality instead and allow a bit of
    349 	 * compressing, instead of directly mapping a codepoint
    350 	 * 0xAAAABB with comp->offset, we generate two arrays major
    351 	 * and minor such that
    352 	 *    comp->offset(0xAAAABB) == minor[major[0xAAAA] + 0xBB]
    353 	 * This yields a major-array of length 2^16 and a minor array
    354 	 * of variable length depending on how many common subsequences
    355 	 * can be filtered out.
    356 	 */
    357 
    358 	/* initialize */
    359 	if (!(mm->major = malloc((size_t)0x1100 * sizeof(*(mm->major))))) {
    360 		fprintf(stderr, "malloc: %s\n", strerror(errno));
    361 		exit(1);
    362 	}
    363 	mm->minor = NULL;
    364 	mm->minorlen = 0;
    365 
    366 	for (i = 0; i < (size_t)0x1100; i++) {
    367 		/*
    368 		 * we now look at the cp-range (i << 8)..(i << 8 + 0xFF)
    369 		 * and check if its corresponding offset-data already
    370 		 * exists in minor (because then we just point there
    371 		 * and need less storage)
    372 		 */
    373 		for (j = 0; j + 0xFF < mm->minorlen; j++) {
    374 			if (!memcmp(&(comp->offset[i << 8]), &(mm->minor[j]),
    375 			            sizeof(*(comp->offset)) * 0x100)) {
    376 				break;
    377 			}
    378 		}
    379 		if (j + 0xFF < mm->minorlen) {
    380 			/* found an index */
    381 			compression_count++;
    382 			mm->major[i] = j;
    383 		} else {
    384 			/*
    385 			 * add "new" sequence to minor and point to it
    386 			 * in major
    387 			 */
    388 			mm->minorlen += 0x100;
    389 			if (!(mm->minor =
    390 			              reallocate_array(mm->minor, mm->minorlen,
    391 			                               sizeof(*(mm->minor))))) {
    392 				fprintf(stderr, "reallocate_array: %s\n",
    393 				        strerror(errno));
    394 				exit(1);
    395 			}
    396 			memcpy(&(mm->minor[mm->minorlen - 0x100]),
    397 			       &(comp->offset[i << 8]),
    398 			       sizeof(*(mm->minor)) * 0x100);
    399 			mm->major[i] = mm->minorlen - 0x100;
    400 		}
    401 	}
    402 
    403 	/* return compression ratio */
    404 	return (double)compression_count / 0x1100 * 100;
    405 }
    406 
    407 void
    408 properties_print_lookup_table(const char *name, const size_t *data,
    409                               size_t datalen)
    410 {
    411 	const char *type;
    412 	size_t i, maxval;
    413 
    414 	for (i = 0, maxval = 0; i < datalen; i++) {
    415 		if (data[i] > maxval) {
    416 			maxval = data[i];
    417 		}
    418 	}
    419 
    420 	type = (maxval <= UINT_LEAST8_MAX)  ? "uint_least8_t" :
    421 	       (maxval <= UINT_LEAST16_MAX) ? "uint_least16_t" :
    422 	       (maxval <= UINT_LEAST32_MAX) ? "uint_least32_t" :
    423 	                                      "uint_least64_t";
    424 
    425 	printf("static const %s %s[] = {\n\t", type, name);
    426 	for (i = 0; i < datalen; i++) {
    427 		printf("%zu", data[i]);
    428 		if (i + 1 == datalen) {
    429 			printf("\n");
    430 		} else if ((i + 1) % 8 != 0) {
    431 			printf(", ");
    432 		} else {
    433 			printf(",\n\t");
    434 		}
    435 	}
    436 	printf("};\n");
    437 }
    438 
    439 void
    440 properties_print_derived_lookup_table(
    441 	char *name, size_t *offset, size_t offsetlen,
    442 	int_least64_t (*get_value)(const struct properties *, size_t),
    443 	const void *payload)
    444 {
    445 	const char *type;
    446 	size_t i;
    447 	int_least64_t minval, maxval;
    448 
    449 	for (i = 0, minval = INT_LEAST64_MAX, maxval = INT_LEAST64_MIN;
    450 	     i < offsetlen; i++) {
    451 		if (get_value(payload, offset[i]) > maxval) {
    452 			maxval = get_value(payload, offset[i]);
    453 		} else if (get_value(payload, offset[i]) < minval) {
    454 			minval = get_value(payload, offset[i]);
    455 		}
    456 	}
    457 
    458 	if (minval < 0) {
    459 		/* we need a signed type */
    460 		type = (minval >= INT_LEAST8_MIN && maxval <= INT_LEAST8_MAX) ?
    461 		               "int_least8_t" :
    462 		       (minval >= INT_LEAST16_MIN &&
    463 		        maxval <= INT_LEAST16_MAX) ?
    464 		               "int_least16_t" :
    465 		       (minval >= INT_LEAST32_MIN &&
    466 		        maxval <= INT_LEAST32_MAX) ?
    467 		               "int_least32_t" :
    468 		               "int_least64_t";
    469 	} else {
    470 		/* we are fine with an unsigned type */
    471 		type = (maxval <= UINT_LEAST8_MAX)  ? "uint_least8_t" :
    472 		       (maxval <= UINT_LEAST16_MAX) ? "uint_least16_t" :
    473 		       (maxval <= UINT_LEAST32_MAX) ? "uint_least32_t" :
    474 		                                      "uint_least64_t";
    475 	}
    476 
    477 	printf("static const %s %s[] = {\n\t", type, name);
    478 	for (i = 0; i < offsetlen; i++) {
    479 		printf("%" PRIiLEAST64, get_value(payload, offset[i]));
    480 		if (i + 1 == offsetlen) {
    481 			printf("\n");
    482 		} else if ((i + 1) % 8 != 0) {
    483 			printf(", ");
    484 		} else {
    485 			printf(",\n\t");
    486 		}
    487 	}
    488 	printf("};\n");
    489 }
    490 
    491 static void
    492 properties_print_enum(const struct property_spec *spec, size_t speclen,
    493                       const char *enumname, const char *enumprefix)
    494 {
    495 	size_t i;
    496 
    497 	printf("enum %s {\n", enumname);
    498 	for (i = 0; i < speclen; i++) {
    499 		printf("\t%s_%s,\n", enumprefix, spec[i].enumname);
    500 	}
    501 	printf("\tNUM_%sS,\n};\n\n", enumprefix);
    502 }
    503 
    504 static int
    505 set_value_bp(struct properties_payload *payload, uint_least32_t cp,
    506              int_least64_t value)
    507 {
    508 	if (payload->prop[cp].property != payload->speclen) {
    509 		if (payload->handle_conflict == NULL) {
    510 			fprintf(stderr,
    511 			        "set_value_bp: "
    512 			        "Unhandled character break property "
    513 			        "overwrite for 0x%06X (%s <- %s).\n",
    514 			        cp,
    515 			        payload->spec[payload->prop[cp].property]
    516 			                .enumname,
    517 			        payload->spec[value].enumname);
    518 			return 1;
    519 		} else {
    520 			value = payload->handle_conflict(
    521 				cp, (uint_least8_t)payload->prop[cp].property,
    522 				(uint_least8_t)value);
    523 		}
    524 	}
    525 	payload->prop[cp].property = value;
    526 
    527 	return 0;
    528 }
    529 
    530 static int_least64_t
    531 get_value_bp(const struct properties *prop, size_t offset)
    532 {
    533 	return prop[offset].property;
    534 }
    535 
    536 void
    537 properties_generate_break_property(
    538 	const struct property_spec *spec, uint_least8_t speclen,
    539 	uint_least8_t (*fill_missing)(uint_least32_t),
    540 	uint_least8_t (*handle_conflict)(uint_least32_t, uint_least8_t,
    541                                          uint_least8_t),
    542 	void (*post_process)(struct properties *), const char *prefix,
    543 	const char *argv0)
    544 {
    545 	struct properties_compressed comp;
    546 	struct properties_major_minor mm;
    547 	struct properties_payload payload;
    548 	struct properties *prop;
    549 	size_t i, j, prefixlen = strlen(prefix);
    550 	char buf1[64], prefix_uc[64], buf2[64], buf3[64], buf4[64];
    551 
    552 	/*
    553 	 * allocate property buffer for all 0x110000 codepoints and
    554 	 * initialize its entries to the known invalid value "speclen"
    555 	 */
    556 	if (!(prop = calloc(UINT32_C(0x110000), sizeof(*prop)))) {
    557 		fprintf(stderr, "calloc: %s\n", strerror(errno));
    558 		exit(1);
    559 	}
    560 	for (i = 0; i < UINT32_C(0x110000); i++) {
    561 		prop[i].property = speclen;
    562 	}
    563 
    564 	/* generate data */
    565 	payload.prop = prop;
    566 	payload.spec = spec;
    567 	payload.speclen = speclen;
    568 	payload.set_value = set_value_bp;
    569 	payload.handle_conflict = handle_conflict;
    570 
    571 	/* parse each file exactly once and ignore NULL-fields */
    572 	for (i = 0; i < speclen; i++) {
    573 		for (j = 0; j < i; j++) {
    574 			if (spec[i].file && spec[j].file &&
    575 			    !strcmp(spec[i].file, spec[j].file)) {
    576 				/* file has already been parsed */
    577 				break;
    578 			}
    579 		}
    580 		if (i == j && spec[i].file) {
    581 			/* file has not been processed yet */
    582 			parse_file_with_callback(spec[i].file,
    583 			                         properties_callback, &payload);
    584 		}
    585 	}
    586 
    587 	/* fill in the missing properties that weren't explicitly given */
    588 	for (i = 0; i < UINT32_C(0x110000); i++) {
    589 		if (payload.prop[i].property == speclen) {
    590 			if (fill_missing != NULL) {
    591 				payload.prop[i].property =
    592 					fill_missing((uint_least32_t)i);
    593 			} else {
    594 				payload.prop[i].property = 0;
    595 			}
    596 		}
    597 	}
    598 
    599 	/* post-processing */
    600 	if (post_process != NULL) {
    601 		post_process(payload.prop);
    602 	}
    603 
    604 	/* compress data */
    605 	printf("/* Automatically generated by %s */\n#include <stdint.h>\n\n",
    606 	       argv0);
    607 	properties_compress(prop, &comp);
    608 
    609 	fprintf(stderr, "%s: %s-LUT compression-ratio: %.2f%%\n", argv0, prefix,
    610 	        properties_get_major_minor(&comp, &mm));
    611 
    612 	/* prepare names */
    613 	if ((size_t)snprintf(buf1, LEN(buf1), "%s_property", prefix) >=
    614 	    LEN(buf1)) {
    615 		fprintf(stderr, "snprintf: String truncated.\n");
    616 		exit(1);
    617 	}
    618 	if (LEN(prefix_uc) + 1 < prefixlen) {
    619 		fprintf(stderr, "snprintf: Buffer too small.\n");
    620 		exit(1);
    621 	}
    622 	for (i = 0; i < prefixlen; i++) {
    623 		prefix_uc[i] = (char)toupper(prefix[i]);
    624 	}
    625 	prefix_uc[prefixlen] = '\0';
    626 	if ((size_t)snprintf(buf2, LEN(buf2), "%s_PROP", prefix_uc) >=
    627 	            LEN(buf2) ||
    628 	    (size_t)snprintf(buf3, LEN(buf3), "%s_major", prefix) >=
    629 	            LEN(buf3) ||
    630 	    (size_t)snprintf(buf4, LEN(buf4), "%s_minor", prefix) >=
    631 	            LEN(buf4)) {
    632 		fprintf(stderr, "snprintf: String truncated.\n");
    633 		exit(1);
    634 	}
    635 
    636 	/* print data */
    637 	properties_print_enum(spec, speclen, buf1, buf2);
    638 	properties_print_lookup_table(buf3, mm.major, 0x1100);
    639 	printf("\n");
    640 	properties_print_derived_lookup_table(buf4, mm.minor, mm.minorlen,
    641 	                                      get_value_bp, comp.data);
    642 
    643 	/* free data */
    644 	free(prop);
    645 	free(comp.data);
    646 	free(comp.offset);
    647 	free(mm.major);
    648 	free(mm.minor);
    649 }
    650 
    651 static int
    652 break_test_callback(const char *fname, char **field, size_t nfields,
    653                     char *comment, void *payload)
    654 {
    655 	struct break_test *t,
    656 		**test = ((struct break_test_payload *)payload)->test;
    657 	size_t i, *testlen = ((struct break_test_payload *)payload)->testlen;
    658 	char *token;
    659 
    660 	(void)fname;
    661 
    662 	if (nfields < 1) {
    663 		return 1;
    664 	}
    665 
    666 	/* append new testcase and initialize with zeroes */
    667 	if ((*test = realloc(*test, ++(*testlen) * sizeof(**test))) == NULL) {
    668 		fprintf(stderr, "break_test_callback: realloc: %s.\n",
    669 		        strerror(errno));
    670 		return 1;
    671 	}
    672 	t = &(*test)[*testlen - 1];
    673 	memset(t, 0, sizeof(*t));
    674 
    675 	/* parse testcase "<÷|×> <cp> <÷|×> ... <cp> <÷|×>" */
    676 	for (token = strtok(field[0], " "), i = 0; token != NULL;
    677 	     i++, token = strtok(NULL, " ")) {
    678 		if (i % 2 == 0) {
    679 			/* delimiter or start of sequence */
    680 			if (i == 0 ||
    681 			    !strncmp(token, "\xC3\xB7", 2)) { /* UTF-8 */
    682 				/*
    683 				 * '÷' indicates a breakpoint,
    684 				 * the current length is done; allocate
    685 				 * a new length field and set it to 0
    686 				 */
    687 				if ((t->len = realloc(
    688 					     t->len,
    689 					     ++t->lenlen * sizeof(*t->len))) ==
    690 				    NULL) {
    691 					fprintf(stderr,
    692 					        "break_test_"
    693 					        "callback: realloc: %s.\n",
    694 					        strerror(errno));
    695 					return 1;
    696 				}
    697 				t->len[t->lenlen - 1] = 0;
    698 			} else if (!strncmp(token, "\xC3\x97", 2)) { /* UTF-8 */
    699 				/* '×' indicates a non-breakpoint, do nothing */
    700 			} else {
    701 				fprintf(stderr,
    702 				        "break_test_callback: "
    703 				        "Malformed delimiter '%s'.\n",
    704 				        token);
    705 				return 1;
    706 			}
    707 		} else {
    708 			/* add codepoint to cp-array */
    709 			if ((t->cp = realloc(t->cp,
    710 			                     ++t->cplen * sizeof(*t->cp))) ==
    711 			    NULL) {
    712 				fprintf(stderr,
    713 				        "break_test_callback: "
    714 				        "realloc: %s.\n",
    715 				        strerror(errno));
    716 				return 1;
    717 			}
    718 			if (hextocp(token, strlen(token),
    719 			            &t->cp[t->cplen - 1])) {
    720 				return 1;
    721 			}
    722 			if (t->lenlen > 0) {
    723 				t->len[t->lenlen - 1]++;
    724 			}
    725 		}
    726 	}
    727 	if (t->lenlen > 0 && t->len[t->lenlen - 1] == 0) {
    728 		/*
    729 		 * we allocated one more length than we needed because
    730 		 * the breakpoint was at the end
    731 		 */
    732 		t->lenlen--;
    733 	}
    734 
    735 	/* store comment */
    736 	if (comment != NULL &&
    737 	    ((*test)[*testlen - 1].descr = strdup(comment)) == NULL) {
    738 		fprintf(stderr, "break_test_callback: strdup: %s.\n",
    739 		        strerror(errno));
    740 		return 1;
    741 	}
    742 
    743 	return 0;
    744 }
    745 
    746 void
    747 break_test_list_parse(char *fname, struct break_test **test, size_t *testlen)
    748 {
    749 	struct break_test_payload pl = {
    750 		.test = test,
    751 		.testlen = testlen,
    752 	};
    753 	*test = NULL;
    754 	*testlen = 0;
    755 
    756 	parse_file_with_callback(fname, break_test_callback, &pl);
    757 }
    758 
    759 void
    760 break_test_list_print(const struct break_test *test, size_t testlen,
    761                       const char *identifier, const char *progname)
    762 {
    763 	size_t i, j;
    764 
    765 	printf("/* Automatically generated by %s */\n"
    766 	       "#include <stdint.h>\n#include <stddef.h>\n\n"
    767 	       "#include \"../gen/types.h\"\n\n",
    768 	       progname);
    769 
    770 	printf("static const struct break_test %s[] = {\n", identifier);
    771 	for (i = 0; i < testlen; i++) {
    772 		printf("\t{\n");
    773 
    774 		printf("\t\t.cp     = (uint_least32_t[]){");
    775 		for (j = 0; j < test[i].cplen; j++) {
    776 			printf(" UINT32_C(0x%06X)", test[i].cp[j]);
    777 			if (j + 1 < test[i].cplen) {
    778 				putchar(',');
    779 			}
    780 		}
    781 		printf(" },\n");
    782 		printf("\t\t.cplen  = %zu,\n", test[i].cplen);
    783 
    784 		printf("\t\t.len    = (size_t[]){");
    785 		for (j = 0; j < test[i].lenlen; j++) {
    786 			printf(" %zu", test[i].len[j]);
    787 			if (j + 1 < test[i].lenlen) {
    788 				putchar(',');
    789 			}
    790 		}
    791 		printf(" },\n");
    792 		printf("\t\t.lenlen = %zu,\n", test[i].lenlen);
    793 
    794 		printf("\t\t.descr  = \"%s\",\n", test[i].descr);
    795 
    796 		printf("\t},\n");
    797 	}
    798 	printf("};\n");
    799 }
    800 
    801 void
    802 break_test_list_free(struct break_test *test, size_t testlen)
    803 {
    804 	size_t i;
    805 
    806 	for (i = 0; i < testlen; i++) {
    807 		free(test[i].cp);
    808 		free(test[i].len);
    809 		free(test[i].descr);
    810 	}
    811 
    812 	free(test);
    813 }