More work on the scanner and tests for it

FossilOrigin-Name: 037c88ce86e1ba3a399c03a27d4c7252b52c2f500bc13f449e19e497b0607880

1 files changed, 52 insertions, 2 deletions

diff --git a/src/lib/string.c b/src/lib/string.c
index f6fa49d..8499385 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -47,7 +47,7 @@ invalid:
 			charcount++;
 			switch(seqlen){
 			case 0:
-				if(NEVER(c > 0x7F))
+				if(NEVER(c == 0) || NEVER(c > 0x7F))
 					goto invalid;
 				break;
 			case 1:
@@ -73,10 +73,60 @@ invalid:
 	if(string){
 		string->bytes = bytecount;
 		string->chars = charcount;
-		string->utf8 = ((uint8_t *)string)+sizeof(*string);
 		memcpy(string->utf8, utf8, bytecount);
 	}else
 		aplwc->last_error = APLWC_OUT_OF_MEMORY;
 
 	return string;
 }
+
+uint32_t
+string_char(struct string *string, size_t offset)
+{
+	uint32_t c;
+	size_t len, i;
+
+	c = 0;
+	len = char_length(string, offset);
+	switch(char_length(string, offset)){
+	case 1:
+		c |= string->utf8[offset] & 0x7F;
+		break;
+	case 2:
+		c |= string->utf8[offset] & 0x1F;
+		break;
+	case 3:
+		c |= string->utf8[offset] & 0x0F;
+		break;
+	case 4:
+		c |= string->utf8[offset] & 0x07;
+		break;
+	}
+
+	for(i = 1; i < len; i++){
+		c <<= 6;
+		c |= string->utf8[offset] & 0x3F;
+	}
+
+	return c;
+}
+
+size_t
+char_length(struct string *string, size_t offset)
+{
+	uint8_t b;
+
+	if(offset >= string->bytes)
+		return 0;
+	b = string->utf8[offset];
+	assert(!((b & 0xC0) == 0x80));
+
+	if((b & 0x80) == 0)
+		return 1;
+	else if((b & 0xE0) == 0xC0)
+		return 2;
+	else if((b & 0xF0) == 0xE0)
+		return 3;
+	else
+		return 4;
+}