Start working on the scanner.

For now, all that is implemented is some functionality which imports UTF-8 into the
memory managed by aplwc, after checking that the data is in fact valid UTF-8.

FossilOrigin-Name: c861e88777d8180cb7b37c11fc7016a900f9d5821a7e3c8e8aab54118d2fca94

1 files changed, 82 insertions, 0 deletions

diff --git a/src/lib/string.c b/src/lib/string.c
new file mode 100644
index 0000000..f6fa49d
--- /dev/null
+++ b/src/lib/string.c
@@ -0,0 +1,82 @@
+#include "aplwc_internal.h"
+
+struct string *
+chars_to_string(struct aplwc *aplwc, const char *chars)
+{
+	const uint8_t *utf8;
+	size_t bytecount, charcount;
+	struct string *string;
+	uint32_t c;
+	uint8_t b;
+	int seqlen, extra;
+
+	utf8 = (const uint8_t *)chars;
+	seqlen = extra = 0;
+	for(bytecount = 0, charcount = 0; utf8[bytecount]; bytecount++){
+		b = utf8[bytecount];
+		if(extra == 0){
+			c = 0;
+			if((b & 0x80) == 0){
+				seqlen = 0;
+				c |= (b & 0x7F);
+			}else if((b & 0xE0) == 0xC0){
+				seqlen = 1;
+				c |= (b & 0x1F);
+			}else if((b & 0xF0) == 0xE0){
+				seqlen = 2;
+				c |= (b & 0x0F);
+			}else if((b & 0xF8) == 0xF0){
+				seqlen = 3;
+				c |= (b & 0x7);
+			}else{
+invalid:
+				aplwc->last_error = APLWC_INVALID_UTF8;
+				return NULL;
+			}
+			extra = seqlen;
+		}else{
+			extra--;
+			c <<= 6;
+			if((b & 0xC0) == 0x80)
+				c |= (b & 0x3F);
+			else
+				goto invalid;	
+		}
+
+		if(extra == 0){
+			charcount++;
+			switch(seqlen){
+			case 0:
+				if(NEVER(c > 0x7F))
+					goto invalid;
+				break;
+			case 1:
+				if((c < 0x80) || NEVER(c > 0x07FF))
+					goto invalid;
+				break;
+			case 2:
+				if((c < 0x0800) || NEVER(c > 0xFFFF))
+					goto invalid;
+				break;
+			case 3:
+				if((c < 0x010000) || NEVER(c > 0x10FFFF))
+					goto invalid;
+				break;
+			}
+		}
+	}
+	
+	if(extra != 0)
+		goto invalid;
+
+	string = memory_alloc_string(&aplwc->memory, bytecount);
+	if(string){
+		string->bytes = bytecount;
+		string->chars = charcount;
+		string->utf8 = ((uint8_t *)string)+sizeof(*string);
+		memcpy(string->utf8, utf8, bytecount);
+	}else
+		aplwc->last_error = APLWC_OUT_OF_MEMORY;
+
+	return string;
+}