////////////////////////////////////////////////////////////////////////////////
// BGCPString implementation
// A class that stores a string and knows its codepage
// Copyright Bjoern Ganster
// Developed since 2011
////////////////////////////////////////////////////////////////////////////////

#include "BGCPString.h"
#include <libinstrumentation.h>
#include "MemoryBlock.h" // used for debugging via dump()

#include <assert.h>

#include <iostream>
#include <cmath>
#include <vector>

#ifndef _WIN32
#include <iconv.h>
#endif

#define FILE_NUMBER FILE_NO_BGCPSTRING_CPP

const BGCPString BGCPString::EOL = "\n";

size_t lengthIfOK(size_t len, size_t maxLen)
{
	if (len <= maxLen) {
		return len;
	} else {
		return 0;
	}
}

// Return number of bytes used by one character - code pages 932 to 950 still need testing
// A return value of 0 indicates an invalid encoding
size_t getCharSize(const void* chars, WORD cp, size_t maxLen)
{
	if (chars == nullptr || maxLen < 1) {
		return 0;
	}

	SET_SCOPE;
	const unsigned char* cptr = ((const unsigned char*)(chars));
	const char16_t* wptr = ((const char16_t*)(chars));

	switch (cp) {
	case CP_UTF8:
		// We could test validity of the characters - multibyte UTF-8 sequences
		// require the highest bit set in all character bytes after the first
		if ((cptr[0] & 128) == 0) {
			RETURN 1;
		} else {
			// todo: test if subsequent bytes start with bits 10
			if ((cptr[0] & 224) == 192) {
				SET_OP;
				size_t result = lengthIfOK(2, maxLen);
				RETURN result;
			} else {
				if ((cptr[0] & 240) == 224) {
					SET_OP;
					size_t result = lengthIfOK(3, maxLen);
					RETURN result;
				} else {
					SET_OP;
					size_t result = lengthIfOK(4, maxLen);
					RETURN result;
				}
			}
		}
	case CP_UTF16_LE:
		if (maxLen < 2) {
			RETURN 0;
		}
		if (wptr[0] >= 0xD800 && wptr[0] <= 0xDBFF) {
			if (maxLen < 4) {
				RETURN 0;
			}
			if (wptr[1] >= 0xDC00 && wptr[1] <= 0xDFFF) {
				RETURN 4;
			} else {
				RETURN 0; // Cut-off surrogate pair
			}
		} else {
			RETURN 2;
		}
	case CP_UTF16_BE:
		if (maxLen < 2) {
			RETURN 0;
		}
		if (wptr[0] >= 0xD800 && wptr[0] <= 0xDBFF) {
			if (maxLen < 4) {
				RETURN 0;
			}
			if (wptr[1] >= 0xDC00 && wptr[1] <= 0xDFFF) {
				RETURN 4;
			} else {
				RETURN 0; // Cut-off surrogate pair
			}
		} else {
			RETURN 2;
		}
	case 932:
		if ((cptr[0] > 0x80 && cptr[0] < 0xa0) || cptr[0] >= 0xe0) {
			SET_OP;
			size_t result = lengthIfOK(2, maxLen);
			RETURN result;
		} else {
			RETURN 1;
		}
	case 936:
	case 949:
	case 950:
		if (cptr[0] > 0x80) {
			SET_OP;
			size_t result = lengthIfOK(2, maxLen);
			RETURN result;
		} else {
			RETURN 1;
		}
	default:
		RETURN 1;
	}

	RETURN 1;
}

int getTerminatorSize(WORD cp)
{
	switch (cp) {
	case CP_UTF32:
		return 4;
	case CP_UTF16_LE:
	case CP_UTF16_BE:
		return 2;
	case 932:
	case 936:
	case 949:
	case 950:
	default:
		return 1;
	}
}

// todo: special codepages for outlook terminate strings with characters 
// that differ from 0, here we assume zero-terminated strings!
void terminateStr(void* StrEnd, WORD cp)
{
	SET_SCOPE;
	int ts = getTerminatorSize(cp);
	memset(StrEnd, 0, ts);
	RETURN;
}

// Test if word is zero-length, i.e. points to a terminator for the given code page
bool isTerminator(const char* word, WORD cp)
{
	if (word == NULL) {
		return true;
	}

	SET_SCOPE;
	int terminatorSize = getTerminatorSize(cp);

	switch (terminatorSize) {
	case 1:
		// UTF-8: valid multibyte tuples always have the high bit set in all bytes 
		RETURN (word[0] == 0);
	case 2:
		RETURN (((const char16_t*)(word))[0] == 0);
	default:
		while (terminatorSize > 0) {
			if (word[0] == 0) {
				++word;
				--terminatorSize;
			} else {
				RETURN false;
			}
		}
		RETURN true;
	}
}

// Return number of characters in string
size_t getCharCount(const char* word, size_t maxBytes, WORD cp)
{
	if (word == NULL) {
		return 0;
	}

	SET_SCOPE;
	size_t result = 0;
	while (!isTerminator(word, cp) && maxBytes > 0) {
		size_t charSize = getCharSize(word, cp, maxBytes);

		if (charSize > 0 && maxBytes >= charSize) {
			++result;
			maxBytes -= charSize;
			word += charSize;
		} else
			maxBytes = 0; // invalid character - abort counting
	}

	RETURN result;
}

// Return number of bytes used for the given number of characters
size_t countBytesForChars(const char* word, size_t charCount, WORD cp, int maxLen)
{
	SET_SCOPE;
	if (word == NULL) {
		RETURN 0;
	}

	size_t result = 0;
	while (charCount > 0 && !isTerminator(word, cp)) {
		size_t charSize = getCharSize(word, cp, maxLen);
		if (charSize > 0) {
			result += charSize;
			word += charSize;
			--charCount;
			maxLen -= (int)charSize;
		} else
			charCount = 0; // invalid character - abort
	}

	RETURN result;
}

// Default constructor
BGCPString::BGCPString()
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(CP_ASCII),
  m_byteCount(0),
  m_reserved(internalBufSize),
  m_terminatorSize(1)
{
	m_internalBuf[0] = 0;
}

// Construct from single ASCII character
BGCPString::BGCPString(char c)
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(CP_ASCII),
  m_byteCount(0),
  m_reserved(internalBufSize),
  m_terminatorSize(1)
{
	assign(&c, 1, CP_ASCII);
}

// Construct from a string with known length
// cp alone defines the type for str
BGCPString::BGCPString(const void* str, size_t byteCount, WORD cp)
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(cp),
  m_byteCount(0),
  m_reserved(internalBufSize)
{
	SET_SCOPE;
	m_internalBuf[0] = 0;
	m_terminatorSize = getTerminatorSize(cp);
	assign(str, byteCount, cp);
	RETURN;
}

// Construct from a string with unknown length
// Only ASCII strings are allowed without stating a codepage ...
BGCPString::BGCPString(const char* str)
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(CP_ASCII),
  m_byteCount(0),
  m_reserved(internalBufSize)
{
	SET_SCOPE;
	m_internalBuf[0] = 0;
	m_terminatorSize = 1;
	size_t len = bgstrlen<char>((char*)str);
	assign(str, len, CP_ASCII);
	RETURN;
}

// Construct from a string with unknown length
// cp alone defines the type for str
BGCPString::BGCPString(const void* str, WORD cp)
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(cp),
  m_byteCount(0),
  m_reserved(internalBufSize)
{
	SET_SCOPE;
	m_internalBuf[0] = 0;
	//m_terminatorSize = getTerminatorSize(cp);

	if (cp == CP_UTF16_LE || cp == CP_UTF16_BE) {
		size_t charCount = bgstrlen<char16_t>((char16_t*)str);
		assign(str, 2 * charCount, cp);
	} else {
		size_t charCount = bgstrlen<char>((char*)str);
		assign(str, charCount+1, cp);
	}

	RETURN;
}

#define UNEXPECTED_CODEPAGE_IN_STRING_CONSTRUCTION "Unexpected codepage in string construction"


// Copy constructor
BGCPString::BGCPString(const BGCPString& other)
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(other.m_codepage),
  m_byteCount(0),
  m_reserved(internalBufSize),
  m_terminatorSize(other.m_terminatorSize)
{
	SET_SCOPE;
	operator=(other);
	RETURN;
}

// Move constructor
BGCPString::BGCPString(BGCPString&& other) 
: VirtualClass("BGCPString", BGCPStringID),
  m_chars(m_internalBuf),
  m_codepage(other.m_codepage),
  m_byteCount(0),
  m_reserved(internalBufSize),
  m_terminatorSize(other.m_terminatorSize)
{
	if (other.m_chars == other.m_internalBuf) {
		// Real copy needed
		operator=(other);
	} else {
		// Steal string from other
		m_chars = other.m_chars;
		m_byteCount = other.m_byteCount;
		m_reserved = other.m_reserved;

		other.m_chars = other.m_internalBuf;
		other.m_internalBuf[0] = 0;
		other.m_byteCount = 0;
		other.m_reserved = other.internalBufSize;
	}
}

// Destructor
BGCPString::~BGCPString()
{
	SET_SCOPE;
	if (m_chars != nullptr) {
		setByteCount(0);
		if (m_chars != m_internalBuf) {
			SET_OP;
			traceFree(m_chars, BGCPStringBufID, m_reserved);
		}
	}
	SET_OP;
	m_chars = NULL;
	m_byteCount = 0;
	m_reserved = 0;
	m_terminatorSize = 0;
	RETURN;
}

// Check if the string is empty
bool BGCPString::empty() const
{
	return (m_byteCount == 0);
}

// Determine string length in bytes
size_t BGCPString::getByteCount(const char* str, WORD cp, size_t maxLen)
{
	if (str == NULL || maxLen == 0) {
		return 0;
	}

	SET_SCOPE;
	const char* curr = str;
	while (!isTerminator(curr, cp) && maxLen > 0) {
		size_t cs = ::getCharSize(curr, cp, maxLen);
		if (cs <= maxLen) {
			curr += cs;
			maxLen -= (int) cs;
		} else {
			maxLen = 0; // end of buffer, abort
		}
	}

	RETURN (curr - str);
}

// Assign copy from other BGCPString
void BGCPString::operator=(const BGCPString& str)
{
	SET_SCOPE;
	assign(str.m_chars, str.m_byteCount, str.m_codepage);
	RETURN;
}

// Assign move from other BGCPString
void BGCPString::operator=(BGCPString&& other)
{
	SET_SCOPE;
	clear();

	m_codepage = other.m_codepage;
	m_terminatorSize = other.m_terminatorSize;
	m_byteCount = other.m_byteCount;
	m_reserved = other.m_reserved;

	if (other.m_chars == other.m_internalBuf) {
		m_chars = m_internalBuf;
		memcpy(m_chars, other.m_chars, other.m_byteCount + other.m_terminatorSize);
	} else {
		m_chars = other.m_chars;
		other.m_chars = nullptr;
		other.m_reserved = 0;
		other.m_byteCount = 0;
	}

	RETURN;
}

// Compare to other BGCPString
bool BGCPString::operator==(const BGCPString& other) const
{
	SET_SCOPE;

	// This test can be done without conversion, so do it first
	auto cc1 = getCharCount();
	auto cc2 = other.getCharCount();
	if (cc1 != cc2) {
		RETURN false;
	}

	// Actually compare the strings
	if (m_codepage == other.m_codepage) {
		if (m_codepage == CP_UTF16_LE || m_codepage == CP_UTF16_BE) {
			const char16_t* p1 = get<char16_t>();
			const char16_t* p2 = other.get<char16_t>();
			int cmpRes = bgstrcmp(p1, p2);
			RETURN (cmpRes == 0);
		} else {
			const char* p1 = get<char>();
			const char* p2 = other.get<char>();
			int cmpRes = bgstrcmp(p1, p2);
			RETURN (cmpRes == 0);
		}
	} else {
		BGCPString tmp1 = (*this);
		tmp1.convertToCP(CP_UTF16_LE);
		BGCPString tmp2 = other;
		tmp2.convertToCP(CP_UTF16_LE);
		const char16_t* p1 = tmp1.get<char16_t>();
		const char16_t* p2 = tmp2.get<char16_t>();
		int cmpRes = bgstrcmp(p1, p2);
		RETURN (cmpRes == 0);
	}
}

bool BGCPString::operator!=(const BGCPString& other) const
{
	return !(operator==(other));
}

// Create new BGCPString by string concatenation
BGCPString BGCPString::operator+(const BGCPString& other) const
{
	SET_SCOPE;
	BGCPString result(*this);
	result += other;
	RETURN result;
}

// Copy part of a string
bool BGCPString::assign(const void* str, size_t byteCount, WORD cp)
{
	SET_SCOPE;

	// Verify byteCount - value given may be too large
	switch(cp) {
		//case CP_UTF32:
		case CP_UTF16_LE:
			{
				SET_OP;
				// For performance reasons, really don't count chars beyond byteCount!
				size_t len = bgstrlen<char16_t>((const char16_t*)str, byteCount);
				byteCount = std::min(byteCount, 2*len);
				m_terminatorSize = 2;
				break;
			}
		default:
			{
				SET_OP;
				// For performance reasons, really don't count chars beyond byteCount!
				size_t len = bgstrlen<char>((const char*)str, byteCount);
				byteCount = std::min(byteCount, len);
				m_terminatorSize = 1;
				break;
			}
	}

	SET_OP;
	m_codepage = cp;
	resizeBuffer(byteCount + m_terminatorSize);
	memcpy(m_chars, str, byteCount);
	setByteCount(byteCount);
	RETURN true;
}

// Set length of string and add terminator
// Does not check whether a new character starts at the given offset!
void BGCPString::setByteCount(size_t newLen)
{
	SET_SCOPE;
	assert(newLen + m_terminatorSize <= m_reserved);
	m_byteCount = newLen;
	terminateStr(newLen);
	RETURN;
}

// Clear string
void BGCPString::clear()
{
	SET_SCOPE;
	setByteCount(0);
	RETURN;
}

// Append part of a string
bool BGCPString::appendPart(const char* str, size_t byteCount, WORD cp)
{
	if (byteCount == 0 || str == nullptr) {
		return true;
	}

	SET_SCOPE;
	if (cp == m_codepage) {
		resizeBuffer(m_byteCount + byteCount);
		memcpy(m_chars + m_byteCount, str, byteCount);
		setByteCount(m_byteCount + byteCount);
	} else {
		BGCPString help(str, byteCount, cp);
		operator+=(help);
	}

	RETURN true;
}

// Make sure buffer is large enough to hold newSize bytes
void BGCPString::resizeBuffer(size_t newSize)
{
	SET_SCOPE;
	if (newSize + m_terminatorSize >= m_reserved) {
		size_t newReserve = 2 * newSize + m_terminatorSize;
		SET_OP;
		char* newChars = (char*) traceAlloc(newReserve, "BGCPStringBuf", BGCPStringBufID);
		if (newChars == nullptr) {
			throw BGException("Bad allocation");
		}
		SET_OP;
		memcpy(newChars, m_chars, m_byteCount + m_terminatorSize);
		SET_OP;
		if (m_chars != NULL && m_chars != m_internalBuf) {
			SET_OP;
			traceFree(m_chars, BGCPStringBufID, m_reserved);
		}
		m_chars = newChars;
		m_reserved = newReserve;
	}

	RETURN;
}

// Append another BGCPString
bool BGCPString::operator+=(const BGCPString& other)
{
	SET_SCOPE;
	bool result = true;

	if (m_byteCount == 0) {
		operator=(other);
	} else {
		if (m_codepage == other.m_codepage) {
			resizeBuffer(m_byteCount + other.m_byteCount);
			memcpy(&m_chars[m_byteCount], other.m_chars, other.m_byteCount + m_terminatorSize);
			m_byteCount += other.m_byteCount;
		} else {
			if (m_codepage == CP_UTF16_LE) {
				BGCPString copy = other;
				copy.convertToCP(CP_UTF16_LE);
				result = operator+=(copy);
			} else {
				convertToCP(CP_UTF16_LE);
				result = operator+=(other);
			}
		}
	}

	RETURN result;
}

#ifndef _WIN32
struct CP_Codec {
	WORD inputCodepage;
	WORD outputCodepage;
	iconv_t converter;
};
std::vector<CP_Codec> converters;

const char* getCPStr(WORD cp)
{
	switch(cp) {
		case CP_ASCII:
			return "437";
		case 1252:
			return "WINDOWS-1252";
		case CP_UTF8:
			return "UTF-8";
		case CP_UTF16_BE:
			return "UTF-16BE";
		case CP_UTF16_LE:
			return "UTF-16LE";
		case CP_UTF32:
			return "UTF-32LE";
		default:
			return "";
	}
}
#endif

size_t getMaxCharSize(WORD cp)
{
	switch(cp) {
		case CP_ASCII:
		case 1252:
			return 1;
		case CP_UTF8:
			return 2;
		case CP_UTF16_BE:
		case CP_UTF16_LE:
		case CP_UTF32:
			return 4;
		default:
			return 1;
	}
}

// Convert to codepage
bool BGCPString::convertToCP(WORD cp)
{
	// Cover some simple cases
	SET_SCOPE;
	if (cp == m_codepage) {
		RETURN true;
	}

	if (m_byteCount == 0) {
		SET_OP;
		m_codepage = cp;
		terminateStr(0);
		RETURN true;
	}

	if ((m_codepage == CP_ASCII && (cp == 1252 || cp == CP_UTF8))
	||  (m_codepage == CP_UTF8  && cp == 1252)) // todo: some characters would need special treatment ...
	{
		m_codepage = cp;
		RETURN true;
    }

	/*if (m_codepage == CP_UTF16_LE) {
		COUT << "iconv from " << m_codepage << " to " << cp << " starting for real ..." << std::endl;
		std::wcout << "Input uses cp " << m_codepage << std::endl;
		MemoryBlock mb((unsigned char*)m_chars, m_byteCount, false);
		mb.dump(0, m_byteCount);
	}*/

	if (m_codepage == CP_ASCII && cp == CP_UTF16_LE) {
		SET_OP;
		size_t newCharReserve = m_byteCount + 1;
		char16_t* tempBuf = (char16_t*)traceAlloc(2*newCharReserve, "BGCPStringBuf", BGCPStringBufID);
		if (tempBuf == nullptr) { 
			throw BGException("Bad allocation");
		}
		bgstrcpy(tempBuf, m_chars, newCharReserve);
		if (m_chars != NULL && m_chars != m_internalBuf) {
			traceFree(m_chars, BGCPStringBufID, m_reserved);
		}
		m_chars = (char*) tempBuf;
		m_reserved = 2 * newCharReserve;
		m_terminatorSize = 2;
		m_byteCount *= 2;
		m_codepage = CP_UTF16_LE;
		RETURN true;
	}

	// Convert UTF-8 to UTF-16?
	if (m_codepage == CP_UTF8 
	&&  cp == CP_UTF16_LE)
	{
		SET_OP;
		size_t inputOffset = 0;
		size_t charCount = getCharCount();
		// At most, every character requires 4 bytes in UTF-16
		char16_t* temp = (char16_t*)traceAlloc(4*charCount+2, "BGCPStringBuf", BGCPStringBufID);
		size_t outputOffset = 0;
		
		while (inputOffset < m_byteCount) {
			uint32_t c = (unsigned char)(m_chars[inputOffset]);
			
			// Decode input char
			if (c >= 128) {
				if ((c & 224) == 192) {
					++inputOffset;
					if (inputOffset < m_byteCount) {
						unsigned char d = m_chars[inputOffset];
						c = ((c & 0x1f) << 6) + (d & 0x3F);
						++inputOffset;
					}
				} else if ((c & 240) == 224) {
					if (inputOffset+2 < m_byteCount) {
						unsigned char c2 = m_chars[inputOffset+1];
						unsigned char c3 = m_chars[inputOffset+2];
						c = ((c & 15) << 12)+ ((c2 & 0x3F) << 6) + (c3 & 0x3F);
					}
					inputOffset += 3;
				} else { //if ((c & 248) == 240) {
					if (inputOffset+3 < m_byteCount) {
						unsigned char c2 = m_chars[inputOffset+1];
						unsigned char c3 = m_chars[inputOffset+2];
						unsigned char c4 = m_chars[inputOffset+3];
						c = ((c & 7) << 18) + ((c2 & 0x3F) << 12) + ((c3 & 0x3F) << 6) + (c4 & 0x3f);
					}
					inputOffset += 4;
				}
			} else {
				++inputOffset;
			}
			
			// Write output char
			if (c <= 10000) {
				temp[outputOffset] = (uint16_t)(c);
				++outputOffset;
			} else {
				c -= 0x10000;
				uint16_t c1 = 0xd800 + (c >> 10);
				uint16_t c2 = 0xdc00 + (c & 0x3ff);
				temp[outputOffset] = c1;
				++outputOffset;
				temp[outputOffset] = c2;
				++outputOffset;
			}
		}
		
		if (m_chars != nullptr && m_chars != m_internalBuf) {
			traceFree(m_chars, BGCPStringBufID, m_byteCount);
		}
		temp[outputOffset] = 0;
		m_chars = (char*)temp;
		m_byteCount = 2*outputOffset;
		m_codepage = CP_UTF16_LE;
		m_terminatorSize = 2;
		RETURN true;
	}

	// Convert UTF-16 to UTF-8?
	if (m_codepage == CP_UTF16_LE
	&&  cp == CP_UTF8)
	{
		SET_OP;
		size_t inputOffset = 0;
		size_t charCount = getCharCount();
		size_t newBufSize = 4*charCount+20;
		char* temp = (char*)traceAlloc(newBufSize, "BGCPStringBuf", BGCPStringBufID);
		size_t outputOffset = 0;
		char16_t* input = (char16_t*)(m_chars);
		
		while (inputOffset < charCount) {
			// Decode UTF-16
			char32_t c = input[inputOffset];
			if (c >= 0xd800 && c < 0xdc00 && 2*inputOffset+2 < m_byteCount) {
				++inputOffset;
				char16_t d = input[inputOffset];
				++inputOffset;
				// todo: check range of d?
				c = ((c-0xd800) << 10) + (d-0xdc00) + 0x10000;
			} else {
				++inputOffset;
			}
			
			assert(outputOffset < newBufSize);
			
			// Encode UTF-8
			if (c < 128) {
				temp[outputOffset] = (char)c;
				++outputOffset;
			} else if (c < 0x800) {
				temp[outputOffset] = (char) (192 + (c >> 6));
				++outputOffset;
				temp[outputOffset] = (char) (128 + (c & 0x3F));
				++outputOffset;				
			} else if (c <= 0xFFff) {
				temp[outputOffset] = (char) (224 + (c >> 12));
				++outputOffset;
				temp[outputOffset] = (char) (128 + ((c >> 6) & 0x3F));
				++outputOffset;				
				temp[outputOffset] = (char) (128 + (c &0x3F));
				++outputOffset;				
			} else {
				temp[outputOffset] = (char) (240 + (c >> 18));
				++outputOffset;
				temp[outputOffset] = (char) (128 + ((c >> 12) & 0x3F));
				++outputOffset;				
				temp[outputOffset] = (char) (128 + ((c >> 6) & 0x3F));
				++outputOffset;				
				temp[outputOffset] = (char) (128 + (c & 0x3f));
				++outputOffset;				
			}
		}

		if (m_chars != nullptr && m_chars != m_internalBuf) {
			traceFree(m_chars, BGCPStringBufID, m_byteCount);
		}
		temp[outputOffset] = 0;
		m_chars = (char*)temp;
		m_byteCount = outputOffset;
		m_codepage = CP_UTF8;
		m_terminatorSize = 1;

		RETURN true;
	}

	// Use Windows function to convert the string
	#ifdef WIN32
	SET_OP;
	if (m_codepage != CP_UTF16_LE) {
		// Convert to UTF-16
		SET_OP;
		int ts = getTerminatorSize(CP_UTF16_LE);
		int bytesNeeded = 2 * MultiByteToWideChar(m_codepage, 0, m_chars, (int)m_byteCount, NULL, 0);
		SET_OP;
		if (bytesNeeded > 0) {
			SET_OP;
			char* tempBuf = (char*)traceAlloc(bytesNeeded + ts, "BGCPStringBuf", BGCPStringBufID);
			CONDITIONAL_THROW_WITH_BACKTRACE(tempBuf == nullptr, "Bad allocation");
			SET_OP;
			MultiByteToWideChar(m_codepage, 0, m_chars, (int)m_byteCount, 
				(wchar_t*)tempBuf, bytesNeeded + 2);

			if (m_chars != NULL && m_chars != m_internalBuf) {
				SET_OP;
				traceFree(m_chars, BGCPStringBufID, m_reserved);
			}
			m_chars = tempBuf;
			m_codepage = CP_UTF16_LE;
			m_reserved = bytesNeeded + ts;
			m_terminatorSize = ts;
			setByteCount(bytesNeeded);
		} else {
			// Error: non-empty string reduced to zero length
			SET_OP;
			clear();
		}
	}

	if (cp != CP_UTF16_LE) {
		// Convert to final codepage
		// UTF-8 requires flags=0 and both final parameters NULL
		SET_OP;
		int ts = getTerminatorSize(cp);
		int bytesNeeded =
			WideCharToMultiByte(cp, 0, (wchar_t*)m_chars,
				(int)(m_byteCount / 2), NULL, 0, NULL, NULL);

		if (bytesNeeded > 0) {
			SET_OP;
			// Can't use resizeBuffer() because we need a second buffer
			char* tempBuf = (char*)traceAlloc(bytesNeeded + ts, "BGCPStringBuf", BGCPStringBufID);
			CONDITIONAL_THROW_WITH_BACKTRACE(tempBuf == nullptr, "Bad allocation");
			SET_OP;
			WideCharToMultiByte(cp, 0, (wchar_t*)m_chars,
				(int)(m_byteCount / 2) + 1, tempBuf, bytesNeeded, NULL, NULL);
			SET_OP;
			if (m_chars != NULL && m_chars != m_internalBuf) {
				SET_OP;
				traceFree(m_chars, BGCPStringBufID, m_reserved);
			}
			SET_OP;
			m_chars = tempBuf;
			m_reserved = bytesNeeded + ts;
			m_terminatorSize = ts;
			m_codepage = cp;
			setByteCount(bytesNeeded);
		} else {
			// Error: non-empty string reduced to zero length
			SET_OP;
			clear();
		}
	}

	SET_OP;
	m_terminatorSize = getTerminatorSize(cp);
	RETURN true;

	#else
/*	SET_OP;
	COUT << "I don't know how to convert a string from cp "
	     << m_codepage << " to " << cp << std::endl;
	getStackTraceString().printLn();
	RETURN false;*/

	iconv_t converter = (iconv_t) -1;
	
	// Check if we have converter already
	for (size_t i = 0; i < converters.size(); ++i) {
		const CP_Codec& candidate = converters[i];
		if (candidate.inputCodepage == m_codepage && candidate.outputCodepage == cp) {
			converter = candidate.converter;
		}
	}
	
	if (converter == (iconv_t) -1) {
		// todo: get these strings
		const char* inputCPStr = getCPStr(m_codepage);
		const char* outputCPStr = getCPStr(cp);
		converter = iconv_open(outputCPStr, inputCPStr);
		if (converter != (iconv_t) -1) {
			CP_Codec codec;
			codec.inputCodepage = m_codepage;
			codec.outputCodepage = cp;
			codec.converter = converter;
			converters.push_back(codec);
		} else {
			COUT << "Converter construction failed" << std::endl;
		}
	}
	
	if (converter != (iconv_t) -1) {
		// iconv alters all its parameters, so we need copies
		char* input = m_chars;
		size_t inputCount = m_byteCount;
		size_t outputSize = getMaxCharSize(cp);
		SET_OP;
		char* output = (char*)traceAlloc(outputSize, "BGStringBuf", BGCPStringBufID);
		char* outputCopy = output;
		SET_OP;
		if (iconv(converter, &input, &inputCount, &outputCopy, &outputSize) == (size_t)-1)
		{
			// Failure ...
			SET_OP;
			COUT << "iconv from " << m_codepage << " to " << cp << " failed" << std::endl;
			
			if (m_codepage == CP_UTF16_LE) {
				std::wcout << "String was " << ((char16_t*) m_chars) << std::endl;
			}
			
			logStackTrace("Stacktrace: ");
			SET_OP;
			traceFree(output, BGCPStringBufID, outputSize);
			RETURN false;
		} else {
			// Success
			SET_OP;
			m_chars = output;
			m_byteCount = outputSize;
			m_terminatorSize = getTerminatorSize(cp);
			terminateStr(m_byteCount);
			RETURN true;
		}
	} else {
		RETURN false;
	}

	#endif
}

// Create string that represents an integer
BGCPString BGCPString::fromInt(int64_t num, int base)
{
	SET_SCOPE;
	BGCPString result;

	if (num == 0) {
		result.add("0");
	} else {
		int digits = 0;
		const char* digitChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
		if (num < 0) {
			result.add("-");
			num = -num;
		}

		// Build an array with the digits, reversed
		const size_t bufSize = 50;
		char buf[bufSize];
		while (num != 0 && digits < bufSize) {
			int digit = (num % base);
			num = num / base;
			buf[digits] = digitChars[digit];
			++digits;
		}

		while (digits > 0) {
			--digits;
			result.add(&buf[digits]);
		}
	}
	
	RETURN result;
}

// Create a string that represents a double
// custom solution is twice as fast for big data ...
BGCPString BGCPString::fromDouble(double d, int precision)
{
	SET_SCOPE;
	BGCPString result;

	if (d < 0.0) {
		result.add("-");
		d = -d;
	}

	if (d > 10000000000) {
		int exp = 0;
		while (d >= 10.0) {
			++exp;
			d /= 10.0;
		}
		result += fromDouble(d, 3) + "e" + fromInt(exp);
	} else {
		int64_t i = (int64_t) d;
		result += BGCPString::fromInt(i);
		d -= i;
	
			if (d > 0 && precision > 0 && d < 1) {
			result.add(".");
			const char* digits = "0123456789";
			while (d > 0 && precision > 0) {
				int digit = (int) (10*d);
				result.add(&digits[digit]);
				--precision;
				d = 10.0 * d - digit;
			}
		}
}

	RETURN result;
}

// Convert string to a double
double BGCPString::toDouble() const
{
	SET_SCOPE;
	if (empty()) {
		RETURN 0.0;
	}

	double result = 0.0;
	switch (m_codepage) {
		case CP_UTF16_BE:
			{
				BGCPString copy;
				copy.convertToCP(CP_UTF16_LE);
				result = bgstrToD<char16_t>((char16_t*)copy.m_chars);
			}
			break;
		case CP_UTF16_LE:
			result = bgstrToD<char16_t>((char16_t*)m_chars);
			break;
		default:
			result = bgstrToD<char>(m_chars);
			break;
	}
	RETURN result;
}

// Add path separator to the end of the string
void BGCPString::addPathSep()
{
	add(&OSPathSep, CP_SYSTEM, sizeof(PathChar));
}

// Get codepage currently used by the internal representation
WORD BGCPString::getCP() const
{
	return m_codepage;
}

// Get number of bytes used by current internal string representation
// The returned value does not include the terminating zero
size_t BGCPString::getByteCount() const
{
	return m_byteCount;
}

// Get number of characters in current string
size_t BGCPString::getCharCount() const
{
	return ::getCharCount(m_chars, m_byteCount, m_codepage);
}

int64_t BGCPString::toInt(int base) const
{
	SET_SCOPE;
	switch (m_codepage) {
		case CP_UTF16_BE:
			{
				BGCPString copy;
				copy.convertToCP(CP_UTF16_LE);
				RETURN bgstrToI<char16_t>((char16_t*)copy.m_chars, base);
			}
		case CP_UTF16_LE:
			RETURN bgstrToI<char16_t>((char16_t*)m_chars, base);
		default:
			RETURN bgstrToI(m_chars, base);
	}
}

// Append windows error message to string
#ifdef _WIN32
BGCPString BGCPString::getOSError(DWORD winErrCode)
{
	SET_SCOPE;
	BGCPString result;
    const size_t winErrSize = 500;
	wchar_t winErr[winErrSize];
	winErr[0] = 0;
	auto formatSuccess = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM, NULL, winErrCode, 0, 
		(TCHAR*) (&winErr), winErrSize, NULL);

	if (formatSuccess) {
		result = BGCPString(winErr, CP_UTF16_LE);
	} else {
		auto formatError = GetLastError();
		result = BGCPString("Windows error code is ") + BGCPString::fromInt(winErrCode);
	}
	RETURN result;
}
#else
BGCPString BGCPString::getOSError(int code)
{
	SET_SCOPE;
	switch (code) {
		case E2BIG: RETURN BGCPString("Argument list too long", CP_ASCII);
		case EACCES: RETURN BGCPString("Permission denied", CP_ASCII);
		case EADDRINUSE: RETURN BGCPString("Address already in use", CP_ASCII);
		case EADDRNOTAVAIL: RETURN BGCPString("Address not available", CP_ASCII);
		case EAFNOSUPPORT: RETURN BGCPString("Address family not supported", CP_ASCII);
		case EAGAIN: RETURN BGCPString("Resource temporarily unavailable (may be the same value as EWOULDBLOCK)", CP_ASCII);
		case EALREADY: RETURN BGCPString("Connection already in progress", CP_ASCII);
		case EBADE: RETURN BGCPString("Invalid exchange", CP_ASCII);
		case EBADF: RETURN BGCPString("Bad file descriptor", CP_ASCII);
		case EBADFD: RETURN BGCPString("File descriptor in bad state", CP_ASCII);
		case EBADMSG: RETURN BGCPString("Bad message", CP_ASCII);
		case EBADR: RETURN BGCPString("Invalid request descriptor", CP_ASCII);
		case EBADRQC: RETURN BGCPString("Invalid request code", CP_ASCII);
		case EBADSLT: RETURN BGCPString("Invalid slot", CP_ASCII);
		case EBUSY: RETURN BGCPString("Device or resource busy", CP_ASCII);
		case ECANCELED: RETURN BGCPString("Operation canceled", CP_ASCII);
		case ECHILD: RETURN BGCPString("No child processes", CP_ASCII);
		case ECHRNG: RETURN BGCPString("Channel number out of range", CP_ASCII);
		case ECOMM: RETURN BGCPString("Communication error on send", CP_ASCII);
		case ECONNABORTED: RETURN BGCPString("Connection aborted", CP_ASCII);
		case ECONNREFUSED: RETURN BGCPString("Connection refused", CP_ASCII);
		case ECONNRESET: RETURN BGCPString("Connection reset", CP_ASCII);
		case EDEADLK: RETURN BGCPString("Resource deadlock avoided", CP_ASCII);
		/*case EDEADLOCK: RETURN BGCPString("On most architectures, a synonym for EDEADLK. "
			"On some architectures (e.g., Linux MIPS, PowerPC), it is a separate "
			"error code \"File locking deadlock error\"", CP_ASCII);*/
		case EDESTADDRREQ: RETURN BGCPString("Destination address required", CP_ASCII);       
		case EDOM: RETURN BGCPString("Mathematics argument out of domain of functio", CP_ASCII);
		case EDQUOT: RETURN BGCPString("Disk quota exceeded", CP_ASCII);
		case EEXIST: RETURN BGCPString("File exists", CP_ASCII);
		case EFAULT: RETURN BGCPString("Bad address", CP_ASCII);
		case EFBIG: RETURN BGCPString("File too large", CP_ASCII);
		case EHOSTDOWN: RETURN BGCPString("Host is down", CP_ASCII);
		case EHOSTUNREACH: RETURN BGCPString("Host is unreachable", CP_ASCII);
		case EHWPOISON: RETURN BGCPString("Memory page has hardware error", CP_ASCII);
		case EIDRM: RETURN BGCPString("Identifier removed", CP_ASCII);
		case EILSEQ: RETURN BGCPString(" Invalid or incomplete multibyte or wide characte", CP_ASCII);
		case EINPROGRESS: RETURN BGCPString("Operation in progress", CP_ASCII);
		case EINTR: RETURN BGCPString("Interrupted function call; see signal(7)", CP_ASCII);
		case EINVAL: RETURN BGCPString("Invalid argument", CP_ASCII);
		case EIO: RETURN BGCPString("Input/output error", CP_ASCII);
		case EISCONN: RETURN BGCPString("Socket is connected", CP_ASCII);
		case EISDIR: RETURN BGCPString("Is a directory", CP_ASCII);
		case EISNAM: RETURN BGCPString("Is a named type file", CP_ASCII);
		case EKEYEXPIRED: RETURN BGCPString("Key has expired", CP_ASCII);
		case EKEYREJECTED: RETURN BGCPString("Key was rejected by service", CP_ASCII);
		case EKEYREVOKED: RETURN BGCPString("Key has been revoked", CP_ASCII);
		case EL2HLT: RETURN BGCPString("Level 2 halted", CP_ASCII);
		case EL2NSYNC: RETURN BGCPString("Level 2 not synchronized", CP_ASCII);
		case EL3HLT: RETURN BGCPString("Level 3 halted", CP_ASCII);
		case EL3RST: RETURN BGCPString("Level 3 reset", CP_ASCII);
		case ELIBACC: RETURN BGCPString("Cannot access a needed shared library", CP_ASCII);
		case ELIBBAD: RETURN BGCPString("Accessing a corrupted shared library", CP_ASCII);
		case ELIBMAX: RETURN BGCPString("Attempting to link in too many shared libraries", CP_ASCII);
		case ELIBSCN: RETURN BGCPString(".lib section in a.out corrupte", CP_ASCII);
		case ELIBEXEC: RETURN BGCPString("Cannot exec a shared library directly", CP_ASCII);
		//case ELNRANGE: RETURN BGCPString("Link number out of range", CP_ASCII);
		case ELOOP: RETURN BGCPString("Too many levels of symbolic links", CP_ASCII);
		case EMEDIUMTYPE: RETURN BGCPString("Wrong medium type", CP_ASCII);
		case EMFILE: RETURN BGCPString("Too many open files. Commonly caused by exceeding the "
			"RLIMIT_NOFILE resource limit described in getrlimit(2).", CP_ASCII);
		case EMLINK: RETURN BGCPString("Too many links", CP_ASCII);
		case EMSGSIZE: RETURN BGCPString("Message too long", CP_ASCII);
		case EMULTIHOP: RETURN BGCPString("Multihop attempted", CP_ASCII);
		case ENAMETOOLONG: RETURN BGCPString("Filename too long", CP_ASCII);
		case ENETDOWN: RETURN BGCPString("Network is down", CP_ASCII);
		case ENETRESET: RETURN BGCPString("Connection aborted by network", CP_ASCII);
		case ENETUNREACH: RETURN BGCPString("Network unreachable", CP_ASCII);
		case ENFILE: RETURN BGCPString("Too many open files in system.  On "
			"Linux, this is probably a result of encountering the /proc/sys/fs/file-max limit.", CP_ASCII);
		case ENOANO: RETURN BGCPString("No anode", CP_ASCII);
		case ENOBUFS: RETURN BGCPString("No buffer space available (POSIX.1 (XSI STREAMS option))", CP_ASCII);
		case ENODATA: RETURN BGCPString("No message is available on the STREAM head read queue", CP_ASCII);
		case ENODEV: RETURN BGCPString("No such device", CP_ASCII);
		case ENOENT: RETURN BGCPString("No such file or directory. Typically, this error results when "
			"a specified path name does not exist, or one of the components in the "
			"directory prefix of a pathname does not exist, or the specified pathname "
			"is a dangling symbolic link.", CP_ASCII);
		case ENOEXEC: RETURN BGCPString("Exec format error", CP_ASCII);
		case ENOKEY: RETURN BGCPString("Required key not available", CP_ASCII);
		case ENOLCK: RETURN BGCPString("No locks available", CP_ASCII);
		case ENOLINK: RETURN BGCPString("Link has been severed", CP_ASCII);
		case ENOMEDIUM: RETURN BGCPString("No medium found", CP_ASCII);
		case ENOMEM: RETURN BGCPString("Not enough space/cannot allocate memor", CP_ASCII);
		case ENOMSG: RETURN BGCPString("No message of the desired type", CP_ASCII);
		case ENONET: RETURN BGCPString("Machine is not on the network", CP_ASCII);
		case ENOPKG: RETURN BGCPString("Package not installed", CP_ASCII);
		case ENOPROTOOPT: RETURN BGCPString("Protocol not available", CP_ASCII);
		case ENOSPC: RETURN BGCPString("No space left on device", CP_ASCII);
		case ENOSR: RETURN BGCPString("No STREAM resources (POSIX.1 (XSI STREAMS option))", CP_ASCII);
		case ENOSTR: RETURN BGCPString("Not a STREAM (POSIX.1 (XSI STREAMS option))", CP_ASCII);
		case ENOSYS: RETURN BGCPString("Function not implemented", CP_ASCII);
		case ENOTBLK: RETURN BGCPString("Block device required", CP_ASCII);
		case ENOTCONN: RETURN BGCPString("The socket is not connected", CP_ASCII);
		case ENOTDIR: RETURN BGCPString("Not a directory", CP_ASCII);
		case ENOTEMPTY: RETURN BGCPString("Directory not empty", CP_ASCII);
		case ENOTRECOVERABLE: RETURN BGCPString("State not recoverable (POSIX.1-2008)", CP_ASCII);
		case ENOTSOCK: RETURN BGCPString("Not a socket", CP_ASCII);
		case ENOTSUP: RETURN BGCPString("Operation not supported", CP_ASCII);
		case ENOTTY: RETURN BGCPString("Inappropriate I/O control operation", CP_ASCII);
		case ENOTUNIQ: RETURN BGCPString("Name not unique on network", CP_ASCII);
		case ENXIO: RETURN BGCPString("No such device or address", CP_ASCII);
		//case EOPNOTSUPP: RETURN BGCPString("Operation not supported on socket", CP_ASCII);
		case EOVERFLOW: RETURN BGCPString("Value too large to be stored in data typ", CP_ASCII);
		case EOWNERDEAD: RETURN BGCPString("Owner died (POSIX.1-2008)", CP_ASCII);
		case EPERM: RETURN BGCPString("Operation not permitted", CP_ASCII);
		case EPFNOSUPPORT: RETURN BGCPString("Protocol family not supported", CP_ASCII);
		case EPIPE: RETURN BGCPString("Broken pipe", CP_ASCII);
		case EPROTO: RETURN BGCPString("Protocol error", CP_ASCII);
		case EPROTONOSUPPORT: RETURN BGCPString("Protocol not supported", CP_ASCII);
		case EPROTOTYPE: RETURN BGCPString("Protocol wrong type for socket", CP_ASCII);
		case ERANGE: RETURN BGCPString("Result too large (POSIX.1, C99)", CP_ASCII);
		case EREMCHG: RETURN BGCPString("Remote address changed", CP_ASCII);
		case EREMOTE: RETURN BGCPString("Object is remote", CP_ASCII);
		case EREMOTEIO: RETURN BGCPString("Remote I/O error", CP_ASCII);
		case ERESTART: RETURN BGCPString("Interrupted system call should be restarted", CP_ASCII);
		case ERFKILL: RETURN BGCPString("Operation not possible due to RF-kill", CP_ASCII);
		case EROFS: RETURN BGCPString("Read-only filesystem", CP_ASCII);
		case ESHUTDOWN: RETURN BGCPString("Cannot send after transport endpoint shutdown", CP_ASCII);
		case ESPIPE: RETURN BGCPString("Invalid seek", CP_ASCII);
		case ESOCKTNOSUPPORT: RETURN BGCPString("Socket type not supported", CP_ASCII);
		case ESRCH: RETURN BGCPString("No such process", CP_ASCII);
		case ESTALE: RETURN BGCPString("Stale file handle", CP_ASCII);
		case ESTRPIPE: RETURN BGCPString("Streams pipe error", CP_ASCII);
		case ETIME: RETURN BGCPString("Timer expired", CP_ASCII);
		case ETIMEDOUT: RETURN BGCPString("Connection timed out", CP_ASCII);
		case ETOOMANYREFS: RETURN BGCPString("Too many references: cannot splice", CP_ASCII);
		case ETXTBSY: RETURN BGCPString("Text file busy", CP_ASCII);
		case EUCLEAN: RETURN BGCPString("Structure needs cleaning", CP_ASCII);
		case EUNATCH: RETURN BGCPString("Protocol driver not attached", CP_ASCII);
		case EUSERS: RETURN BGCPString("Too many users", CP_ASCII);
		//case EWOULDBLOCK: RETURN BGCPString("Operation would block (may be same value as EAGAIN)", CP_ASCII);
		case EXDEV: RETURN BGCPString("Improper link", CP_ASCII);
		case EXFULL: RETURN BGCPString("Exchange full", CP_ASCII);
		default:
			RETURN BGCPString("Unknown error code ", CP_ASCII) + BGCPString::fromInt(code);
	}
}
#endif

// Print this string to the console
void BGCPString::print() const
{
	SET_SCOPE;
	if (empty()) {
		RETURN;
	}

	#ifdef _WIN32
	BGCPString temp = *this;
	temp.convertToCP(CP_UTF16_LE);
	wprintf(L"%s", (wchar_t*)(temp.m_chars));
	#else
	if (m_codepage == CP_UTF16_LE) {
		wprintf(L"%s", (char16_t*)(m_chars));
	} else {
		printf("%s", m_chars);
	}
	#endif
	RETURN;
}

// Print this string to the console and add a line feed
void BGCPString::printLn() const
{
	SET_SCOPE;
	print();
	EOL.print();
	RETURN;
}


bool BGCPString::startsWith(const BGCPString& other) const
{
	SET_SCOPE;
	BGCPString str1 = *this;
	BGCPString str2 = other;
	str1.convertToCP(CP_UTF16_LE);
	str2.convertToCP(CP_UTF16_LE);

	if (str1.getByteCount() < str2.getByteCount()) {
		RETURN false;
	} else {
		size_t len = str2.getByteCount();
		str1.setByteCount(len);
		RETURN (str1 == str2);
	}
}

// Get number of bytes used by chara at given byte offset
int BGCPString::getCharSize(size_t byteOffset) const
{
	return ::getCharSize(m_chars+byteOffset, m_codepage, m_byteCount);
}

// Create size string, e.g. 1024MB for 1024*1024
BGCPString BGCPString::sizeStr(int64_t size)
{
	SET_SCOPE;
	BGCPString result;

	if (size < 2 * 1024) {
		// Report size in bytes
		result = BGCPString::fromInt(size) + BGCPString(" B");
	} else if (size < 1024 * 1024) {
		// Report size in kbytes
		result = BGCPString::fromDouble(size / 1024.0, 1) + BGCPString(" kB");
	} else if (size < 1024 * 1024 * 1024) {
		// Report size in MBytes
		result = BGCPString::fromDouble(size / (1024.0 * 1024.0), 1) + BGCPString(" MB");
	} else {
		// Report size in GBytes
		result = BGCPString::fromDouble(size / (1024.0 * 1024.0 * 1024), 1) + BGCPString(" GB");
	}

	RETURN result;
}

// This function is not const: this may be altered by converting to UTF-16.
// It is possible to work with a copy of this and convert that to UTF-16 and then declare this function
// const, but that is wrong, since then pos will be an index to a UTF-16 character, whereas the original
// could be ASCII ...
/*bool BGCPString::find(const BGCPString& other, size_t& pos) //const
{
	SET_SCOPE;
	//BGCPString str1 = (*this);
	convertToCP(CP_UTF16_LE);
	size_t len1 = getByteCount();

	BGCPString str2 = other;
	str2.convertToCP(CP_UTF16_LE);
	size_t len2 = str2.getByteCount();

	if (len2 > len1) {
		RETURN false;
	}

	pos = 0;
	bool found = false;
	while (pos+len2 <= len1 && !found) {
		//BGCPString temp(getPtr<char16_t>(pos), len2, CP_UTF16_LE);
		if (bgstrcmp(getPtr<char16_t>(pos), str2.getPtr<char16_t>(0), len2/2) == 0) {
			found = true;
		} else {
			pos += 2;
		}
	}
	RETURN found;
}*/

// Check if this string ends with another string
bool BGCPString::endsWith(const BGCPString& other) const
{
	SET_SCOPE;
	if (other.empty()) {
		RETURN true;
	}
	size_t thisCC = getCharCount();
	size_t otherCC = other.getCharCount();

	bool result = false;
	if (otherCC > thisCC) {
		RETURN false;
	} else {
		size_t offset = 0;
		while (thisCC > otherCC) {
			offset += getCharSize(offset);
			--thisCC;
		}
		BGCPString thisCopy(m_chars+offset, m_byteCount-offset, m_codepage);
		RETURN (thisCopy == other);
	}
}

bool isPathSep(const void* c, WORD codepage)
{
	bool result = false;

	if (codepage != CP_UTF16_LE) {
		const char* c8p = (const char*)(c);
		#ifdef _WIN32
		result = (c8p[0] == '/' || c8p[0] == '\\');
		#else
		result = (c8p[0] == '/');
		#endif
	} else {
		const char16_t* c16p = (const char16_t*)(c);
		#ifdef _WIN32
		result = (c16p[0] == '/' || c16p[0] == '\\');
		#else
		result = (c16p[0] == '/');
		#endif
	}

	return result;
}

bool BGCPString::saveToFile(const PathChar* fn, WORD cp) const
{
	SET_SCOPE;
	if (m_codepage == cp) {
		bool result = ::saveToFile(m_chars, m_byteCount, fn);
		RETURN result;
	} else {
		BGCPString temp = *this;
		temp.convertToCP(cp);
		bool result = ::saveToFile(temp.m_chars, temp.m_byteCount, fn);
		RETURN result;
	}
}

// !!!!!!!!!!!!!!!!!!!!
// Warning: may not work in all cases.
// Works for "C:\Data\devel\MechVM-2021-03-26\mechs\Kingfisher\Kingfisher.mvmk"
// but not for "C:\temp\MechVM-win32-2021-03-26\mechs\Kingfisher\Kingfisher.mvmk"
// Except if that error was caused by something else ...
// !!!!!!!!!!!!!!!!!!!!
BGCPString getDirectory(const BGCPString& path)
{
	size_t lastSlashPos = 0;
	size_t pos = 0;

	// Not sure what to do when no path sep is found, given path is relative
	// and could be a directory or a file ...
	if (path.getCP() != CP_UTF16_LE && path.getCP() != CP_UTF16_BE) {
		while (pos < path.getByteCount()) {
			if (isPathSep(path.getPtr<char>(), path.getCP())) {
				lastSlashPos = pos;
			}
			pos += path.getCharSize(pos);
		}

		return BGCPString(path.getPtr<char>(), lastSlashPos, path.getCP());
	} else {
		const char16_t* str16 = path.getPtr<char16_t>();
		while (pos < path.getByteCount()) {
			if (isPathSep(str16+pos, path.getCP())) {
				lastSlashPos = pos;
			}
			++pos;
		}

		return BGCPString(path.getPtr<char16_t>(), 2*lastSlashPos, path.getCP());
	}
}

// Get file name (last part of path)
BGCPString getFilename(const PathChar* path)
{
	size_t lastPathSep = 0;
	size_t len = bgstrlen(path);
	bool found = true;
	do {
		size_t candidate = lastPathSep;
		found = bgstrscan(path, OSPathSep, candidate, len, candidate);
		if (found) {
			lastPathSep = candidate + 1;
		}
	} while (found);
	return BGCPString(path + lastPathSep, sizeof(PathChar) * (len-lastPathSep), CP_SYSTEM);
}

// Scan this string for searchStr
// Returns either the byte offset of the first match for searchStr
// past startPos or endPos if nothing is found.
// The size of a character is often not fixed, therefore even a single
// character should be treated as a string, so it does not make sense
// to offer a function that looks for a single character ...
// Keep in mind that codepage conversions can invalidate the indexes ...
// todo: write tests, use less string compare
size_t BGCPString::scan(const BGCPString& searchStr, size_t startPos, size_t endPos) const
{
	// todo: does it matter if the codepage does not match?
	// We should simply compare the substrings, the compare operator will convert if necessary.
	// Then the offsets won't change, because the offsets are not affected by the conversions.
	// Also, if searchStr has characters that cannot be expressed in this' codepage, searchStr cannot
	// possibly have matches in this.
	size_t searchStrLen = searchStr.getByteCount();
	if (searchStr.getCP() != m_codepage) {
		BGCPString str2(searchStr);
		str2.convertToCP(m_codepage);
		searchStrLen = str2.getByteCount();
	}

	if (searchStrLen > endPos - startPos) {
		return endPos;
	}

	size_t pos = startPos;
	bool found = false;

	while (pos < endPos && !found) {
		if (pos+searchStrLen <= endPos) {
			BGCPString cstr(m_chars+pos, searchStrLen, m_codepage);
			found = (cstr == searchStr);
			if (!found) {
				size_t clen = getCharSize(pos);
				pos += clen;
			}
		} else {
			return endPos;
		}
	}

	return pos;
}

void BGCPString::terminateStr(size_t strEnd)
{
	// todo: resize?
	SET_SCOPE;
	size_t terminatorSize = getTerminatorSize(m_codepage);

	if (strEnd + terminatorSize <= m_reserved) {
		memset(m_chars + strEnd, 0, terminatorSize);
	}
	RETURN;
}

// Use table to encode all characters in this string
void BGCPString::transform (const char16_t* table, size_t tableSize)
{
	SET_SCOPE;
	tableSize = (tableSize >> 1); // compute number of char16_ts
	WORD oldCP = m_codepage;
	convertToCP(CP_UTF16_LE);
	size_t len = (m_byteCount >> 1);
	char16_t* wchars = (char16_t*)m_chars;

	for (size_t i = 0; i < len; ++i) {
		char16_t c = wchars[i];
		if ((size_t) c < tableSize)
			wchars[i] = table[c];
	}

	convertToCP(oldCP);
	RETURN;
}

void BGCPString::setCharCount(size_t newCount)
{
	SET_SCOPE;
	size_t i = 0;

	while (newCount > 0 && i < m_byteCount) {
		if (!isTerminator(&m_chars[i], m_codepage)) {
			size_t charSize = getCharSize(i);

			if (charSize > 0) {
				i += charSize;
				--newCount;
			} else
				newCount = 0;
		} else
			newCount = 0;
	}

	setByteCount(i);
	RETURN;
}

// Load string from file
bool BGCPString::loadFromFile(const PathChar* fn)
{
	SET_SCOPE;
	size_t size = 0;
	char* buf = (char*) ::loadFromFile(fn, size);
	bool result = true;
	int BOMsize = 0;
	if (buf != nullptr) {
		// Try to determine the codepage ...
		WORD codepage = CP_ASCII;
		if (size > 2) {
			if (buf[0] == 0xFE && buf[1] == 0xFF) {
				codepage = CP_UTF16_BE;
				BOMsize = 2;
			} else if (buf[0] == 0xFF && buf[1] == 0xFE) {
				codepage = CP_UTF16_LE;
				BOMsize = 2;
			} else if (buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF) {
				codepage = CP_UTF8;
				BOMsize = 3;
			}
		}

		assign(buf + BOMsize, size - BOMsize, codepage);
		traceFree(buf, BGCPStringBufID, size);
	} else {
		clear();
		result = false;
	}
	RETURN result;
}

// Check if the character at offset (interpreted as a byte offset) matches c
// Not a template because the type of c should not decide the type used for comparing ...
bool BGCPString::charAtEquals(size_t offset, char c) const
{
	SET_SCOPE;
	if (m_codepage == CP_UTF16_LE) {
		SET_OP;
		const char16_t* pc = getPtr<char16_t>(offset);
		if (pc != nullptr) {
			RETURN ((*pc) == c);
		}
	} else {
		SET_OP;
		const char* pc = getPtr<char>(offset);
		if (pc != nullptr) {
			RETURN ((*pc) == c);
		}
	}

	RETURN (c == 0); // character 0 may match beyond end of string
}


// Get a representation of this string that is composed of characters as used for specifying file 
// names on this system
const PathChar* BGCPString::getAsPath()
{
	convertToCP(CP_SYSTEM);
	return (const PathChar*)(m_chars);
}

// Count number of occurrences of c in this string 
// todo: allow c to be a template type, allow multibyte c
size_t BGCPString::count(char c) const
{
	size_t count = 0;
	size_t byteOffset = 0;
	while (byteOffset < m_byteCount) {
		if (m_chars[byteOffset] == c) {
			++count;
		}
		byteOffset += getCharSize(byteOffset);
	}

	return count;
}

// Return part of this string as a copy. Start and count are byte offsets into the source string.
BGCPString BGCPString::copyPart(size_t start, size_t count) const
{
	BGCPString result;
	if (start < m_byteCount) {
		if (start+count > m_byteCount) {
			// No, you won't get more out of this than you put in ... ;)
			count = m_byteCount-start;
		}
		result = BGCPString(m_chars+start, count, m_codepage);
	}
	return result;
}

// Append other to this string if this string is not empty
// Useful to generate lists
void BGCPString::appendIfNotEmpty(const BGCPString& other)
{
	if (!empty()) {
		operator+=(other);
	}
}

// Convert string to lower case ASCIIupdateMercsContracts
BGCPString BGCPString::lowerStringASCII() const
{
	BGCPString result = *this;
	result.convertToCP(CP_ASCII);
	for (size_t i = 0; i < result.m_byteCount; ++i) {
		char c= result.m_chars[i];
		if (c >= 'A' && c <= 'Z') {
			c = c - 'A' + 'a';
			result.m_chars[i] = c;
		}
	}
	return result;
}

// Advance from pos by count characters and return the result
size_t BGCPString::forward(size_t pos, size_t count) const
{
	while (count > 0 && pos < m_byteCount) {
		pos += getCharSize(pos);
		--count;
	}
	return pos;
}

// Remove count characters from end of string
bool BGCPString::removeCharsFromEnd(size_t count)
{
	if (count > 0 && empty()) {
		return false;
	}

	std::vector<size_t> charOffsets;
	size_t pos = 0;

	while (pos < m_byteCount) {
		charOffsets.push_back(pos);
		size_t cs = getCharSize(pos);
		pos += cs;
	}

	if (charOffsets.size() < count) {
		return false;
	}

	size_t newSize = charOffsets[charOffsets.size() - count];
	setByteCount(newSize);
	return true;
}

#ifdef USE_QT
QString BGCPString::toQString() const
{
	BGCPString utf16copy = *this;
	utf16copy.convertToCP(CP_UTF16_LE);
	return QString::fromUtf16((const char16_t*)utf16copy.get<char16_t>());
}
#endif

///////////////////////////////////////////////////////////////////////////////
// Add a character to the current string
// Returns the number of bytes used by the character, 0 implies an error
// charP: pointer to the character
// codepage: codepage currently used by charP
// maxLen: maximum length of charP in bytes
// this used to be a template, but charP can be of void type because there is parameter for the codepage

size_t BGCPString::add(const void* charP, WORD codepage, size_t maxLen)
{
	if (charP == nullptr
	||  maxLen < 1)
	{
		return 0;
	}

	SET_SCOPE;
	size_t byteCount = ::getCharSize(charP, codepage, maxLen);

	if (byteCount == 0) {
		RETURN 0;
	}

	SET_OP;
	if (m_byteCount == 0) {
		assign(charP, maxLen, codepage);
		RETURN maxLen;
	}

	if ((codepage == m_codepage)
	|| ((codepage == CP_ASCII) && (m_codepage == CP_UTF8)))
	{
		// Append without conversion
		SET_OP;
		resizeBuffer(m_byteCount + byteCount);
		memcpy(m_chars+m_byteCount, charP, byteCount);
		m_byteCount += byteCount;
		terminateStr(m_byteCount);
	} else {
		// Append with conversion
		SET_OP;
		BGCPString help;
		help.assign(charP, byteCount, codepage);
		operator+=(help);
	}

	RETURN byteCount;
}

// Split this string into several strings, one for each line
std::vector<BGCPString> BGCPString::split(const char* splitChars, bool addEmpties) const
{
	SET_SCOPE;
	std::vector<BGCPString> lines;
	if (m_byteCount == 0) {
		RETURN lines; // empty input -> empty output
	}

	SET_OP;
	size_t curr = 0;
	size_t splitCharCount = bgstrlen(splitChars);
	BGCPString line;

	SET_OP;
	while (curr < m_byteCount) {
		int charSize = getCharSize(curr);
		bool isSplitChar = false;
		size_t i = 0;

		while (i < splitCharCount && !isSplitChar) {
			if (charAtEquals(curr, splitChars[i])) {
				isSplitChar = true;
			} else {
				++i;
			}
		}

		if (isSplitChar) {
			if (!line.empty() || addEmpties) {
				lines.push_back(line);
				line.clear();
			}
		} else {
			line += copyPart(curr, charSize);
		}
		curr += charSize;
	}

	// Copy remainder
	SET_OP;
	if (!line.empty() || addEmpties) {
		lines.push_back(line);
	}
	RETURN lines;
}

// Return a string that equals this string minus all preceeding or trailing whitespace
BGCPString BGCPString::removeWhitespace() const
{
	size_t start = 0;
	bool doneScanningForStart = false;

	while (start < m_byteCount && !doneScanningForStart)
	{
		if (charAtEquals(start, ' ') || charAtEquals(start, 0x9) || charAtEquals(start, 0x0a) || charAtEquals(start, 0x0d)) {
			start += getCharSize(start);
		} else {
			doneScanningForStart = true;
		}
	}

	size_t curr = start;
	size_t endOfNonWhite = start;

	while (curr < m_byteCount) {
		auto cs = getCharSize(curr);
		if (!charAtEquals(curr, ' ')
		&&  !charAtEquals(curr, 0x09)
		&&  !charAtEquals(curr, 0x0a)
		&&  !charAtEquals(curr, 0x0d))
		{
			endOfNonWhite = curr+cs;
		}
		curr += cs;
	}

	if (endOfNonWhite > start) {
		return copyPart(start, endOfNonWhite-start);
	} else {
		return BGCPString();
	}
}

///////////////////////////////////////////////////////////////////////////////

MessageListener* logListener = nullptr;

void logNoLF(const BGCPString& str)
{
	if (logListener == nullptr) {
		str.print();
	} else {
		BGCPString copy = str; // copy needed: str is const, but receiveMessage's 3rd arg isn't
		logListener->receiveMessage(nullptr, MessageType::Log, &copy);
	}
}

void log(const BGCPString& str)
{
	BGCPString msgPlusEOL = str + BGCPString::EOL;
	logNoLF(msgPlusEOL);
}

void log(const BGCPString& str1, const BGCPString& str2)
{
	(str1+str2).printLn();
}

void setLogListener(MessageListener* newListener)
{
	logListener = newListener;
}

////////////////////////////////////////////////////////////////////////////////

BGCPString execDir;

// Set exec dir from string
// The only relative path currently supported is .
void setDataDir (const BGCPString& newDir, bool parse)
{
	execDir = newDir;
	if (parse) {
		int endChar = execDir.findLastChar(OSPathSep);
		if (endChar >= 0) {
			execDir.terminateStr(endChar);
		} else {
			setDataDir();
		}
	}
}

// Set exec dir from current directory
void setDataDir()
{
	#ifdef _WIN32
	const size_t bufSize = 2048;
	wchar_t buf[bufSize];
	GetCurrentDirectory(bufSize, buf);
	execDir = BGCPString(buf, CP_SYSTEM);
	#else
	const size_t bufSize = 2048;
	char buf[bufSize];
	getcwd(buf, bufSize);
	execDir = BGCPString(buf, CP_SYSTEM);
	#endif
}

const BGCPString& getExecDir()
{
	return execDir;
}

const BGCPString& getDataDir()
{
	return getExecDir();
}

bool isRelativePath (const PathChar* path)
{
   if (path == NULL) {
      return true; // treat empty path as relative
   }

   #ifdef _WIN32
   if (bgstrlen(path) >= 3) {
      if (path[1] == ':' && path[2] == '\\') {
         return false;
      } else {
         return true;
      }
   } else {
      return false;
   }
   #else
   return path[0] != '/';
   #endif
}


// Advance the position given by pos by count characters and return the new byte offset
size_t BGCPString::advance(size_t pos, size_t count) const
{
	while (count > 0 && pos < m_byteCount) {
		--count;
		pos += getCharSize(pos);
	}

	if (pos < m_byteCount) {
		return pos;
	} else {
		return m_byteCount;
	}
}

// Add str with a path separator
void BGCPString::addPathSep(const BGCPString& str)
{
	SET_SCOPE;
	add(&OSPathSep, CP_SYSTEM, sizeof(PathChar));
	operator+=(str);
	RETURN;
}

void BGCPString::addPathSep(const BGCPString& str1, const BGCPString& str2)
{
	SET_SCOPE;
	add(&OSPathSep, CP_SYSTEM, sizeof(PathChar));
	operator+=(str1);
	add(&OSPathSep, CP_SYSTEM, sizeof(PathChar));
	operator+=(str2);
	RETURN;
}

// Return the offset of the last occurrence of c if the string contains c, or -1 otherwise
int BGCPString::findLastChar(char c) const
{
	SET_SCOPE;
	int result = -1;
	size_t offset = 0;
	while (offset < m_byteCount) {
		if (charAtEquals(offset, c)) {
			result = offset;
		}
		offset += getCharSize(offset);
	}
		
	RETURN result;
}

// Test if the ASCII chars in other are equal to this string, disregarding case
bool BGCPString::equalsNoCase(const BGCPString& other) const
{
	SET_SCOPE;
	if ((getCP() == CP_UTF16_LE && other.getCP() == CP_UTF16_LE)
	||  (getCP() == CP_UTF16_BE && other.getCP() == CP_UTF16_BE))
	{
		if (getByteCount() != other.getByteCount()) {
			RETURN false;
		}
		size_t pos = 0;
		while (pos < getByteCount()) {
			char16_t c1 = get<char16_t>(pos);
			char16_t c2 = other.get<char16_t>(pos);
			if (c1 >= 'A' && c1 <= 'Z') {
				c1 = c1 -'A' + 'a';
			}
			if (c2 >= 'A' && c2 <= 'Z') {
				c2 = c2-'A' + 'a';
			}
			if (c1 != c2) {
				RETURN false;
			}
			pos += 2;
		}
		RETURN true; // No difference found means the strings are equal
	} else 
	if ((getCP() == CP_ASCII && other.getCP() == CP_ASCII)
	||   getCP() == CP_UTF8 && other.getCP() == CP_UTF8)
	{
		if (getByteCount() != other.getByteCount()) {
			RETURN false;
		}
		size_t pos = 0;
		while (pos < getByteCount()) {
			char c1 = get<char>(pos);
			char c2 = other.get<char>(pos);
			if (c1 >= 'A' && c1 <= 'Z') {
				c1 = c1 -'A' + 'a';
			}
			if (c2 >= 'A' && c2 <= 'Z') {
				c2 = c2-'A' + 'a';
			}
			if (c1 != c2) {
				RETURN false;
			}
			pos += 1;
		}
		RETURN true; // No difference found means the strings are equal
	} else {
		BGCPString a = lowerStringASCII();
		BGCPString b = other.lowerStringASCII();
		RETURN a == b;
	}
}

// Compare two substrings that must share a codepage
bool substrEquals(WORD cp, const BGCPString& str1, size_t pos1, const BGCPString& str2, size_t pos2, size_t len)
{
	if (cp == CP_UTF16_LE || cp == CP_UTF16_BE) {
		return bgstrcmp(str1.getPtr<char16_t>(pos1), str2.getPtr<char16_t>(pos2), len) == 0;
	} else {
		return bgstrcmp(str1.getPtr<char>(pos1), str2.getPtr<char>(pos2), len) == 0;
	}
}

// Return a new string where every occurrence of searhcStr was replaced with replaceStr
BGCPString BGCPString::replace(const BGCPString& searchStr, const BGCPString& replaceStr)
{
	SET_SCOPE;
	BGCPString convertedSearchStr = searchStr;
	if (!convertedSearchStr.convertToCP(getCP())) {
		// If searchStr cannot be converted to this codepage, there is nothing to replace
		RETURN *this;
	}

	SET_OP;
	BGCPString result;
	auto convertedSearchStrCC = convertedSearchStr.getCharCount();
	size_t pos = 0;
	while (pos < getByteCount()) {
		SET_OP;
		if (substrEquals(getCP(), *this, pos, convertedSearchStr, 0, convertedSearchStrCC)) {
			result += replaceStr;
			pos += convertedSearchStr.getByteCount();
		} else {
			auto cs = getCharSize(pos);
			result.appendPart(getPtr<char>(pos), cs, getCP());
			pos += cs;
		}
	}
	RETURN result;
}
