Add openFd, openFilePointer and openWithReadCallback methods

4 files changed, 297 insertions, 18 deletions

diff --git a/src/cc4group.c b/src/cc4group.c
index f3975ee..abf0b12 100644
--- a/src/cc4group.c
+++ b/src/cc4group.c
@@ -28,6 +28,7 @@
 
 #define SET_ERROR(errorCauser, errorCode, errorFormatter, data) do { this->error.code = errorCode; this->error.formatter.formatter = errorFormatter; this->error.causer = errorCauser; this->error.method = __func__; } while(0)
 #define SET_MESSAGE_ERROR(message) SET_ERROR(message, 1337, cc4group_messageFormatter, NULL)
+#define SET_EOD_ERROR(message) SET_MESSAGE_ERROR("Unexpected end of group data while " message)
 #define SET_MALFORMED_MESSAGE_ERROR(message) SET_MESSAGE_ERROR("The group file is malformed: " message)
 #define SET_ERRNO_ERROR(causer) SET_ERROR(causer, errno, cc4group_strerrorFormatter, NULL)
 #define SET_ZERROR_ERROR(causer, error) SET_ERROR(causer, error, cc4group_zerrorFormatter, (void*)errno)
@@ -96,6 +97,28 @@ typedef enum {
 	Reference
 } CC4Group_MemoryManagement;
 
+static bool cc4group_applyMemoryManagementStart(CC4Group_MemoryManagement const management, const uint8_t** data, size_t size)
+{
+	if(management == Copy)
+	{
+		uint8_t* copy = malloc(size);
+		if(copy == NULL)
+		{
+			return false;
+		}
+		memcpy(copy, data, size);
+		*data = copy;
+	}
+	return true;
+}
+
+static void cc4group_applyMemoryManagementEnd(CC4Group_MemoryManagement const management, const uint8_t* data)
+{
+	if(management == Copy || management == Take)
+	{
+		free((void*)data);
+	}
+}
 
 static const uint8_t* cc4group_getOnlyEntryData(CC4Group* const this, const C4GroupEntryData* entry);
 static const C4GroupEntryData* cc4group_getEntryByPath(const CC4Group* const this, const char* const entryPath);
@@ -339,8 +362,61 @@ static bool cc4group_buildGroupHierarchy(CC4Group* const this)
 	return ret;
 }
 
-static bool cc4group_uncompressGroup(CC4Group* const this, const uint8_t* const compressedData, size_t const size)
+static bool cc4group_inflateFillOutput(z_stream* const strm, CC4Group_ReadCallback const callback, void* const callbackArg, CC4Group_MemoryManagement const memoryManagement, bool* eof, const uint8_t** const lastData, int* inflateRet)
+{
+	int ret = Z_OK;
+	while(strm->avail_out > 0)
+	{
+		if(strm->avail_in == 0)
+		{
+			if(*eof)
+			{
+				*inflateRet = Z_OK;
+				return false;
+			}
+
+			if(*lastData != NULL)
+			{
+				cc4group_applyMemoryManagementEnd(memoryManagement, *lastData);
+				*lastData = NULL;
+			}
+
+			size_t readSize = 0;
+			*eof = callback(lastData, &readSize, callbackArg);
+
+			if(readSize == 0)
+			{
+				continue;
+			}
+
+			cc4group_applyMemoryManagementStart(memoryManagement, lastData, readSize);
+
+			strm->avail_in = readSize;
+			strm->next_in = (uint8_t*)*lastData; // I don't know why this must be non-const; anyway a read-only mapped input doesn't segfault. so it seems to be used read-only
+		}
+
+		ret = inflate(strm, *eof ? Z_FINISH : Z_NO_FLUSH);
+		if(ret != Z_OK && (ret != Z_STREAM_END || strm->avail_in != 0) && !(ret == Z_BUF_ERROR && strm->avail_out == 0))
+		{
+			break;
+		}
+	}
+
+	if(ret == Z_BUF_ERROR && strm->avail_out == 0)
+	{
+		return true;
+	}
+
+	*inflateRet = ret;
+	return ret == Z_OK || (ret == Z_STREAM_END && strm->avail_in == 0 && eof);
+}
+
+static bool cc4group_uncompressGroup(CC4Group* const this, CC4Group_ReadCallback const callback, void* const callbackArg, int const memoryManagement)
 {
+	assert(this);
+	assert(callback);
+	assert(callbackArg);
+
 	// only if the group is still empty
 	assert(this->root.children == NULL);
 
@@ -352,9 +428,58 @@ static bool cc4group_uncompressGroup(CC4Group* const this, const uint8_t* const
 	CC4Group_CleanupJob tmpCleanup;
 	uint8_t* mappedTmpFile = NULL;
 
+
+	size_t totalReadSize = 0;
+
+	const uint8_t* readData = NULL;
+	const uint8_t* readDataAfterMagic = NULL;
+	size_t readSize = 0;
+	bool eof = false;
+
+	uint8_t magic1, magic2;
+
+	while(!eof && totalReadSize < 2)
+	{
+		if(readData != NULL)
+		{
+			cc4group_applyMemoryManagementEnd(memoryManagement, readData);
+		}
+		readData = NULL;
+		readSize = 0;
+
+		eof = callback(&readData, &readSize, callbackArg);
+
+		if(readSize == 0)
+		{
+			continue;
+		}
+
+		cc4group_applyMemoryManagementStart(memoryManagement, &readData, readSize);
+
+		size_t newTotalReadSize = totalReadSize + readSize;
+		if(totalReadSize < 1 && newTotalReadSize > 0)
+		{
+			magic1 = readData[0];
+		}
+
+		if(totalReadSize < 2 && newTotalReadSize > 1)
+		{
+			magic2 = readData[1 - totalReadSize];
+			readDataAfterMagic = readData + (2 - totalReadSize);
+		}
+
+		totalReadSize = newTotalReadSize;
+	}
+
+	if(eof && totalReadSize < 2)
+	{
+		SET_EOD_ERROR("reading the group magic");
+		goto ret;
+	}
+
 	static uint8_t magic[] = {0x1f, 0x8b};
 
-	if((compressedData[0] != C4GroupMagic1 && compressedData[0] != magic[0]) || (compressedData[1] != C4GroupMagic2 && compressedData[1] != magic[1]))
+	if((magic1 != C4GroupMagic1 && magic1 != magic[0]) || (magic2 != C4GroupMagic2 && magic2 != magic[1]))
 	{
 		SET_MESSAGE_ERROR("The file is not a valid group file. Magic bytes don't match.");
 		goto ret;
@@ -396,13 +521,22 @@ static bool cc4group_uncompressGroup(CC4Group* const this, const uint8_t* const
 		goto ret;
 	}
 
-	strm.next_in = (uint8_t*)compressedData + 2; // I don't know why this must be non-const; anyway a read-only mapped input doesn't segfault. so it seems to be used read-only
-	strm.avail_in = size - 2;
+	if(totalReadSize > 2)
+	{
+		strm.next_in = (uint8_t*)readDataAfterMagic;
+		strm.avail_in = totalReadSize - 2;
+	}
 
-	ret = inflate(&strm, Z_SYNC_FLUSH);
-	if(ret != Z_OK && (ret != Z_STREAM_END || strm.avail_in != 0))
+	if(!cc4group_inflateFillOutput(&strm, callback, callbackArg, memoryManagement, &eof, &readData, &ret))
 	{
-		SET_ZERROR_ERROR("inflate: inflating the group header", ret);
+		if(eof && ret == Z_BUF_ERROR)
+		{
+			SET_EOD_ERROR("inflating group header");
+		}
+		else
+		{
+			SET_ZERROR_ERROR("inflate: inflating the group header", ret);
+		}
 		goto ret;
 	}
 
@@ -412,7 +546,7 @@ static bool cc4group_uncompressGroup(CC4Group* const this, const uint8_t* const
 	{
 		// the group is empty
 		// so the stream should have ended also
-		if(ret != Z_STREAM_END)
+		if(ret != Z_STREAM_END || !eof || strm.avail_in > 0)
 		{
 			SET_MALFORMED_MESSAGE_ERROR("The group is empty but the gzip stream has not ended yet.");
 			goto ret;
@@ -453,10 +587,16 @@ static bool cc4group_uncompressGroup(CC4Group* const this, const uint8_t* const
 	strm.next_out = (Bytef*)cores;
 	strm.avail_out = sizeof(C4GroupEntryCore) * header->Entries;
 
-	ret = inflate(&strm, Z_SYNC_FLUSH);
-	if(ret != Z_OK)
+	if(!cc4group_inflateFillOutput(&strm, callback, callbackArg, memoryManagement, &eof, &readData, &ret))
 	{
-		SET_ZERROR_ERROR("inflate: inflating the group entry dictionary", ret);
+		if(eof && ret == Z_BUF_ERROR)
+		{
+			SET_EOD_ERROR("inflating the group entry dictionary");
+		}
+		else
+		{
+			SET_ZERROR_ERROR("inflate: inflating the group entry dictionary", ret);
+		}
 		goto ret;
 	}
 
@@ -475,16 +615,33 @@ static bool cc4group_uncompressGroup(CC4Group* const this, const uint8_t* const
 	strm.next_out = data;
 	strm.avail_out = uncompressedSize;
 
-	ret = inflate(&strm, Z_SYNC_FLUSH);
-	if(ret != Z_STREAM_END || strm.avail_in != 0 || strm.avail_out != 0)
+	if(!cc4group_inflateFillOutput(&strm, callback, callbackArg, memoryManagement, &eof, &readData, &ret))
+	{
+		if(eof && ret == Z_BUF_ERROR)
+		{
+			SET_EOD_ERROR("inflating the group contents");
+		}
+		else
+		{
+			SET_ZERROR_ERROR("inflate: inflating the group contents", ret);
+		}
+		goto ret;
+	}
+
+	if(strm.avail_in > 0 || ret != Z_STREAM_END)
 	{
-		SET_ZERROR_ERROR("inflate: inflating group contents", ret);
+		SET_MALFORMED_MESSAGE_ERROR("The group contents are read completely but more data is left to read");
 		goto ret;
 	}
 
 	retData = mappedTmpFile;
 
 ret:
+	if(readData != NULL)
+	{
+		cc4group_applyMemoryManagementEnd(memoryManagement, readData);
+	}
+
 	if(header != NULL)
 	{
 		free(header);
@@ -514,13 +671,69 @@ ret:
 	return cc4group_buildGroupHierarchy(this);
 }
 
-static bool cc4group_openMemory(CC4Group* const this, const uint8_t* const compressedData, size_t const size)
+typedef struct {
+	const uint8_t* data;
+	size_t size;
+} CompleteDataReadCallbackArg;
+
+static bool cc4group_completeDataReadCallback(const uint8_t** const data, size_t* const size, void* callbackArg)
+{
+	CompleteDataReadCallbackArg* argData = callbackArg;
+	*data = argData->data;
+	*size = argData->size;
+	return true;
+}
+
+static bool cc4group_readFdReadCallback(const uint8_t** const data, size_t* const size, void* callbackArg)
+{
+#define CHUNK_SIZE 1000*1000
+	uint8_t* readData = malloc(CHUNK_SIZE);
+	if(data == NULL)
+	{
+		return true;
+	}
+
+	ssize_t count = read(*(int*)callbackArg, readData, CHUNK_SIZE);
+
+	if(count > 0)
+	{
+		*data = readData;
+		*size = count;
+		return false;
+	}
+	return true;
+#undef CHUNK_SIZE
+}
+
+static bool cc4group_readFilePointerReadCallback(const uint8_t** const data, size_t* const size, void* callbackArg)
+{
+#define CHUNK_SIZE 1000*1000
+	uint8_t* readData = malloc(CHUNK_SIZE);
+	if(data == NULL)
+	{
+		return true;
+	}
+
+	size_t count = fread(readData, 1, CHUNK_SIZE, callbackArg);
+
+	if(count > 0)
+	{
+		*data = readData;
+		*size = count;
+		return false;
+	}
+	return true;
+#undef CHUNK_SIZE
+}
+
+static bool cc4group_openMemory(CC4Group* const this, const uint8_t* const compressedData, size_t const size, int const memoryManagement)
 {
 	assert(this);
 	assert(compressedData);
 	assert(size);
 
-	return cc4group_uncompressGroup(this, compressedData, size);
+	CompleteDataReadCallbackArg data = {.data = compressedData, .size = size};
+	return cc4group_uncompressGroup(this, cc4group_completeDataReadCallback, &data, memoryManagement);
 }
 
 static bool cc4group_setSubRoot(CC4Group* const this, const char* const subPath)
@@ -618,7 +831,8 @@ static bool cc4group_uncompressGroupFromFile(CC4Group* const this, const char* c
 		goto ret;
 	}
 
-	success = cc4group_uncompressGroup(this, mappedFile, size);
+	CompleteDataReadCallbackArg data = {.data = mappedFile, .size = size};
+	success = cc4group_uncompressGroup(this, cc4group_completeDataReadCallback, &data, Reference);
 
 ret:
 	if(mappedFile != MAP_FAILED)
@@ -826,9 +1040,25 @@ static bool cc4group_create(CC4Group* const this)
 	return true;
 }
 
+static bool cc4group_openFd(CC4Group* const this, int fd)
+{
+	return cc4group_uncompressGroup(this, cc4group_readFdReadCallback, &fd, Take);
+}
+
+static bool cc4group_openFilePointer(CC4Group* const this, FILE* file)
+{
+	return cc4group_uncompressGroup(this, cc4group_readFilePointerReadCallback, file, Take);
+}
+
 static bool cc4group_openExisting(CC4Group* const this, const char* const path)
 {
 	assert(this);
+	assert(path);
+
+	if(strcmp(path, "-") == 0)
+	{
+		return cc4group_openFd(this, STDIN_FILENO);
+	}
 
 	return cc4group_uncompressGroupFromFile(this, path);
 }
@@ -1688,6 +1918,9 @@ CC4Group_API cc4group = {
 	.delete = cc4group_delete,
 	.openExisting = cc4group_openExisting,
 	.openMemory = cc4group_openMemory,
+	.openFd = cc4group_openFd,
+	.openFilePointer = cc4group_openFilePointer,
+	.openWithReadCallback = cc4group_uncompressGroup,
 	.save = cc4group_save,
 	.saveOverwrite = cc4group_saveOverwrite,
 	.extractAll = cc4group_extractAll,
diff --git a/src/cc4group.h b/src/cc4group.h
index 16c684a..a63d995 100644
--- a/src/cc4group.h
+++ b/src/cc4group.h
@@ -10,6 +10,7 @@ extern "C" {
 #include <stddef.h>
 #include <stdint.h>
 #include <stdbool.h>
+#include <stdio.h>
 
 typedef struct {
 	const char* fileName;
@@ -31,6 +32,11 @@ typedef struct {
 } CC4Group_CleanupJob;
 typedef void* (*CC4Group_TmpMemoryStrategy)(CC4Group* const this, const size_t size, CC4Group_CleanupJob* cleanupJob);
 
+// the callback has to store a pointer to the newly read data in data, as well as the amount of read data in size
+// the callback must return true if the end of data is reached or any read error happens and false otherwise
+// the pointer passed in will be handled as specified with the corresponding MemoryManagement
+typedef bool (*CC4Group_ReadCallback)(const uint8_t** const data, size_t* const size, void* const arg);
+
 typedef struct {
 	CC4Group* (*new)(void);
 	bool (*create)(CC4Group* const this);
@@ -42,7 +48,20 @@ typedef struct {
 
 	// opens a group that is stored entirely in memory
 	// only open an in-memory group on a frehsly created group object
-	bool (*openMemory)(CC4Group* const this, const uint8_t* const groupData, size_t const size);
+	// see the description of MemoryManagement to know if and when data has to be freed by the caller
+	// if the lazy mode is not used, the data can be freed immediately after this function returns
+	bool (*openMemory)(CC4Group* const this, const uint8_t* const groupData, size_t const size, int const memoryManagement);
+
+	// opens a group through a file descriptor
+	// the file descriptor must have been opened with read access
+	bool (*openFd)(CC4Group* const this, int fd);
+
+	// opens a group through a FILE*
+	// the file must have been opened with read access
+	bool (*openFilePointer)(CC4Group* const this, FILE* fd);
+
+	// opens a group and calls the callback to get the group data
+	bool (*openWithReadCallback)(CC4Group* const this, CC4Group_ReadCallback const callback, void* const callbackArg, int const memoryManagement);
 
 	bool (*save)(CC4Group* const this, const char* const path);
 	bool (*saveOverwrite)(CC4Group* const this, const char* const path);
diff --git a/src/cppc4group.cpp b/src/cppc4group.cpp
index d8bfbb3..2768afd 100644
--- a/src/cppc4group.cpp
+++ b/src/cppc4group.cpp
@@ -76,6 +76,26 @@ bool CppC4Group::openExisting(const std::string& path)
 	return cc4group.openExisting(p->g, path.c_str());
 }
 
+bool CppC4Group::openFd(const int fd)
+{
+	return cc4group.openFd(p->g, fd);
+}
+
+bool CppC4Group::openFilePointer(FILE* file)
+{
+	return cc4group.openFilePointer(p->g, file);
+}
+
+bool CppC4Group::openMemory(const void* const data, const size_t size, const MemoryManagement management)
+{
+	return cc4group.openMemory(p->g, reinterpret_cast<const uint8_t* const>(data), size, convertMemoryManagement(management));
+}
+
+bool CppC4Group::openWithReadCallback(const ReadCallback callback, void* const callbackArg, const MemoryManagement management)
+{
+	return cc4group.openWithReadCallback(p->g, reinterpret_cast<CC4Group_ReadCallback>(callback), callbackArg, convertMemoryManagement(management));
+}
+
 bool CppC4Group::save(const std::string& path, const bool overwrite)
 {
 	return (overwrite ? cc4group.saveOverwrite : cc4group.save)(p->g, path.c_str());
diff --git a/src/cppc4group.hpp b/src/cppc4group.hpp
index 67d350d..0787c97 100644
--- a/src/cppc4group.hpp
+++ b/src/cppc4group.hpp
@@ -3,6 +3,7 @@
 #include <memory>
 #include <vector>
 #include <optional>
+#include <cstdio>
 
 class CppC4Group {
 	struct Private;
@@ -30,6 +31,8 @@ public:
 	};
 
 public:
+	using ReadCallback = bool(*)(const void** const data, size_t* const size, void* const arg);
+
 	enum TmpMemoryStrategy {
 		Memory,
 		File,
@@ -50,6 +53,10 @@ public:
 
 	void create();
 	bool openExisting(const std::string& path);
+	bool openFd(const int fd);
+	bool openFilePointer(FILE* file);
+	bool openMemory(const void* const data, const size_t size, const MemoryManagement management = Reference);
+	bool openWithReadCallback(const ReadCallback callback, void* const callbackArg, const MemoryManagement management = Take);
 	bool save(const std::string& path, const bool overwrite = false);
 
 	bool extractAll(const std::string& path);