Efficient HTTP file download by chunks in Unreal Engine C++

Chunk downloading is a technique used to retrieve large binary data from the server in separate parts, ensuring reliability and compatibility across different platforms. Unreal Engine’s HTTP module has a limitation of 2GB for binary HTTP response content due to internal restrictions (specifically, TArray<uint8> uses the int32 size type, which has a maximum value of 2,147,483,647, approximately 2 GB in our case). To overcome this limitation, we can use the Range HTTP header supported by most servers, without requiring any file preparation or segmentation.

As a more generic example, you can take a look at the recent update of my open-source plugin, RuntimeFilesDownloader . This plugin provides functionality to download files larger than 2GB to both memory and storage, with the only exception being Blueprints that don’t support TArray64<uint8>, and thus, it only has the functionality to save >2GB of data to storage. You can view the relevant source code here .

Another implementation where chunk downloading is used is my recent open-source plugin, RuntimeSpeechRecognizer , which includes a feature to download language model files larger than 2GB through the Unreal Engine Editor.

Below is a simplified code snippet for downloading files by chunks. In this example, the MaxChunkSize is set to 1048576 bytes (1MB), but you can modify it according to your specific requirements.

Simplified code snippet

Header file:

#include "CoreMinimal.h"
#include "Http.h"
#include "Templates/SharedPointer.h"
#include "Async/Future.h"

UCLASS(BlueprintType)
class YOURMODULE_API UDownloaderExample : public UObject
{
	GENERATED_BODY()

public:
	UFUNCTION(BlueprintCallable)
	void DownloadFunctionExample_BP(const FString& URL, const FString& SavePath);
	
	virtual TFuture<TArray64<uint8>> DownloadFile(const FString& URL);
	virtual TFuture<TArray64<uint8>> DownloadFileByChunk(const FString& URL, int64 ContentSize, int64 MaxChunkSize, FInt64Vector2 InternalContentRange = FInt64Vector2(), TArray64<uint8>&& InternalResultData = TArray64<uint8>());

protected:
	TWeakPtr<IHttpRequest, ESPMode::ThreadSafe> HttpRequestPtr;
	TFuture<int64> GetContentSize(const FString& URL);
};

Source file:

void UDownloaderExample::DownloadFunctionExample_BP(const FString& URL, const FString& SavePath)
{
	DownloadFile(URL).Next([this, SavePath](TArray64<uint8>&& ResultData)
	{
		UE_LOG(LogTemp, Display, TEXT("Downloaded %lld bytes"), ResultData.Num());
		FFileHelper::SaveArrayToFile(ResultData, *SavePath);
	});
}

TFuture<TArray64<uint8>> UDownloaderExample::DownloadFile(const FString& URL)
{
	TSharedRef<TPromise<TArray64<uint8>>> PromisePtr = MakeShared<TPromise<TArray64<uint8>>>();
	GetContentSize(URL).Next([WeakThis = MakeWeakObjectPtr(this), PromisePtr, URL](int64 ContentSize)
	{
		if (!WeakThis.IsValid())
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to download file: this is no longer valid"));
			PromisePtr->SetValue(TArray64<uint8>());
			return;
		}

		if (ContentSize <= 0)
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to download file: content size is 0"));
			PromisePtr->SetValue(TArray64<uint8>());
			return;
		}

		// 1048576 bytes = 1 MB. Just an arbitrary number to use as the chunk size
		WeakThis->DownloadFileByChunk(URL, ContentSize, 1048576).Next([PromisePtr](TArray64<uint8>&& ResultData)
		{
			PromisePtr->SetValue(MoveTemp(ResultData));
		});
	});
	return PromisePtr->GetFuture();
}

TFuture<TArray64<uint8>> UDownloaderExample::DownloadFileByChunk(const FString& URL, int64 ContentSize, int64 MaxChunkSize, FInt64Vector2 InternalContentRange, TArray64<uint8>&& InternalResultData)
{
	if (MaxChunkSize <= 0)
	{
		UE_LOG(LogTemp, Error, TEXT("Failed to download chunk from %s: MaxChunkSize is 0"), *URL);
		return MakeFulfilledPromise<TArray64<uint8>>(TArray64<uint8>()).GetFuture();
	}

	// If the InternalResultData was not provided, initialize it to the size of the file
	if (InternalResultData.Num() <= 0)
	{
		InternalResultData.SetNumUninitialized(ContentSize);
	}

	// If the InternalContentRange was not provided, set it to the first chunk of size MaxChunkSize
	if (InternalContentRange.X == 0 && InternalContentRange.Y == 0)
	{
		InternalContentRange.Y = FMath::Min(ContentSize, MaxChunkSize) - 1;
	}

	TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequestRef = FHttpModule::Get().CreateRequest();
	HttpRequestRef->SetVerb("GET");
	HttpRequestRef->SetURL(URL);

	const FString RangeHeaderValue = FString::Format(TEXT("bytes={0}-{1}"), {InternalContentRange.X, InternalContentRange.Y});
	HttpRequestRef->SetHeader(TEXT("Range"), RangeHeaderValue);

	TSharedRef<TPromise<TArray64<uint8>>> PromisePtr = MakeShared<TPromise<TArray64<uint8>>>();
	HttpRequestRef->OnProcessRequestComplete().BindLambda([WeakThis = MakeWeakObjectPtr(this), PromisePtr, URL, ContentSize, MaxChunkSize, InternalContentRange, InternalResultData = MoveTemp(InternalResultData)](FHttpRequestPtr Request, FHttpResponsePtr Response, bool bSuccess) mutable
	{
		if (!WeakThis.IsValid())
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to download chunk from %s: this is no longer valid"), *Request->GetURL());
			PromisePtr->SetValue(TArray64<uint8>());
			return;
		}

		if (!bSuccess || !Response.IsValid())
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to download chunk from %s: request failed"), *Request->GetURL());
			PromisePtr->SetValue(TArray64<uint8>());
			return;
		}

		if (Response->GetContentSize() <= 0)
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to download chunk from %s: content size is 0"), *Request->GetURL());
			PromisePtr->SetValue(TArray64<uint8>());
			return;
		}

		const int64 DataOffset = InternalContentRange.X;
		const TArray<uint8>& ResponseContent = Response->GetContent();

		// Calculate the overall size of the downloaded content in the result buffer
		const int64 OverallDownloadedSize = DataOffset + ResponseContent.Num();

		// Check if some values are out of range
		{
			if (DataOffset < 0 || DataOffset >= InternalResultData.Num())
			{
				UE_LOG(LogTemp, Error, TEXT("Failed to download chunk from %s: data offset is out of range"), *Request->GetURL());
				PromisePtr->SetValue(TArray64<uint8>());
				return;
			}

			if (OverallDownloadedSize > InternalResultData.Num())
			{
				UE_LOG(LogTemp, Error, TEXT("Failed to download chunk from %s: overall downloaded size is out of range"), *Request->GetURL());
				PromisePtr->SetValue(TArray64<uint8>());
				return;
			}
		}

		FMemory::Memcpy(InternalResultData.GetData() + DataOffset, ResponseContent.GetData(), ResponseContent.Num());

		// Check if there's still more content to download
		if (OverallDownloadedSize < ContentSize)
		{
			// Calculate how much more data needs to be downloaded in the next chunk
			const int64 BytesRemaining = ContentSize - OverallDownloadedSize;
			const int64 BytesToDownload = FMath::Min(BytesRemaining, MaxChunkSize);

			// Calculate the range of data to download in the next chunk
			const FInt64Vector2 NewContentRange = FInt64Vector2(OverallDownloadedSize, OverallDownloadedSize + BytesToDownload - 1);

			// Initiate the next download chunk
			WeakThis->DownloadFileByChunk(URL, ContentSize, MaxChunkSize, NewContentRange, MoveTemp(InternalResultData)).Next([PromisePtr](TArray64<uint8>&& ResultData)
			{
				PromisePtr->SetValue(MoveTemp(ResultData));
			});
		}
		else
		{
			// If there is no more content to download, then the download is complete
			PromisePtr->SetValue(MoveTemp(InternalResultData));
		}
	});

	HttpRequestRef->ProcessRequest();
	HttpRequestPtr = HttpRequestRef;
	return PromisePtr->GetFuture();
}

TFuture<int64> UDownloaderExample::GetContentSize(const FString& URL)
{
	TSharedPtr<TPromise<int64>> PromisePtr = MakeShared<TPromise<int64>>();
	TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequestRef = FHttpModule::Get().CreateRequest();

	HttpRequestRef->SetVerb("HEAD");
	HttpRequestRef->SetURL(URL);

	HttpRequestRef->SetTimeout(5.0f);

	HttpRequestRef->OnProcessRequestComplete().BindLambda([PromisePtr](const FHttpRequestPtr& Request, const FHttpResponsePtr& Response, const bool bSucceeded)
	{
		const int64 ContentSize = FCString::Atoi64(Response.IsValid() ? *Response->GetHeader("Content-Length") : TEXT("0"));
		if (ContentSize <= 0)
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to get size: content size is 0"));
			PromisePtr->SetValue(0);
			return;
		}
		PromisePtr->SetValue(ContentSize);
	});

	if (!HttpRequestRef->ProcessRequest())
	{
		UE_LOG(LogTemp, Error, TEXT("Failed to get size: request failed"));
		return MakeFulfilledPromise<int64>(0).GetFuture();
	}

	HttpRequestPtr = HttpRequestRef;
	return PromisePtr->GetFuture();
}