Compression class manages the opening and extraction of compressed files (zip, tar, tar.gz). More...

Static Public Member Functions
static bool	ZipData (string zipPath, Dictionary< string, string > filenamesAndData)
	Create a zip file of the supplied file names and string data source More...

static bool	ZipData (string zipPath, IEnumerable< KeyValuePair< string, byte[]>> filenamesAndData)
	Create a zip file of the supplied file names and data using a byte array More...

static bool	ZipData (string zipPath, string zipEntry, IEnumerable< string > lines)
	Zips the specified lines of text into the zipPath More...

static bool	ZipCreateAppendData (string path, string entry, string data, bool overrideEntry=false)
	Append the zip data to the file-entry specified. More...

static bool	ZipCreateAppendData (string path, string entry, byte[] data, bool overrideEntry=false)
	Append the zip data to the file-entry specified. More...

static Dictionary< string, string >	UnzipData (byte[] zipData, Encoding encoding=null)
	Uncompress zip data byte array into a dictionary string array of filename-contents. More...

static async Task< Dictionary< string, string > >	UnzipDataAsync (Stream stream, Encoding encoding=null)
	Uncompress zip data byte array into a dictionary string array of filename-contents. More...

static byte[]	ZipBytes (byte[] bytes, string zipEntryName)
	Performs an in memory zip of the specified bytes More...

static async Task	ZipBytesAsync (Stream target, byte[] data, string zipEntryName, ZipArchiveMode? mode=null, CompressionLevel? compressionLevel=null)
	Performs an in memory zip of the specified bytes in the target stream More...

static async Task	ZipBytesAsync (Stream target, IEnumerable< KeyValuePair< byte[], string >> data, ZipArchiveMode? mode=null, CompressionLevel? compressionLevel=null)
	Performs an in memory zip of the specified bytes in the target stream More...

static async Task	ZipStreamsAsync (string target, IEnumerable< KeyValuePair< string, Stream >> data, ZipArchiveMode? mode=null, CompressionLevel? compressionLevel=null)
	Performs an in memory zip of the specified stream in the target stream More...

static async Task	ZipStreamsAsync (Stream target, IEnumerable< KeyValuePair< string, Stream >> data, ZipArchiveMode? mode=null, CompressionLevel? compressionLevel=null, bool leaveStreamOpen=false)
	Performs an in memory zip of the specified stream in the target stream More...

static string	UnGZip (string gzipFileName, string targetDirectory)
	Extract .gz files to disk More...

static string	Zip (string textPath, string zipEntryName, bool deleteOriginal=true)
	Compress a given file and delete the original file. Automatically rename the file to name.zip. More...

static void	Zip (string source, string destination, string zipEntryName, bool deleteOriginal)
	Compresses the specified source file. More...

static string	Zip (string textPath, bool deleteOriginal=true)
	Compress a given file and delete the original file. Automatically rename the file to name.zip. More...

static void	Zip (string data, string zipPath, string zipEntry)
	Compress given data to the path given More...

static bool	ZipDirectory (string directory, string destination, bool includeRootInZip=true)
	Zips the specified directory, preserving folder structure More...

static bool	Unzip (string zip, string directory, bool overwrite=false)
	Unzips the specified zip file to the specified directory More...

static void	ZipFiles (string destination, IEnumerable< string > files)
	Zips all files specified to a new zip at the destination path More...

static StreamReader	Unzip (string filename, out ZipFile zip)
	Streams a local zip file using a streamreader. Important: the caller must call Dispose() on the returned ZipFile instance. More...

static StreamReader	Unzip (string filename, string zipEntryName, out ZipFile zip)
	Streams a local zip file using a streamreader. Important: the caller must call Dispose() on the returned ZipFile instance. More...

static IEnumerable< KeyValuePair< string, List< string > > >	Unzip (string filename)
	Streams the unzipped file as key value pairs of file name to file contents. NOTE: When the returned enumerable finishes enumerating, the zip stream will be closed rendering all key value pair Value properties unaccessible. Ideally this would be enumerated depth first. More...

static IEnumerable< KeyValuePair< string, List< string > > >	Unzip (Stream stream)
	Lazily unzips the specified stream More...

static List< string >	ReadLines (string filename)
	Streams each line from the first zip entry in the specified zip file More...

static StreamReader	UnzipStreamToStreamReader (Stream zipstream)
	Unzip a local file and return its contents via streamreader: More...

static Stream	UnzipStream (Stream zipstream, out ZipFile zipFile, string entryName=null)
	Unzip a stream that represents a zip file and return the first entry as a stream More...

static List< string >	UnzipToFolder (byte[] zipData, string outputFolder)
	Unzip the given byte array and return the created file names. More...

static List< string >	UnzipToFolder (string zipFile)
	Unzip a local file and return the created file names More...

static void	UnTarFiles (string source, string destination)
	Extracts all file from a zip archive and copies them to a destination folder. More...

static void	UnTarGzFiles (string source, string destination)
	Extract tar.gz files to disk More...

static IEnumerable< KeyValuePair< string, byte[]> >	UnTar (Stream stream, bool isTarGz)
	Enumerate through the files of a TAR and get a list of KVP names-byte arrays More...

static IEnumerable< KeyValuePair< string, byte[]> >	UnTar (string source)
	Enumerate through the files of a TAR and get a list of KVP names-byte arrays. More...

static bool	ValidateZip (string path)
	Validates whether the zip is corrupted or not More...

static IEnumerable< string >	GetZipEntryFileNames (string zipFileName)
	Returns the entry file names contained in a zip file More...

static IEnumerable< string >	GetZipEntryFileNames (Stream zipFileStream)
	Return the entry file names contained in a zip file More...

static void	Extract7ZipArchive (string inputFile, string outputDirectory, int execTimeout=60000)
	Extracts a 7-zip archive to disk, using the 7-zip CLI utility More...

Detailed Description

Compression class manages the opening and extraction of compressed files (zip, tar, tar.gz).

QuantConnect's data library is stored in zip format locally on the hard drive.

Definition at line 40 of file Compression.cs.

Member Function Documentation

◆ ZipData() [1/3]

static bool QuantConnect.Compression.ZipData	(	string	zipPath,
		Dictionary< string, string >	filenamesAndData
	)

static

Create a zip file of the supplied file names and string data source

Parameters

zipPath	Output location to save the file.
filenamesAndData	File names and data in a dictionary format.

Returns: True on successfully creating the zip file.

Definition at line 60 of file Compression.cs.

Here is the call graph for this function:

◆ ZipData() [2/3]

static bool QuantConnect.Compression.ZipData	(	string	zipPath,
		IEnumerable< KeyValuePair< string, byte[]>>	filenamesAndData
	)

static

Create a zip file of the supplied file names and data using a byte array

Parameters

zipPath	Output location to save the file.
filenamesAndData	File names and data in a dictionary format.

Returns: True on successfully saving the file

Definition at line 98 of file Compression.cs.

Here is the call graph for this function:

◆ ZipData() [3/3]

static bool QuantConnect.Compression.ZipData	(	string	zipPath,
		string	zipEntry,
		IEnumerable< string >	lines
	)

static

Zips the specified lines of text into the zipPath

Parameters

zipPath	The destination zip file path
zipEntry	The entry name in the zip
lines	The lines to be written to the zip

Returns: True if successful, otherwise false

Definition at line 147 of file Compression.cs.

Here is the call graph for this function:

◆ ZipCreateAppendData() [1/2]

static bool QuantConnect.Compression.ZipCreateAppendData	(	string	path,
		string	entry,
		string	data,
		bool	overrideEntry = `false`
	)

static

Append the zip data to the file-entry specified.

Parameters

path	The zip file path
entry	The entry name
data	The entry data
overrideEntry	True if should override entry if it already exists

Returns: True on success

Definition at line 178 of file Compression.cs.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ZipCreateAppendData() [2/2]

static bool QuantConnect.Compression.ZipCreateAppendData	(	string	path,
		string	entry,
		byte[]	data,
		bool	overrideEntry = `false`
	)

static

Append the zip data to the file-entry specified.

Parameters

path	The zip file path
entry	The entry name
data	The entry data
overrideEntry	True if should override entry if it already exists

Returns: True on success

Definition at line 210 of file Compression.cs.

Here is the call graph for this function:

◆ UnzipData()

static Dictionary<string, string> QuantConnect.Compression.UnzipData	(	byte[]	zipData,
		Encoding	encoding = `null`
	)

static

Uncompress zip data byte array into a dictionary string array of filename-contents.

Parameters

zipData	Byte data array of zip compressed information
encoding	Specifies the encoding used to read the bytes. If not specified, defaults to ASCII

Returns: Uncompressed dictionary string-sting of files in the zip

Definition at line 240 of file Compression.cs.

Here is the call graph for this function:

◆ UnzipDataAsync()

static async Task<Dictionary<string, string> > QuantConnect.Compression.UnzipDataAsync	(	Stream	stream,
		Encoding	encoding = `null`
	)

static

Uncompress zip data byte array into a dictionary string array of filename-contents.

Parameters

stream	Stream data of zip compressed information
encoding	Specifies the encoding used to read the bytes. If not specified, defaults to ASCII

Returns: Uncompressed dictionary string-sting of files in the zip

Definition at line 252 of file Compression.cs.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ZipBytes()

static byte [] QuantConnect.Compression.ZipBytes	(	byte[]	bytes,
		string	zipEntryName
	)

static

Performs an in memory zip of the specified bytes

Parameters

bytes	The file contents in bytes to be zipped
zipEntryName	The zip entry name

Returns: The zipped file as a byte array

Definition at line 298 of file Compression.cs.

Here is the call graph for this function:

◆ ZipBytesAsync() [1/2]

static async Task QuantConnect.Compression.ZipBytesAsync	(	Stream	target,
		byte[]	data,
		string	zipEntryName,
		ZipArchiveMode?	mode = `null`,
		CompressionLevel?	compressionLevel = `null`
	)

static

Performs an in memory zip of the specified bytes in the target stream

Parameters

target	The target stream
data	The file contents in bytes to be zipped
zipEntryName	The zip entry name
mode	The archive mode
compressionLevel	The desired compression level

Returns: The zipped file as a byte array

Definition at line 314 of file Compression.cs.

Here is the caller graph for this function:

◆ ZipBytesAsync() [2/2]

static async Task QuantConnect.Compression.ZipBytesAsync	(	Stream	target,
		IEnumerable< KeyValuePair< byte[], string >>	data,
		ZipArchiveMode?	mode = `null`,
		CompressionLevel?	compressionLevel = `null`
	)

static

Performs an in memory zip of the specified bytes in the target stream

Parameters

target	The target stream
data	The file contents in bytes to be zipped
mode	The archive mode
compressionLevel	The desired compression level

Returns: The zipped file as a byte array

Definition at line 328 of file Compression.cs.

◆ ZipStreamsAsync() [1/2]

static async Task QuantConnect.Compression.ZipStreamsAsync	(	string	target,
		IEnumerable< KeyValuePair< string, Stream >>	data,
		ZipArchiveMode?	mode = `null`,
		CompressionLevel?	compressionLevel = `null`
	)

static

Performs an in memory zip of the specified stream in the target stream

Parameters

target	The target stream
data	The file contents in bytes to be zipped
mode	The archive mode
compressionLevel	The desired compression level

Returns: The zipped file as a byte array

Definition at line 349 of file Compression.cs.

◆ ZipStreamsAsync() [2/2]

static async Task QuantConnect.Compression.ZipStreamsAsync	(	Stream	target,
		IEnumerable< KeyValuePair< string, Stream >>	data,
		ZipArchiveMode?	mode = `null`,
		CompressionLevel?	compressionLevel = `null`,
		bool	leaveStreamOpen = `false`
	)

static

Performs an in memory zip of the specified stream in the target stream

Parameters

target	The target stream
data	The file contents in bytes to be zipped
mode	The archive mode
compressionLevel	The desired compression level
leaveStreamOpen	True to leave the taget stream open

Returns: The zipped file as a byte array

Definition at line 367 of file Compression.cs.

◆ UnGZip()

static string QuantConnect.Compression.UnGZip	(	string	gzipFileName,
		string	targetDirectory
	)

static

Extract .gz files to disk

Parameters

gzipFileName
targetDirectory

Definition at line 390 of file Compression.cs.

◆ Zip() [1/4]

static string QuantConnect.Compression.Zip	(	string	textPath,
		string	zipEntryName,
		bool	deleteOriginal = `true`
	)

static

Compress a given file and delete the original file. Automatically rename the file to name.zip.

Parameters

textPath	Path of the original file
zipEntryName	The name of the entry inside the zip file
deleteOriginal	Boolean flag to delete the original file after completion

Returns: String path for the new zip file

Definition at line 411 of file Compression.cs.

Here is the caller graph for this function:

◆ Zip() [2/4]

static void QuantConnect.Compression.Zip	(	string	source,
		string	destination,
		string	zipEntryName,
		bool	deleteOriginal
	)

static

Compresses the specified source file.

Parameters

source	The source file to be compressed
destination	The destination zip file path
zipEntryName	The zip entry name for the file
deleteOriginal	True to delete the source file upon completion

Definition at line 425 of file Compression.cs.

Here is the call graph for this function:

◆ Zip() [3/4]

static string QuantConnect.Compression.Zip	(	string	textPath,
		bool	deleteOriginal = `true`
	)

static

Compress a given file and delete the original file. Automatically rename the file to name.zip.

Parameters

textPath	Path of the original file
deleteOriginal	Boolean flag to delete the original file after completion

Returns: String path for the new zip file

Definition at line 466 of file Compression.cs.

Here is the call graph for this function:

◆ Zip() [4/4]

static void QuantConnect.Compression.Zip	(	string	data,
		string	zipPath,
		string	zipEntry
	)

static

Compress given data to the path given

Parameters

data	Data to write to zip
zipPath	Path to write to
zipEntry	Entry to save the data as

Definition at line 477 of file Compression.cs.

◆ ZipDirectory()

static bool QuantConnect.Compression.ZipDirectory	(	string	directory,
		string	destination,
		bool	includeRootInZip = `true`
	)

static

Zips the specified directory, preserving folder structure

Parameters

directory	The directory to be zipped
destination	The output zip file destination
includeRootInZip	True to include the root 'directory' in the zip, false otherwise

Returns: True on a successful zip, false otherwise

Definition at line 504 of file Compression.cs.

Here is the call graph for this function:

◆ Unzip() [1/5]

static bool QuantConnect.Compression.Unzip	(	string	zip,
		string	directory,
		bool	overwrite = `false`
	)

static

Unzips the specified zip file to the specified directory

Parameters

zip	The zip to be unzipped
directory	The directory to place the unzipped files
overwrite	Flag specifying whether or not to overwrite existing files

Definition at line 537 of file Compression.cs.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ZipFiles()

static void QuantConnect.Compression.ZipFiles	(	string	destination,
		IEnumerable< string >	files
	)

static

Zips all files specified to a new zip at the destination path

Definition at line 579 of file Compression.cs.

Here is the call graph for this function:

◆ Unzip() [2/5]

static StreamReader QuantConnect.Compression.Unzip	(	string	filename,
		out ZipFile	zip
	)

static

Streams a local zip file using a streamreader. Important: the caller must call Dispose() on the returned ZipFile instance.

Parameters

filename	Location of the original zip file
zip	The ZipFile instance to be returned to the caller

Returns: Stream reader of the first file contents in the zip file

Definition at line 616 of file Compression.cs.

Here is the call graph for this function:

◆ Unzip() [3/5]

static StreamReader QuantConnect.Compression.Unzip	(	string	filename,
		string	zipEntryName,
		out ZipFile	zip
	)

static

Streams a local zip file using a streamreader. Important: the caller must call Dispose() on the returned ZipFile instance.

Parameters

filename	Location of the original zip file
zipEntryName	The zip entry name to open a reader for. Specify null to access the first entry
zip	The ZipFile instance to be returned to the caller

Returns: Stream reader of the first file contents in the zip file

Definition at line 629 of file Compression.cs.

Here is the call graph for this function:

◆ Unzip() [4/5]

static IEnumerable<KeyValuePair<string, List<string> > > QuantConnect.Compression.Unzip ( string filename )

static

Streams the unzipped file as key value pairs of file name to file contents. NOTE: When the returned enumerable finishes enumerating, the zip stream will be closed rendering all key value pair Value properties unaccessible. Ideally this would be enumerated depth first.

This method has the potential for a memory leak if each kvp.Value enumerable is not disposed

Parameters

filename The zip file to stream

Returns: The stream zip contents

Definition at line 680 of file Compression.cs.

Here is the call graph for this function:

◆ Unzip() [5/5]

static IEnumerable<KeyValuePair<string, List<string> > > QuantConnect.Compression.Unzip ( Stream stream )

static

Lazily unzips the specified stream

Parameters

stream The zipped stream to be read

Returns: An enumerable whose elements are zip entry key value pairs with a key of the zip entry name and the value of the zip entry's file lines

Definition at line 705 of file Compression.cs.

◆ ReadLines()

static List<string> QuantConnect.Compression.ReadLines ( string filename )

static

Streams each line from the first zip entry in the specified zip file

Parameters

filename The zip file path to stream

Returns: An enumerable containing each line from the first unzipped entry

Definition at line 721 of file Compression.cs.

Here is the call graph for this function:

◆ UnzipStreamToStreamReader()

static StreamReader QuantConnect.Compression.UnzipStreamToStreamReader ( Stream zipstream )

static

Unzip a local file and return its contents via streamreader:

Definition at line 772 of file Compression.cs.

Here is the call graph for this function:

◆ UnzipStream()

static Stream QuantConnect.Compression.UnzipStream	(	Stream	zipstream,
		out ZipFile	zipFile,
		string	entryName = `null`
	)

static

Unzip a stream that represents a zip file and return the first entry as a stream

Definition at line 806 of file Compression.cs.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ UnzipToFolder() [1/2]

static List<string> QuantConnect.Compression.UnzipToFolder	(	byte[]	zipData,
		string	outputFolder
	)

static

Unzip the given byte array and return the created file names.

Parameters

zipData	A byte array containing the zip
outputFolder	The target output folder

Returns: List of unzipped file names

Definition at line 847 of file Compression.cs.

Here is the caller graph for this function:

◆ UnzipToFolder() [2/2]

static List<string> QuantConnect.Compression.UnzipToFolder ( string zipFile )

static

Unzip a local file and return the created file names

Parameters

zipFile Location of the zip on the HD

Returns: List of unzipped file names

Definition at line 858 of file Compression.cs.

Here is the call graph for this function:

◆ UnTarFiles()

static void QuantConnect.Compression.UnTarFiles	(	string	source,
		string	destination
	)

static

Extracts all file from a zip archive and copies them to a destination folder.

Parameters

source	The source zip file.
destination	The destination folder to extract the file to.

Definition at line 934 of file Compression.cs.

◆ UnTarGzFiles()

static void QuantConnect.Compression.UnTarGzFiles	(	string	source,
		string	destination
	)

static

Extract tar.gz files to disk

Parameters

source	Tar.gz source file
destination	Location folder to unzip to

Definition at line 948 of file Compression.cs.

◆ UnTar() [1/2]

static IEnumerable<KeyValuePair<string, byte[]> > QuantConnect.Compression.UnTar	(	Stream	stream,
		bool	isTarGz
	)

static

Enumerate through the files of a TAR and get a list of KVP names-byte arrays

Parameters

stream	The input tar stream
isTarGz	True if the input stream is a .tar.gz or .tgz

Returns: An enumerable containing each tar entry and it's contents

Definition at line 965 of file Compression.cs.

◆ UnTar() [2/2]

static IEnumerable<KeyValuePair<string, byte[]> > QuantConnect.Compression.UnTar ( string source )

static

Enumerate through the files of a TAR and get a list of KVP names-byte arrays.

Parameters

source

Returns

Definition at line 988 of file Compression.cs.

◆ ValidateZip()

static bool QuantConnect.Compression.ValidateZip ( string path )

static

Validates whether the zip is corrupted or not

Parameters

path	Path to the zip file

Returns: true if archive tests ok; false otherwise.

Definition at line 1023 of file Compression.cs.

◆ GetZipEntryFileNames() [1/2]

static IEnumerable<string> QuantConnect.Compression.GetZipEntryFileNames ( string zipFileName )

static

Returns the entry file names contained in a zip file

Parameters

zipFileName The zip file name

Returns: An IEnumerable of entry file names

Definition at line 1036 of file Compression.cs.

Here is the caller graph for this function:

◆ GetZipEntryFileNames() [2/2]

static IEnumerable<string> QuantConnect.Compression.GetZipEntryFileNames ( Stream zipFileStream )

static

Return the entry file names contained in a zip file

Parameters

zipFileStream Stream to the file

Returns: IEnumerable of entry file names

Definition at line 1049 of file Compression.cs.

◆ Extract7ZipArchive()

static void QuantConnect.Compression.Extract7ZipArchive	(	string	inputFile,
		string	outputDirectory,
		int	execTimeout = `60000`
	)

static

Extracts a 7-zip archive to disk, using the 7-zip CLI utility

Parameters

inputFile	Path to the 7z file
outputDirectory	Directory to output contents of 7z
execTimeout	Timeout in seconds for how long we should wait for the extraction to complete

Exceptions

Exception The extraction failed because of a timeout or the exit code was not 0

Definition at line 1064 of file Compression.cs.

The documentation for this class was generated from the following file:

Compression/Compression.cs

Static Public Member Functions

Detailed Description

Member Function Documentation

◆ ZipData() [1/3]

◆ ZipData() [2/3]

◆ ZipData() [3/3]

◆ ZipCreateAppendData() [1/2]

◆ ZipCreateAppendData() [2/2]

◆ UnzipData()

◆ UnzipDataAsync()

◆ ZipBytes()

◆ ZipBytesAsync() [1/2]

◆ ZipBytesAsync() [2/2]

◆ ZipStreamsAsync() [1/2]

◆ ZipStreamsAsync() [2/2]

◆ UnGZip()

◆ Zip() [1/4]

◆ Zip() [2/4]

◆ Zip() [3/4]

◆ Zip() [4/4]

◆ ZipDirectory()

◆ Unzip() [1/5]

◆ ZipFiles()

◆ Unzip() [2/5]

◆ Unzip() [3/5]

◆ Unzip() [4/5]

◆ Unzip() [5/5]

◆ ReadLines()

◆ UnzipStreamToStreamReader()

◆ UnzipStream()

◆ UnzipToFolder() [1/2]

◆ UnzipToFolder() [2/2]

◆ UnTarFiles()

◆ UnTarGzFiles()

◆ UnTar() [1/2]

◆ UnTar() [2/2]

◆ ValidateZip()

◆ GetZipEntryFileNames() [1/2]

◆ GetZipEntryFileNames() [2/2]

◆ Extract7ZipArchive()