Deserialize Raw Data - Revision history

Salkin at 12:59, 21 June 2023

2023-06-21T12:59:09Z

← Older revision		Revision as of 12:59, 21 June 2023
Line 18:		Line 18:
	ret=[]		ret=[]
	for i in range(int(len(data)/4)):		for i in range(int(len(data)/4)):
	#if (data[i4]==0 and data[i4+1]==0 and data[i4+2]==0 and data[i4+3]==0): continue # ~~uncomment this if you do not want to include~~ null values		if (data[i4]==0 and data[i4+1]==0 and data[i4+2]==0 and data[i4+3]==0): continue # remove null values
	ret.append(struct.unpack('<f',data[i4:i4+4])[0])		ret.append(struct.unpack('<f',data[i4:i4+4])[0])
	return ret		return ret

2022-11-09T10:26:51Z

← Older revision		Revision as of 10:26, 9 November 2022
Line 21:		Line 21:
	ret.append(struct.unpack('<f',data[i4:i4+4])[0])		ret.append(struct.unpack('<f',data[i4:i4+4])[0])
	return ret		return ret

			This simple example does not include a correct offset for the values. You may check our [https://github.com/nrodemund/sicdb/tree/main/Scripts/Unpack%20raw%20data unpack script] to find an example on how to compute the offset.

	<noinclude>== C# example ==</noinclude>		<noinclude>== C# example ==</noinclude>

2022-11-09T10:15:03Z

← Older revision		Revision as of 10:15, 9 November 2022
Line 4:		Line 4:

	The intermediate data, where minute values are not aggregated, has roughly about 240gb, and it is to be expected that is increases by about 15% in 2023 update. By aggregating and serializing these values the database can be compresssed to under 10gb. Nevertheless it complicates the deserialization of this data. While our software can export the minute values one-click, using it in raw data requires a bit of coding.		The intermediate data, where minute values are not aggregated, has roughly about 240gb, and it is to be expected that is increases by about 15% in 2023 update. By aggregating and serializing these values the database can be compresssed to under 10gb. Nevertheless it complicates the deserialization of this data. While our software can export the minute values one-click, using it in raw data requires a bit of coding.

			Refer to chapter Full Unpack to learn how to completely unpack minute data.

	<noinclude>== Encoding ==</noinclude>		<noinclude>== Encoding ==</noinclude>
Line 39:		Line 41:
	}		}

	<noinclude>== ~~Build up table~~ ==</noinclude>		<noinclude>== Full Unpack ==</noinclude>
	<includeonly>=== ~~Build up table~~ ===</includeonly>		<includeonly>=== Full Unpack ===</includeonly>

			We provide a simple unpack python script on our [https://github.com/nrodemund/sicdb/tree/main/Scripts/Unpack%20raw%20data github code repository].

2022-11-03T17:17:14Z

← Older revision		Revision as of 17:17, 3 November 2022
Line 23:		Line 23:
	<includeonly>=== C# example ===</includeonly>		<includeonly>=== C# example ===</includeonly>

	public static float[] GetRawValues(byte[] data)		public static float?[] GetRawValues(byte[] data)
	{		{
	byte[] buf = new byte[4];		byte[] buf = new byte[4];
	float[] ret = new float[data.Length / 4];		float?[] ret = new float?[data.Length / 4];
	for(int i = 0; i < data.Length; i += 4)		for(int i = 0; i < data.Length; i += 4)
	{		{
Line 33:		Line 33:
	buf[2] = data[i+2];		buf[2] = data[i+2];
	buf[3] = data[i+3];		buf[3] = data[i+3];
	// if (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0) continue; // ~~uncomment this if you do not want to include~~ null values		if (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0) continue; // ignore null values
	ret[i / 4] = BitConverter.ToSingle(buf); // note: if you are on a BigEndian machine you need to flip buf		ret[i / 4] = BitConverter.ToSingle(buf); // note: if you are on a BigEndian machine you need to flip buf
	}		}

2022-11-03T16:55:44Z

← Older revision		Revision as of 16:55, 3 November 2022
Line 16:		Line 16:
	ret=[]		ret=[]
	for i in range(int(len(data)/4)):		for i in range(int(len(data)/4)):
	if (data[i4]==0 and data[i4+1]==0 and data[i4+2]==0 and data[i4+3]==0): continue		#if (data[i4]==0 and data[i4+1]==0 and data[i4+2]==0 and data[i4+3]==0): continue # uncomment this if you do not want to include null values
	ret.append(struct.unpack('<f',data[i4:i4+4])[0])		ret.append(struct.unpack('<f',data[i4:i4+4])[0])
	return ret		return ret
Line 33:		Line 33:
	buf[2] = data[i+2];		buf[2] = data[i+2];
	buf[3] = data[i+3];		buf[3] = data[i+3];
	if (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0) continue; // ~~skip~~ null values		// if (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0) continue; // uncomment this if you do not want to include null values
	ret[i / 4] = BitConverter.ToSingle(buf); // note: if you are on a BigEndian machine you need to flip buf		ret[i / 4] = BitConverter.ToSingle(buf); // note: if you are on a BigEndian machine you need to flip buf
	}		}

2022-11-03T09:49:54Z

← Older revision		Revision as of 09:49, 3 November 2022
Line 25:		Line 25:
	public static float[] GetRawValues(byte[] data)		public static float[] GetRawValues(byte[] data)
	{		{
	~~var elements = data.Length / 4;~~
	byte[] buf = new byte[4];		byte[] buf = new byte[4];
	float[] ret = new float[~~elements~~];		float[] ret = new float[data.Length / 4];
	for(int i = 0; i < data.Length; i += 4)		for(int i = 0; i < data.Length; i += 4)
	{		{

2022-11-03T09:48:40Z

← Older revision		Revision as of 09:48, 3 November 2022
Line 16:		Line 16:
	ret=[]		ret=[]
	for i in range(int(len(data)/4)):		for i in range(int(len(data)/4)):
	if (data[i4]==0 and data[i4+1]==0 and data[i4+2]==0 and data[i4+3]==0):		if (data[i4]==0 and data[i4+1]==0 and data[i4+2]==0 and data[i4+3]==0): continue
	continue
	ret.append(struct.unpack('<f',data[i4:i4+4])[0])		ret.append(struct.unpack('<f',data[i4:i4+4])[0])
	return ret		return ret

2022-11-03T09:46:54Z

2022-11-03T09:39:48Z

← Older revision		Revision as of 09:39, 3 November 2022
Line 24:		Line 24:
	<includeonly>=== C# example ===</includeonly>		<includeonly>=== C# example ===</includeonly>

	public static float?[] GetRawValues(byte[] data)		public static float[] GetRawValues(byte[] data)
	{		{
	var elements = data.Length / 4;		var elements = data.Length / 4;
Line 35:		Line 35:
	buf[2] = data[i+2];		buf[2] = data[i+2];
	buf[3] = data[i+3];		buf[3] = data[i+3];
	if (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0) continue; // ~~ret[i/4] will be~~ null		if (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0) continue; // skip null values
	ret[i / 4] = BitConverter.ToSingle(buf); // note: if you are on a BigEndian machine you need to flip buf		ret[i / 4] = BitConverter.ToSingle(buf); // note: if you are on a BigEndian machine you need to flip buf
	}		}

2022-11-03T09:38:57Z

@@ Line 1: / Line 1: @@
-== Theory ==
 The raw data field is a stream of 60 little endian IEEE 754 floats, so it has exactly 240 bytes. The first 4 bytes represent the first minute of the hour and so on. Note that 0x000000 is defined to be NULL (no value).
-== C# example ==
+  def GetRawValues(data):
          public static float?[] GetRawValues(byte[] data)
@@ Line 22: / Line 40: @@
              return ret;
+         }