Support Board

def get_scid_np(scidFile, limitsize=sys.maxsize):
  # Check Path Exists
  f = Path(scidFile)
  assert f.exists(), "SCID file not found"
  # set the offset from limitsize
  stat = f.stat()
  offset = 56 if stat.st_size < limitsize else stat.st_size - (
    (limitsize // 40) * 40)

  ### Setup the SCID dtype
  sciddtype = np.dtype([
    ('Time', '<u8'),
    ('Open', '<f4'),
    ('High', '<f4'),
    ('Low', '<f4'),
    ('Close', '<f4'),
    ('Trades','<i4'),
    ('Volume', '<i4'),
    ('BidVolume', '<i4'),
    ('AskVolume', '<i4')
  ])
  scid = np.fromfile(scidFile, dtype=sciddtype, offset=offset)
  return scid

def write_scid(scidarray,outfile,mode = 'w'):
  ## Write header file in write mode
  if mode == 'w':
    with open(outfile, 'wb') as fscid:
      header = b'SCID8\x00\x00\x00(\x00\x00\x00\x01\x00'
      fscid.write(header)
      for i in range(21):
        fscid.write(b'\x00\x00')
      
      fscid.write(scidarray.tobytes())
      fscid.close()

  # Append the scid array else
  else:
    with open(outfile, 'ab') as fscid:
        fscid.write(scidarray.tobytes())
        fscid.close()
    
  return

BCOLS = ['datetime', 'open', 'high', 'low', 'close', 'volume', 'trades', 'bidVolume', 'askVolume']

rectype = np.dtype([

(BCOLS[0], '<u8'), (BCOLS[1], '<f4'), (BCOLS[2], '<f4'),

(BCOLS[3], '<f4'), (BCOLS[4], '<f4'), (BCOLS[6], '<i4'),

(BCOLS[5], '<i4'), (BCOLS[7], '<i4'), (BCOLS[8], '<i4')

])

sciddtype = np.dtype([
('Time', '<u8'),
('Open', '<f4'),
('High', '<f4'),
('Low', '<f4'),
('Close', '<f4'),
('Trades','<i4'),
('Volume', '<i4'),
('BidVolume', '<i4'),
('AskVolume', '<i4')
])

import sys
from pathlib import Path

import numpy as np
import polars as pl

def get_scid_np(scidFile, limitsize=sys.maxsize):
# Check Path Exists

f = Path(scidFile)

assert f.exists(), "SCID file not found"

# set the offset from limitsize

stat = f.stat()

offset = 56 if stat.st_size < limitsize else stat.st_size - ((limitsize // 40) * 40)

### Setup the SCID dtype

sciddtype = np.dtype(
[
("SCDateTime", "<u8"),
("Open", "<f4"),
("High", "<f4"),
("Low", "<f4"),
("Close", "<f4"),
("NumTrades", "<u4"),
("TotalVolume", "<u4"),
("BidVolume", "<u4"),
("AskVolume", "<u4"),
]
)

scid_as_np_array = np.fromfile(scidFile, dtype=sciddtype, offset=offset)

return scid_as_np_array

intermediate_np_array = get_scid_np("ESH24_FUT_CME.scid")

df_raw = pl.DataFrame(intermediate_np_array)

df_SCDateTime_as_string = df_raw.with_columns(
pl.col("SCDateTime").cast(pl.Utf8).alias("SCDateTime_string"),
)

df_extract = df_SCDateTime_as_string.with_columns(
pl.col("SCDateTime_string")
.str.extract(r"^.{13}", 0)
.cast(pl.Int64)
.alias("SCDateTime_ms"),
pl.col("SCDateTime_string")
.str.extract(r".{3}$", 0)
.cast(pl.Int32)
.alias("SCDateTime_trade_counter"),
)

# takes a sierra Time and returns it as a dt.datetime
def unSierraTime(timeint):
  exceldate = -2209161600.0
  timeint = timeint/1000000
  delta = timeint + exceldate
  delta = dt.datetime.fromtimestamp(delta).replace(tzinfo=dt.timezone.utc)
  return delta

# Takes a Pandas TimeStamp or a datetime index and turns it into a sierra time integer
def sierraTime(dtg):
  excelDate = (pd.to_datetime('1899-12-30')).tz_localize(tz="utc")
  delta = dtg - excelDate

  if type(dtg) == pd._libs.tslibs.timestamps.Timestamp:
    delta = int(delta.total_seconds()*1000000)

  else:
    delta = (delta.total_seconds()*1000000).astype('int64')
  
  return delta

import sys
from pathlib import Path
import numpy as np
import pandas as pd

def get_scid_df(filename, limitsize=sys.maxsize):
f = Path(filename)
assert f.exists(), f"{f} file not found"
stat = f.stat()
offset = 56 if stat.st_size < limitsize else stat.st_size - ((limitsize // 40) * 40)
sciddtype = np.dtype(
[
("Time", "<u8"),
("Open", "<f4"),
("High", "<f4"),
("Low", "<f4"),
("Close", "<f4"),
("Trades", "<i4"),
("Volume", "<i4"),
("BidVolume", "<i4"),
("AskVolume", "<i4"),
]
)
df = pd.DataFrame(
data=np.memmap(f, dtype=sciddtype, offset=offset, mode="r"), copy=False
)
df.dropna(inplace=True)
df["Time"] = df["Time"] - 2209161600000000
df.drop(df[(df.Time < 1)].index, inplace=True)
df.set_index("Time", inplace=True)
df.index = pd.to_datetime(df.index, unit="us")
df.index = df.index.tz_localize(tz="utc")
return df


def resample_scdf(scdf_or_filename, period="1Min", tz="UTC", limitsize=sys.maxsize):
df = (
get_scid_df(scdf_or_filename, limitsize)
if isinstance(scdf_or_filename, str)
else scdf_or_filename
)
assert isinstance(
df, pd.DataFrame
), f"{scdf_or_filename} has to be SC df or valid scid file"
df = df.resample(period).agg(
{
"Open": "first",
"High": "max",
"Low": "min",
"Close": "last",
"Trades": "sum",
"Volume": "sum",
"BidVolume": "sum",
"AskVolume": "sum",
}
)
if tz != "UTC":
tz = "America/New_York" if tz == "EST" else tz
tz = "America/Los_Angeles" if tz == "PST" else tz
df.index = df.index.tz_convert(tz)
return df

  Open  High  Low  Close  Trades  Volume  BidVolume  AskVolume
count  5.584600e+04  55846.000000  55846.000000  55846.000000  93144.000000  93144.000000  93144.000000  93144.000000
mean  -inf  18176.611328  18170.449219  18173.523438  235.347591  256.042236  128.308200  127.734035
std  inf  360.248291  361.233368  360.750702  571.431255  625.932788  316.699976  313.838125
min  -1.999001e+35  17130.500000  17113.750000  17121.250000  0.000000  0.000000  0.000000  0.000000
25%  0.000000e+00  17912.500000  17908.750000  17910.750000  0.000000  0.000000  0.000000  0.000000
50%  0.000000e+00  18278.250000  18271.750000  18275.000000  21.000000  22.000000  10.000000  10.000000
75%  0.000000e+00  18463.500000  18458.750000  18461.250000  141.000000  154.000000  77.000000  77.000000
max  0.000000e+00  18708.500000  18702.750000  18706.000000  9990.000000  11664.000000  6341.000000  5972.000000

import sys

from pathlib import Path

import numpy as np

import pandas as pd



def get_scid_df(filename, limitsize=sys.maxsize):

f = Path('/Users/Me/SierraPython/NQM4.CME.scid' )

assert f.exists(), f"{f} file not found"

stat = f.stat()

offset = 56 if stat.st_size < limitsize else stat.st_size - ((limitsize // 40) * 40)

sciddtype = np.dtype(

[

("Time", "<u8"),

("Open", "<f4"),

("High", "<f4"),

("Low", "<f4"),

("Close", "<f4"),

("Trades", "<i4"),

("Volume", "<i4"),

("BidVolume", "<i4"),

("AskVolume", "<i4"),

]

)

df = pd.DataFrame(

data=np.memmap(f, dtype=sciddtype, offset=offset, mode="r"), copy=False

)

df.dropna(inplace=True)

df["Time"] = df["Time"] - 2209161600000000

df.drop(df[(df.Time < 1)].index, inplace=True)

df.set_index("Time", inplace=True)

df.index = pd.to_datetime(df.index, unit="us")

df.index = df.index.tz_localize(tz="utc")

return df





def resample_scdf(scdf_or_filename, period="1Min", tz="UTC", limitsize=sys.maxsize):

df = (

get_scid_df(scdf_or_filename, limitsize)

if isinstance(scdf_or_filename, str)

else scdf_or_filename

)

assert isinstance(

df, pd.DataFrame

), f"{scdf_or_filename} has to be SC df or valid scid file"

df = df.resample(period).agg(

{

"Open": "first",

"High": "max",

"Low": "min",

"Close": "last",

"Trades": "sum",

"Volume": "sum",

"BidVolume": "sum",

"AskVolume": "sum",

}

)

if tz != "UTC":

tz = "America/New_York" if tz == "EST" else tz

tz = "America/Los_Angeles" if tz == "PST" else tz

df.index = df.index.tz_convert(tz)

return df

print(resample_scdf("NQM4.CME.scid", tz="EST"))

import datetime as dt
import numpy as np

# takes a scid time and returns it as a dt.datetime
def un_sierra_time(scidtime):
  sierra_epoch = dt.datetime(1899, 12, 30, tzinfo=dt.timezone.utc)
  dtg = sierra_epoch + dt.timedelta(microseconds=int(scidtime))
  return dtg

# takes a dt.datetime object and returns it as a np.uint64 for working with scid files
def re_sierra_time(dtg):
  sierra_epoch = dt.datetime(1899, 12, 30, tzinfo=dt.timezone.utc)
  delta = dtg - sierra_epoch
  scidtime = np.uint64(delta.total_seconds()*1000000)
  return scidtime

# Scid File Operations
## Imports
# Files / System
import sys
from pathlib import Path
## Data
import numpy as np
import pandas as pd
## Time / Date
import datetime as dt
from zoneinfo import ZoneInfo # needed for timezone conversion if required. Available Zones: print(zoneinfo.available_timezones()) / https://en.wikipedia.org/wiki/List_of_tz_database_time_zones

## function returns scidfile as a dataframe
def get_scid_df(scidfile, limitsize=sys.maxsize):
  f = Path(scidfile)
  assert f.exists(), "SCID file not found"
  # Check // set the size
  stat = f.stat()

  offset = 56 if stat.st_size < limitsize else stat.st_size - (
    (limitsize // 40) * 40)

  # rectype for DatFrame
  sciddtype = np.dtype([
    ('Time', '<u8'),
    ('Open', '<f4'),
    ('High', '<f4'),
    ('Low', '<f4'),
    ('Close', '<f4'),
    ('Trades','<i4'),
    ('Volume', '<i4'),
    ('BidVolume', '<i4'),
    ('AskVolume', '<i4'),
    ])

  df = pd.DataFrame(data=np.memmap(scidfile, dtype=sciddtype, offset=offset,mode="r"),copy=False)
  
  # Set the index columns and covert it to UTC time from Sierras Native Time
  df.set_index('Time', inplace=True)
  df.index = pd.to_datetime(df.index - 2209161600000000, unit='us').tz_localize(tz='utc')
  df.sort_index(inplace=True)
  return df

## function returns scidfile as a numpy array - quicker but less useful for some circumstances
def get_scid_np(scidfile, limitsize=sys.maxsize):
  # Check Path Exists
  f = Path(scidfile)
  assert f.exists(), f"SCID file not found: {scidfile}"
  # Check // set the size
  stat = f.stat()
  offset = 56 if stat.st_size < limitsize else stat.st_size - (
    (limitsize // 40) * 40)

  ### Setup the SCID dtype
  sciddtype = np.dtype([
    ('Time', '<u8'),
    ('Open', '<f4'),
    ('High', '<f4'),
    ('Low', '<f4'),
    ('Close', '<f4'),
    ('Trades','<i4'),
    ('Volume', '<i4'),
    ('BidVolume', '<i4'),
    ('AskVolume', '<i4'),
    ])

  scid = np.fromfile(scidfile, dtype=sciddtype, offset=offset)
  return scid


# takes a scid time and returns it as a dt.datetime
def time_from(sierratime):
  sierra_epoch = dt.datetime(1899, 12, 30, tzinfo=dt.timezone.utc)
  dtg = sierra_epoch + dt.timedelta(microseconds=int(sierratime))
  return dtg

# takes a dt.datetime object and returns it as a np.uint64 for working with scid files
def time_to(dtg):
  sierra_epoch = dt.datetime(1899, 12, 30, tzinfo=dt.timezone.utc)
  delta = dtg - sierra_epoch
  sierratime = np.uint64(delta.total_seconds()*1000000)
  return sierratime


## Example Usage
scidfile = 'c:/sierra/charts/data/ESH26-CME.scid'

## scicnp Opperations
scidnp = get_scid_np(scidfile)

# Get the last time from the SCID FIle
last_sierra_time = scidnp['Time'][-1]
# Convert that into a datetime object
last_dt_time = time_from(last_sierra_time)
# get the date of that object
last_date = last_dt_time.date()

# all three operations in one line:
last_date = time_from(scidnp['Time'][-1]).date()

# Now that we have a date to work on, we need to create datetime objects representing the start/end of the period we are looking
# I will use a full globex session as an example

# dt.datetime.combine takes a dt date object and a dt time object and converts it to a dattime object
# See Documentation here: https://docs.python.org/3/library/datetime.html
# as globex open is the prior day, we need to subtract 1 day from the last date, we do this using dt.imedelta(days=1) to get the last date
# as globex open is also in American Time we use a timezone native object set in US Time by specifying the timezone using tzinfo and ZoneInfo
# it is a good idea to use the correct timezone for the product you are looking at to make sure things are consistent
# e.g. if you are looking at say Bund I would use either a Berlin or London Time tzinfo=ZoneInfo('Europe/Berlin') / tzinfo=ZoneInfo('Europe/London') - London is still in Europe according to timezones.

globex_open = dt.datetime.combine(last_date-dt.timedelta(days=1),dt.time(18,0),tzinfo=ZoneInfo("America/New_York"))
globex_close = dt.datetime.combine(last_date,dt.time(17,0),tzinfo=ZoneInfo("America/New_York"))

# Now we get every entry in the numpy array where the Time is greater than Globex open and less than Globex Close
# calling the time_to function to convert the dattime objects to sierra times
window = scidnp[(scidnp['Time'] > time_to(globex_open)) & (scidnp['Time'] < time_to(globex_close))]



## Data Frame Operations
sciddf = get_scid_df(scidfile)

# As our function to get the dataframe converted the Sierra Time Object to a DateTime Index we can index quickly and efficently on the index
# The dataframe oindex in UTC time, however you can index in any timezone provided the start and end are in the same timezone
# to get the same window as on the np file we can slice the index as follows:
window = sciddf[globex_open:globex_close]


## Example of how to resample a dataframe inot a higher timeframe :
# the agg function is on the Close column instead of the Open/High/Low/Close columns due to the storage format used in scidfiles:
# Intraday Data File Format: Data Structures Used

df = sciddf.resample('1min', origin='start_day').agg({'Close': ['first','max', 'min', 'last'], 'Trades' : 'sum', 'Volume' : 'sum', 'BidVolume' : 'sum', 'AskVolume' : 'sum'}).dropna()
df.columns = ['Open', 'High', 'Low', 'Close', 'Trades','Volume','BidVolume','AskVolume']

# this is useful if you wnated to index of a specifc time as well - but that time may not have had trade at that exact milisecond.
# you could either iteratively set the window you want for every possible date... or you could just get every instance of that time in the dateframe:

# as an example:
cash_open_index = sciddf.loc[sciddf.index.tz_convert(ZoneInfo("America/New_York")).time == dt.time(9,30)].index # returns 9 instances and takes a while
cash_open_index = df.loc[df.index.tz_convert(ZoneInfo("America/New_York")).time == dt.time(9,30)].index # returns 23 instances very quickly

# we can then either call the dataframe with that index:
df.loc[cash_open_index]

# or we can iterate over that index as do more complex analysis:
for index in cash_open_index:
  time_end = dt.datetime.combine(index.date(),dt.time(9,35),tzinfo=ZoneInfo("America/New_York"))
  window = sciddf.loc[index:time_end]
  # do stuff here