"""
Convert's Text files to SCID files for SierraChart
It will create a new scid file for every text file it finds
  format is assumed to be D T O H L C V T with comma separated variables
It will continue to update the scid file every REFRESH seconds
  generally refresh would be set to 1/2 the time period for the text file updates
"""

import os
import struct
from time import sleep
from datetime import datetime as dt, timedelta as tdelta
import subprocess

# these are linux format, windows might need to use double backslashes \\
FILES_IN = "/home/john/Python/Trading/textfiles/"
FILES_OUT = "/home/john/zRamdisk/SierraData/"
TIMEZONE = 0  # scid saved as UTC, use this if the text file is another time zone
KEEP_UPDATING = True  # set to False to convert and stop; True for live updating
REFRESH = 0.25  # refresh every 1/4 second for 1/2 second updating text files
TEXT_FILE_SUFFIX = ".txt"  # usually .txt or .csv

sizeHeader = 0x38
sizeRecord = 0x28
file_header = (b'SCID8\x00\x00\x00\x28\x00\x00\x00\x01\x00\x00\x00\x00\x00' +
               b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +
               b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +
               b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')


def excel_date(datetime, offset=TIMEZONE):
    """convert date&time with offset in hours to excel format for scid"""
    delta = datetime - dt(1899, 12, 30) + tdelta(hours=offset)
    return delta.days + delta.seconds / 86400


def python_date(exceldt, offset=TIMEZONE):
    """convert date&time with offset in hours to python format"""
    excel_as_delta = tdelta(days=exceldt//1) + tdelta(seconds=exceldt%1*86400)
    return dt(1899, 12, 30) + excel_as_delta + tdelta(hours=offset)

def write_struct(record, f):
    if len(record) == 8 and record[0][0].isnumeric():
        out = [excel_date(dt.strptime(f'{record[0]} {record[1]}', '%Y-%m-%d %H:%M:%S.%f'))]
        out.extend([float(s) for s in record[2:]])
        # note that SC packs dt, O, H, L, C, T, V not V, T so swap them
        wt = struct.pack('d4f4I', out[0], out[1], out[2], out[3], out[4], int(out[6]), int(out[5]), 0, 0)
        f.write(wt)


def create_base(f):
    record = None
    with open(FILES_OUT + f[: -3] +'scid', "wb") as scidfile:
        scidfile.write(file_header)
        with open(FILES_IN + f, "r") as txtfile:
            for line in txtfile:
                record = line.split(',')
                write_struct(record, scidfile)
    return f'{record[0]} {record[1]}'  # will update when date+time changes


def update_if_changed(files):
    for f in files:
        with open(FILES_IN + f, "r") as txtfile:
            record = txtfile.readlines()[-1].split(',')
            last_dt = f'{record[0]} {record[1]}'
            if files[f] == last_dt:
                continue
            files[f] = last_dt
            with open(FILES_OUT + f[: -3] +'scid', "ab") as scidfile:
                write_struct(record, scidfile)


if __name__ == '__main__':
    files = sorted(k for k in os.listdir(path=FILES_IN) if k[-4:] == TEXT_FILE_SUFFIX)
    files = {k: '' for k in files}
    for f in files:
        files[f] = create_base(f)
    print('finished base')
    if KEEP_UPDATING:
        # now test each input file each REFRESH interval & update if datetime changed
        while True:
            sleep(REFRESH)
            update_if_changed(files)