import numpy as np
import pandas as pd
import struct
import sys
from time import sleep, time

o = O = 'O'; h = H = 'H'; l = L = 'L'; c = C = 'C'
V = 'V'; x = 'x'; y = 'y'; z = 'z'
d2s = 86400

time_list = []
overrun_list = []
overruns = 0


class SierraFile(object):
    """        """
    def __init__(self, filename):
        self.filename = str(filename)
        # self.tzAdjust = t imedelta(hours=+10).seconds/d2s
        self.tzAdjust = np.timedelta64(10, 'h')/np.timedelta64(1, 'D')
        self.excelDate = np.datetime64('1899-12-30')
        self.sizeHeader = 0x38
        self.sizeRecord = 0x28
        self.pos = 0
        self.last = 0

    def get_existing_records(self):
        with open(self.filename, 'rb') as fscid:
            fscid.read(self.sizeHeader)  # discard header
            rows = []
            ts = []
            for i in range(1000000):
                data = fscid.read(self.sizeRecord)
                if data not in ('', b''):
                        d = struct.unpack('d4f4I', data)
                        dt = d[0] + self.tzAdjust
                        ts.append(self.excelDate + np.timedelta64(int(dt))
                         +(np.timedelta64(int(round((dt-int(dt))*86400)),'s')))
                        datarow = [d[1], d[2], d[3], d[4], 0, 0, 0, 0]
                        rows.append(datarow)
                else:
                    break
            self.pos = self.last = fscid.tell()
        return (ts, rows)

    def get_record(self):
        global overruns, overrun_list
        with open(self.filename, 'rb') as fscid:
            fscid.seek(0, 2)  # Go to the end of the file
            self.last = fscid.tell()
            if self.last == self.pos:  # no new data >> nothing to do
                return (-999, 0, 0)
            else:  # data to collect
                if self.pos < self.last - self.sizeRecord:  # > 1 record
                    print('Overrun', self.last-self.pos,
                          (self.last-self.pos)/self.sizeRecord)
                    overruns += 1
                    overrun_list.append(np.datetime64('now'))
                    late_flag = True
                else:
                    late_flag = False
                fscid.seek(self.pos, 0)
                self.pos += self.sizeRecord
                data = fscid.read(self.sizeRecord)
                d = struct.unpack('d4f4I', data)
                dt = d[0] + self.tzAdjust
                new_time = (self.excelDate + np.timedelta64(int(dt))
                  + (np.timedelta64(int(round((dt-int(dt)) * 86400)), 's')))
                datarow = [d[1], d[2], d[3], d[4], 0, 0, 0, 0]
                return (new_time, datarow, late_flag)

    def save_existing_records(self, dataframe):
        with open(self.filename, 'wb') as fscid:
            header = b'SCID8\x00\x00\x00(\x00\x00\x00\x01\x00'
            fscid.write(header)
            for i in range(21):
                fscid.write(b'\x00\x00')
            for i in range(dataframe.end):
                da = ((dataframe.df.index.values[i] - self.excelDate)
                      / np.timedelta64(1, 'D') - self.tzAdjust)
                db, dc, dd, de, df, dg, dh, di = dataframe.df.iloc[i]
                di = 0x11100111
                wt = struct.pack('d4f4I', da, db, dc, dd, de, df, dg, dh, di)
                fscid.write(wt)

    def save_record(self, dataframe):
        with open(self.filename, 'ab') as fscid:
            i = dataframe.end - 1
            da = ((dataframe.df.index.values[i] - self.excelDate)
                  / np.timedelta64(1, 'D') - self.tzAdjust)
            db, dc, dd, de, df, dg, dh, di = dataframe.df.iloc[i]
            di = 0x88300388
            record = struct.pack('d4f4I', da, db, dc, dd, de, df, dg, dh, di)
            fscid.write(record)


class SierraFrame(object):
    """        """
    def __init__(self, time_series, data):
        self.df = pd.DataFrame(data, index=time_series,
                               columns=[O, H, L, C, V, x, y, z])
        self.end = len(self.df)
        self.pos = 0
        self.extend_frame()

    def extend_frame(self):
        '''
        Create a 4999 row array from last time in self.df
         and append it to self.df
        Remove lunch break from array
        '''
        s5 = np.timedelta64(5, 's')
        h1 = np.timedelta64(1, 'h')
        sl = np.datetime64('today') + np.timedelta64(14, 'h')
        el = np.datetime64('today') + np.timedelta64(15, 'h')
        start_time = self.df.index.values[self.end-1]
        dtgen = ((start_time + i * s5) for i in range(1, 5000))
        dtstrip = ((i + h1 if sl <= i < el else i) for i in dtgen)
        dg = pd.DataFrame(index=dtstrip, columns=[O, H, L, C, V, x, y, z])
        self.df = self.df.append(dg)

    def add(self, new_time, datarow):
        np_time = np.datetime64(new_time)
        if np_time < self.df.index.values[self.end]:
            return  # new data is earlier than current
        while np_time > self.df.index.values[self.end]:
            self.df.iloc[self.end] = self.df.iloc[self.end-1]
            self.end += 1  # fill with prior row if new is later
        self.df.iloc[self.end] = datarow  # fill when times match
        self.end += 1
        # if self.end > len(self.df) - 10:
        #     self.extend_frame()  # not needed if first fill > day lenght


def profile_sleep(sleep_time):
    # print(counter, end = '')
    # sys.stdout.flush()
    # if counter % 3 == 0: print()
    sleep(sleep_time)



def SierraRun():
    global time_list
    time0 = time()
    filename = '/home/john/zRamdisk/SierraChart/Data/HSI-201306-HKFE-TD.scid'
    hsi = SierraFile(filename)
    time_series, data = hsi.get_existing_records()
    da = SierraFrame(time_series, data)
    wtst = SierraFile('/home/john/zRamdisk/SierraChart/Data/HSI-INPUT.scid')
    wtst.save_existing_records(da)
    print('df ready', da.end-1, time() - time0, (time() - time0) / (da.end-1))
    print(da.df[da.end-1:da.end+1])
    print()
    df = da.df
    print('\n', np.datetime64('now'), da.end)
    print(df[da.end-5:da.end+5])
    # print('next', hsi.pos, hsi.last)
    # jtst = SierraFile('/home/john/zRamdisk/SierraChart/Data/HSI-INPUT.scid')
    # time_series, data = jtst.get_existing_records()
    # ja = SierraFrame(time_series, data)
    # jf = ja.df
    # print('\n', ja.end)
    # print(df[ja.end-5:ja.end+5])
    # print('next', jtst.pos, jtst.last)
    # return  # ###################
    counter = 0
    # sys.stdout = os.fdopen(sys.stdout.fileno(), "w", newline=None)
    counter_flag = False
    timer_no_data = time()
    timer_no_data_flag = False
    overruns = 0
    overrun_list = []
    while True:
        time0 = time()
        new_time, data, late_flag = hsi.get_record()
        if new_time != -999:
            da.add(new_time, data)
            wtst.save_record(da)
            time_list.append(time() - time0)
            timer_no_data = time()
            # print(df[da.end-1:da.end])
            print('0', end = ' ')
            sys.stdout.flush()
            if timer_no_data_flag:
                print('Data Restored')
                timer_no_data = time()
                timer_no_data_flag = False
            counter += 1
            counter_flag = True
        if time() - timer_no_data >= 120 and not timer_no_data_flag:
            timer_no_data_flag = True
            print('Data lost for two minutes')
        if not late_flag:
            sleep_time = 0.1 - (time() - time0)
            if sleep_time > 0:
                profile_sleep(sleep_time)
        if counter % 12 == 0 and counter_flag:
            counter_flag = False
            print('\nOverruns:', overruns, overrun_list, end = '')
            print('TimeStats', max(time_list), sum(time_list)/len(time_list), '\n')
            print(df[da.end-1:da.end])
            sys.stdout.flush()
            # break


def main():
    SierraRun()

if __name__ == '__main__':
    """
    Takes a SierraChart scid file (input argument 1) and converts
      it to a Pandas DataFrame
    Timezone conversion can follow the users local timezone, or a
      specified integer (input l or an integer but if the default
      filename is being used, '' must be specified for the filename)
    """
    print('start')
    sys.stdout.flush()
    main()
    print('fin')
    print('Overruns:', overruns, overrun_list)
    if time_list != []:
        print('TimeStats', max(time_list), sum(time_list)/len(time_list))

'''
def deserialize(self, date_and_time):

        def helper(factor, units):
            factor /= 86399.99
            result = list()
            for unit in units:
                value, factor = divmod(factor * unit, 1)
                result.append(int(value))
            result[3] = int(0)
            return result

        date_tok = int(date_and_time)
        time_tok = round(86400*(date_and_time - date_tok))
        d = date(1899, 12, 30) + timedelta(date_tok)
        t = time(*helper(time_tok, (24, 60, 60, 1000000)))
        return datetime.combine(d, t)

    dt = d[0] + self.tzAdjust.seconds/86400
    bartime = self.deserialize(dt)

    new_time = self.deserialize(d[0] + self.tzAdjust.seconds/86400.0)
            # datarow = [self.serialize(bar_time)]
            # datarow.extend([dataframe.df.iloc[i,j] for j in range(8)])
            # datarow.extend(dataframe.df.iloc[i])
            # da,db,dc,dd,de,df,dg,dh,di = datarow
            # da = self.serialize(bar_time)
            # print('Writing:', dataframe.df.iloc[i,0], fscid.tell())

                dx = (datetime.utcfromtimestamp(
                          dataframe.df.index.values[i].astype(int) * 1e-9)
                          - self.excelDat3)
                da = (float(dx.days) + (float(dx.seconds)/d2s) - self.tzAdjust)


    def gettimepython(self):
        s5 = timedelta(seconds = 5)
        h1 = timedelta(seconds = 3600)
        sl = datetime.combine(date.today(), time(14))
        el = datetime.combine(date.today(), time(15))
        start_time = self.df.index.values[self.end-1]
        start_time = datetime.utcfromtimestamp(start_time.astype(int) * 1e-9)
        dtgen = ((start_time + i * s5) for i in range(1, 5000))
        dtstrip = ((i+h1 if sl<=i<el else i) for i in dtgen)
        return dtstrip


'''