""" Downloader used to accumulate historical ticks data from Bitstamp on a daily basis. Can also be used to get real-time time if executed frequently enough. But in this case make sure you limit your calls to 600 requests per 10 minutes, or risk getting your IP banned. To just accumulate historical ticks, this script should be executed at least 2 times per day (eg with cron) in order not to miss any data, because Bitstamp gives only 24 hours of ticks. Usage: python create_scid_files.py [volume_multiplier] volume_multiplier defaults to 10000 if omitted (as SierraChart does for bitcoin) (see https://www.sierrachart.com/index.php?page=doc/CryptocurrencyDataServices.php for fractional volumes) The script matches last 5 ticks to find a place where to truncate new data appending to old. It is done because with 1 second precision like we have at Bitstamp we may get situation of the same time and price, though different trade (and we are cannot write trade_id to scid to solve this in common way) Details on particular fields of each tick: According to doc https://www.sierrachart.com/index.php?page=doc/IntradayDataFileFormat.html If a data record represents one single tick/trade, then the Intraday data record Open, High, Low, and Close can be set to the same value and you will write one record for every tick/trade. if you want to record the Bid and Ask price for the trade, then set the Open to 0, the High to the Ask price, the Low to the Bid price, and the Close to the trade price. By examining different scid files, I noticed if bid and ask for tick is unknown, open, high, low, close are set to the same value. Else close is 0. Bitstamp also provides information whether trade is buy (`type` field is 0) or sell (`type` field is 1). This gets reflected in bid/ask volume. (c) Pasha Dudko evilskunk [at] gmail.com """ import os import sys import warnings from datetime import datetime import numpy as np import pandas as pd import requests HEADER_SIZE = 56 scid_dtype = np.dtype([ ('dt', np.int64), # Should be ('dt', np.dtype('M8[us]')), but int64 is less error prone ('open', np.single), ('high', np.single), ('low', np.single), ('close', np.single), ('num_trades', np.int32), ('total_volume', np.int32), ('bid_volume', np.int32), ('ask_volume', np.int32)]) bitstamp_tick_dtype = np.dtype([('dt', np.int64), ('tid', np.int64), ('close', np.float32), ('amount', np.float64), ('type', np.int64)]) MICROSECONDS_PER_SECOND = 1000000 EXCEL_DT = np.datetime64('1899-12-30', 'us').view('i8') def get_bitstamp_day_of_ticks(ticker): """ Returns a NumPy structured array containing the latest day's worth of ticks from the Bitstamp API for the given ticker symbol. """ # API endpoint for fetching trades url = f'https://www.bitstamp.net/api/v2/transactions/{ticker}/' # Fetch trades response = requests.get(url, params={'time': 'day'}) trades = response.json() # Convert the trades to a NumPy structured array ticks = np.array([(trade['date'], trade['tid'], trade['price'], trade['amount'], trade['type']) for trade in trades], dtype=bitstamp_tick_dtype) # Reverse the order of ticks ticks = ticks[::-1] return ticks def write_to_scid(ticks, output_file, volume_multiplier): # Check if the output file exists file_exists = os.path.isfile(output_file) # Open the existing file or create a new one if file_exists: scid_arr = np.memmap(output_file, dtype=scid_dtype, mode='r+', offset=HEADER_SIZE) else: with open(output_file, "wb") as file: file.write(b'SCID8\x00\x00\x00(' b'\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') scid_arr = np.memmap(output_file, dtype=scid_dtype, mode='r+', offset=HEADER_SIZE, shape=(len(ticks),)) if file_exists: # Get the last 5 rows from the existing data (only 'dt' and 'close' columns) last_rows = scid_arr[-5:][['dt', 'close']] temp_last_rows = last_rows.copy() # Convert `us` back to `s` and epoch start time temp_last_rows['dt'] = (temp_last_rows['dt'] + EXCEL_DT) // MICROSECONDS_PER_SECOND # Check if the last 5 rows exist in the new ticks last_rows_found = False for i in range(len(ticks) - len(last_rows) + 1): if np.array_equal(ticks[['dt', 'close']][i:i+len(last_rows)], temp_last_rows): last_rows_found = True start_idx = i + len(last_rows) new_ticks = ticks[start_idx:] break if not last_rows_found: warnings.warn(f'Last rows sequence not found in {output_file}. Appending all new data.') new_ticks = ticks #print(f'Adding {len(new_ticks)} new ticks') # Extend the memory-mapped array with the new ticks scid_arr = np.memmap(output_file, dtype=scid_dtype, mode='r+', offset=HEADER_SIZE, shape=(len(scid_arr) + len(new_ticks),)) else: new_ticks = ticks volume = (new_ticks['amount'] * volume_multiplier).astype(np.int32) # Populate the memory-mapped array with data from the new ticks start_idx = len(scid_arr) - len(new_ticks) scid_arr['dt'][start_idx:] = new_ticks['dt'] * MICROSECONDS_PER_SECOND - EXCEL_DT scid_arr['open'][start_idx:] = new_ticks['close'] scid_arr['high'][start_idx:] = new_ticks['close'] scid_arr['low'][start_idx:] = new_ticks['close'] scid_arr['close'][start_idx:] = new_ticks['close'] scid_arr['num_trades'][start_idx:] = 1 scid_arr['total_volume'][start_idx:] = volume scid_arr['bid_volume'][start_idx:] = np.where(new_ticks['type'], volume, 0) scid_arr['ask_volume'][start_idx:] = np.where(new_ticks['type'], 0, volume) # Flush the data to disk and close the memory-mapped array del scid_arr def main(): if len(sys.argv) < 3 or len(sys.argv) > 4: print("Usage: python create_scid_files.py " " [volume_multiplier]") sys.exit(1) sierra_chart_dir = sys.argv[1] pairs_file = sys.argv[2] volume_multiplier = int(sys.argv[3]) if len(sys.argv) == 4 else 10000 with open(pairs_file, 'r') as f: for line in f: pair = line.strip().lower() if pair: # Skip empty lines output_file = os.path.join(sierra_chart_dir, f"{pair}.scid") try: ticks = get_bitstamp_day_of_ticks(pair) if len(ticks): write_to_scid(ticks, output_file, volume_multiplier) except Exception as e: print(f'Something went wring when downloading (or processing) {pair}, ' f'skipping. Error was:\n {str(e)}') if __name__ == "__main__": main()