// ============================================================================ // COT v.Prod -- Commitments of Traders, direct from CFTC // ============================================================================ // // Plots CFTC Commitments of Traders positioning in a sub-panel. The contract // is detected from the chart symbol, the data is downloaded from CFTC's own // public data portal and cached locally. No account, no API key, no setup. // // REPORT TYPES // Legacy 3 groups Commercial / Non-Commercial / Non-Reportable // Available for every instrument. // Disaggregated 4 groups Producer-Merchant / Swap Dealer / Managed Money // / Other Reportable. Physical commodities only. // TFF 4 groups Dealer / Asset Manager / Leveraged Funds / // Other Reportable. Financial instruments only. // Auto (default) selects Disaggregated for physicals and TFF for // financials; Legacy is the universal fallback. // // COVERAGE // Futures Only (default) or Futures + Options Combined. These are different // figures for the same contract, so the active one is named in the panel. // // DISPLAY MODES (mutually exclusive; their scales are incompatible) // Net Positions long minus short, in contracts. Above the zero line the // group is net long. // COT Index 100 * (net - min(net,N)) / (max(net,N) - min(net,N)), // 0-100 over an N-report window, reference lines at // 80 / 50 / 20. Produced only once a full window exists. // This is the plain net-position form of the index; Larry // Williams' variant additionally divides by open interest // and is not implemented here. // // TIMING // CFTC surveys positions on Tuesday and publishes the following Friday at // 15:30 America/New_York; US federal holidays inside the week push the // release to the Monday after. Values appear on the chart at their // publication moment, never at the survey date, so no bar can show data // that was not public at that bar. The series is weekly by nature. // // DATA SOURCE // publicreporting.cftc.gov (Socrata). Six dataset ids are used, one per // report type and coverage. Downloaded data is cached as CSV under the // Sierra Chart Data folder; CFTC is contacted at most once per refresh // window, and never during a chart replay. // // REQUIREMENTS // Sierra Chart with internet access. Works on any installation: all paths // are resolved at runtime and nothing outside this file is required. // ============================================================================ #include "sierrachart.h" #include #include #include #include #include #include // strict numeric parsing #include #include SCDLLName("COT v.Prod") // ============================================================================ // Instrument -> CFTC contract code. // // A code is only valid if it appears in CFTC's most recent weekly report; // matching by contract name alone is not sufficient, because delisted and // renamed contracts keep their old names in the historical data. Every code // below was confirmed present in the current report. The same code is used // by all three report types. // // To add an instrument: find its cftc_contract_market_code in the current // week of the relevant dataset and add a row here. Category decides which // report type "Auto" selects. // ============================================================================ enum { COT_CAT_FINANCIAL = 0, COT_CAT_PHYSICAL = 1, COT_CAT_UNKNOWN = 2 }; struct s_CotEntry { const char* Ticker; const char* Code; int Category; }; static const s_CotEntry g_CotTable[] = { // Indices (financial -> TFF) {"ES", "13874A", COT_CAT_FINANCIAL}, // E-MINI S&P 500 {"NQ", "209742", COT_CAT_FINANCIAL}, // NASDAQ MINI {"RTY", "239742", COT_CAT_FINANCIAL}, // RUSSELL E-MINI // Energy (physical -> Disaggregated) {"CL", "067651", COT_CAT_PHYSICAL}, // WTI-PHYSICAL, the NYMEX contract {"NG", "023651", COT_CAT_PHYSICAL}, // NAT GAS NYME {"RB", "111659", COT_CAT_PHYSICAL}, // GASOLINE RBOB {"HO", "022651", COT_CAT_PHYSICAL}, // NY HARBOR ULSD // Metals (physical -> Disaggregated) {"GC", "088691", COT_CAT_PHYSICAL}, // GOLD, full size {"SI", "084691", COT_CAT_PHYSICAL}, // SILVER {"HG", "085692", COT_CAT_PHYSICAL}, // COPPER- #1 {"PL", "076651", COT_CAT_PHYSICAL}, // PLATINUM {"PA", "075651", COT_CAT_PHYSICAL}, // PALLADIUM // Currencies (financial -> TFF) {"6E", "099741", COT_CAT_FINANCIAL}, // EURO FX {"6J", "097741", COT_CAT_FINANCIAL}, // JAPANESE YEN {"6B", "096742", COT_CAT_FINANCIAL}, // BRITISH POUND {"6S", "092741", COT_CAT_FINANCIAL}, // SWISS FRANC {"6C", "090741", COT_CAT_FINANCIAL}, // CANADIAN DOLLAR {"6A", "232741", COT_CAT_FINANCIAL}, // AUSTRALIAN DOLLAR {"6N", "112741", COT_CAT_FINANCIAL}, // NZ DOLLAR {"6M", "095741", COT_CAT_FINANCIAL}, // MEXICAN PESO // Rates (financial -> TFF) {"ZT", "042601", COT_CAT_FINANCIAL}, // UST 2Y NOTE {"ZF", "044601", COT_CAT_FINANCIAL}, // UST 5Y NOTE {"ZN", "043602", COT_CAT_FINANCIAL}, // UST 10Y NOTE {"TN", "043607", COT_CAT_FINANCIAL}, // ULTRA UST 10Y {"ZB", "020601", COT_CAT_FINANCIAL}, // UST BOND, classic 30-year {"UB", "020604", COT_CAT_FINANCIAL}, // ULTRA UST BOND // Grains (physical -> Disaggregated) {"ZC", "002602", COT_CAT_PHYSICAL}, // CORN {"ZS", "005602", COT_CAT_PHYSICAL}, // SOYBEANS, full size {"ZL", "007601", COT_CAT_PHYSICAL}, // SOYBEAN OIL {"ZM", "026603", COT_CAT_PHYSICAL}, // SOYBEAN MEAL {"ZW", "001602", COT_CAT_PHYSICAL}, // WHEAT-SRW, CBOT {"KE", "001612", COT_CAT_PHYSICAL}, // WHEAT-HRW, Kansas City {"MWE", "001626", COT_CAT_PHYSICAL}, // WHEAT-HRSpring, MIAX/MGEX {"ZO", "004603", COT_CAT_PHYSICAL}, // OATS. Reports intermittently: falls below // CFTC's reporting threshold in some weeks {"ZR", "039601", COT_CAT_PHYSICAL}, // ROUGH RICE // Softs / livestock (physical -> Disaggregated) {"KC", "083731", COT_CAT_PHYSICAL}, // COFFEE C {"CC", "073732", COT_CAT_PHYSICAL}, // COCOA {"SB", "080732", COT_CAT_PHYSICAL}, // SUGAR NO. 11 {"CT", "033661", COT_CAT_PHYSICAL}, // COTTON NO. 2 {"OJ", "040701", COT_CAT_PHYSICAL}, // FRZN CONCENTRATED ORANGE JUICE {"LE", "057642", COT_CAT_PHYSICAL}, // LIVE CATTLE {"HE", "054642", COT_CAT_PHYSICAL}, // LEAN HOGS {"GF", "061641", COT_CAT_PHYSICAL}, // FEEDER CATTLE {"LBR", "058644", COT_CAT_PHYSICAL}, // LUMBER }; static const int g_CotTableSize = sizeof(g_CotTable) / sizeof(g_CotTable[0]); // [1] Instrument dropdown layout: // 0 = "Auto (from chart symbol)" <- default // 1 .. g_CotTableSize = g_CotTable[index - 1] // g_CotTableSize + 1 = "Manual CFTC Code" static const int g_AutoInputIndex = 0; static const int g_ManualCodeInputIndex = g_CotTableSize + 1; // Report type ([3] dropdown index order must match this enum) enum { RPT_AUTO = 0, RPT_LEGACY = 1, RPT_DISAGG = 2, RPT_TFF = 3 }; // Display mode ([4] dropdown index order must match this enum) enum { DISP_NET = 0, DISP_INDEX = 1 }; static const int COT_MAX_CATS = 4; // --------------------------------------------------------------------------- // Category colours live here, in ONE place, because they are used twice: for // the plotted lines and for that category's row in the info panel. If the two // ever drift apart the panel becomes actively misleading -- you would be // reading a number next to the wrong colour. // --------------------------------------------------------------------------- static const COLORREF COT_CAT_COLOR[COT_MAX_CATS] = { RGB( 0, 160, 255), // 1 Commercial / Producer / Dealer - blue RGB( 0, 200, 120), // 2 Non-Comm / Swap Dealer / Asset Mgr - green RGB(255, 140, 0), // 3 Non-Rept / Managed Money / Leveraged- orange RGB(190, 190, 190), // 4 Other Reportable - grey }; static const COLORREF COT_COLOR_TEXT = RGB(225, 225, 225); static const COLORREF COT_COLOR_DIM = RGB(150, 150, 150); static const COLORREF COT_COLOR_WARN = RGB(255, 195, 80); static const COLORREF COT_COLOR_ERROR = RGB(255, 110, 110); // The panel is drawn as one text drawing PER LINE: a single DRAWING_TEXT has // exactly one Color for its whole contents, so a multi-line drawing cannot // tint each category row to match its line. Each row therefore gets its own // drawing, stacked downward from the top of the region. static const int COT_HUD_MAX_LINES = 12; static const float COT_HUD_TOP_PCT = 96.0f; // % of region height static const float COT_HUD_STEP_PCT = 5.5f; // spacing between rows // --------------------------------------------------------------------------- // SUBGRAPH MAP -- for reading this study from a trading system with // sc.GetStudyArrayUsingID() / sc.GetStudyArrayFromChartUsingID(). // // 0..3 what is DISPLAYED. Contents follow input [4] Display, so these // are net positions OR index values. Do NOT read these from a // system: their meaning changes with a UI setting. // 4 zero line (Net mode only, cosmetic) // 5,6,7 80 / 50 / 20 lines (Index mode only, cosmetic) // // 8..11 NET POSITION per category, in contracts, ALWAYS populated // 12..15 COT INDEX per category, 0-100, ALWAYS populated, // COT_INDEX_NA (-1) where the lookback window is not yet full // 16 DATA VALID: 1 if this bar carries a released report, else 0. // CHECK THIS FIRST. A net position of exactly 0 is a legitimate // reading (a group can be flat), so 0 in slots 8-11 cannot itself // mean "no data" -- only this flag distinguishes the two. // 17 REPORT DATE of the report in force on this bar, as a Sierra date // serial (days since 1899-12-30; feed it to SCDateTime::SetDate). // 0 when slot 16 is 0. Lets a system tell when the reading last // changed and detect a stale/held-over value. Deliberately NOT // YYYYMMDD: a subgraph is float and cannot hold 8-digit integers // exactly, so 20260707 would come back as 20260708. // // Category order within each block matches the active report type: // Legacy 1 Commercial 2 Non-Commercial 3 Non-Reportable 4 unused // Disaggregated 1 Producer/Merch 2 Swap Dealer 3 Managed Money 4 Other // TFF 1 Dealer 2 Asset Manager 3 Leveraged Funds 4 Other // // A bar with no data yet (before the first published report, or a report not // yet released at that bar) holds 0 in the net slots and COT_INDEX_NA in the // index slots -- so a system must check for those rather than trading a zero. // --------------------------------------------------------------------------- static const int COT_SG_NET_BASE = 8; static const int COT_SG_IDX_BASE = 12; static const int COT_SG_VALID = 16; static const int COT_SG_REPORTDATE = 17; // --------------------------------------------------------------------------- // Everything that differs between the three report types, in one place. // --------------------------------------------------------------------------- struct s_CotSchema { const char* DatasetId; const char* Label; int NumCats; const char* CatName[COT_MAX_CATS]; const char* FieldLong[COT_MAX_CATS]; const char* FieldShort[COT_MAX_CATS]; }; // CFTC publishes each report in two flavours, and they are NOT the same // numbers: "Futures Only" counts futures positions, "Futures and Options // Combined" folds in options on a delta-equivalent basis. For gold on // 2026-07-28 the Legacy commercial long is 75,460 futures-only but 103,867 // combined -- a 38% difference, so which one is in use has to be explicit. // All six dataset ids below were verified with live requests. static s_CotSchema snp_cot_schema(int reportType, bool combined) { s_CotSchema s; memset(&s, 0, sizeof(s)); if (reportType == RPT_DISAGG) { s.DatasetId = combined ? "kh3c-gbw2" : "72hh-3qpy"; s.Label = combined ? "Disaggregated F+O" : "Disaggregated"; s.NumCats = 4; s.CatName[0] = "Producer/Merchant"; s.FieldLong[0] = "prod_merc_positions_long"; s.FieldShort[0] = "prod_merc_positions_short"; s.CatName[1] = "Swap Dealer"; s.FieldLong[1] = "swap_positions_long_all"; s.FieldShort[1] = "swap__positions_short_all"; // double underscore is CFTC's s.CatName[2] = "Managed Money"; s.FieldLong[2] = "m_money_positions_long_all"; s.FieldShort[2] = "m_money_positions_short_all"; s.CatName[3] = "Other Reportable"; s.FieldLong[3] = "other_rept_positions_long"; s.FieldShort[3] = "other_rept_positions_short"; } else if (reportType == RPT_TFF) { s.DatasetId = combined ? "yw9f-hn96" : "gpe5-46if"; s.Label = combined ? "TFF F+O" : "TFF"; s.NumCats = 4; s.CatName[0] = "Dealer"; s.FieldLong[0] = "dealer_positions_long_all"; s.FieldShort[0] = "dealer_positions_short_all"; s.CatName[1] = "Asset Manager"; s.FieldLong[1] = "asset_mgr_positions_long"; s.FieldShort[1] = "asset_mgr_positions_short"; s.CatName[2] = "Leveraged Funds"; s.FieldLong[2] = "lev_money_positions_long"; s.FieldShort[2] = "lev_money_positions_short"; s.CatName[3] = "Other Reportable"; s.FieldLong[3] = "other_rept_positions_long"; s.FieldShort[3] = "other_rept_positions_short"; } else // RPT_LEGACY { s.DatasetId = combined ? "jun7-fc8e" : "6dca-aqww"; s.Label = combined ? "Legacy F+O" : "Legacy"; s.NumCats = 3; s.CatName[0] = "Commercial"; s.FieldLong[0] = "comm_positions_long_all"; s.FieldShort[0] = "comm_positions_short_all"; s.CatName[1] = "Non-Commercial"; s.FieldLong[1] = "noncomm_positions_long_all"; s.FieldShort[1] = "noncomm_positions_short_all"; s.CatName[2] = "Non-Reportable"; s.FieldLong[2] = "nonrept_positions_long_all"; s.FieldShort[2] = "nonrept_positions_short_all"; s.CatName[3] = ""; s.FieldLong[3] = ""; s.FieldShort[3] = ""; } return s; } // --------------------------------------------------------------------------- // Chart symbol -> root ticker, so the study configures itself. // "NGV26-NYMEX" -> "NG": drop the "-EXCHANGE" suffix, then a trailing // [month code][2-digit year]. Month codes: F G H J K M N Q U V X Z. // A root-only or continuous symbol ("NG") passes through unchanged. // --------------------------------------------------------------------------- static SCString snp_cot_root_from_symbol(const char* sym) { char buf[64]; int n = 0; for (const char* p = sym; p != NULL && *p != '\0' && n < 63; ++p) { if (*p == '-' || *p == ' ' || *p == '[' || *p == '.') break; buf[n++] = *p; } buf[n] = '\0'; while (n > 0 && (buf[n - 1] == '#' || buf[n - 1] == '!')) buf[--n] = '\0'; if (n >= 3 && isdigit((unsigned char)buf[n - 1]) && isdigit((unsigned char)buf[n - 2])) { const char mc = (char)toupper((unsigned char)buf[n - 3]); if (strchr("FGHJKMNQUVXZ", mc) != NULL) { n -= 3; buf[n] = '\0'; } } else if (n >= 2 && isdigit((unsigned char)buf[n - 1])) { const char mc = (char)toupper((unsigned char)buf[n - 2]); if (strchr("FGHJKMNQUVXZ", mc) != NULL) { n -= 2; buf[n] = '\0'; } } return SCString(buf); } // Sierra Chart returns the literal string "Unset", not an empty string, for a // text input the user has never edited. static bool snp_cot_input_is_set(const char* raw) { return raw != NULL && raw[0] != '\0' && strcmp(raw, "Unset") != 0; } // CFTC codes are alphanumeric; drop anything else rather than trying to escape // it, since this value is interpolated into a SoQL string literal. static SCString snp_cot_sanitize_code(const char* raw) { SCString out; for (const char* p = raw; p != NULL && *p != '\0'; ++p) if (isalnum((unsigned char)*p)) out += *p; return out; } // Strict numeric parse. atof() cannot distinguish a genuine "0" from an empty // cell, "N/A", or any other junk -- all become 0.0, and a fabricated zero net // position then poisons both the plotted value AND the min/max window of the // COT Index for the next three years. A field that is not a clean finite // number is rejected so the whole row can be dropped instead. static bool snp_cot_parse_number(const char* text, double& r_Value) { if (text == NULL || *text == '\0') return false; char* end = NULL; errno = 0; const double v = strtod(text, &end); if (end == text) return false; // nothing numeric at all while (*end == ' ' || *end == '\t' || *end == '\r') ++end; if (*end != '\0') return false; // trailing junk -> not a clean number if (errno == ERANGE) return false; if (!(v == v)) return false; // NaN if (v > 1e15 || v < -1e15) return false; // absurd magnitude / inf r_Value = v; return true; } // Socrata's .csv wraps EVERY field in quotes, numbers and dates included // ("1429304", "2026-07-28T00:00:00.000"). Strip one matched pair. static char* snp_cot_strip_quotes(char* s) { if (s == NULL) return s; const size_t len = strlen(s); if (len >= 2 && s[0] == '"' && s[len - 1] == '"') { s[len - 1] = '\0'; return s + 1; } return s; } // Split on ',' WITHOUT coalescing consecutive delimiters -- strtok would treat // an empty field between two commas as absent and shift every later column // left by one. Returns token count (capped at maxFields). static int snp_cot_split_csv_line(char* line, char* out[], int maxFields) { int count = 0; char* p = line; while (count < maxFields) { out[count++] = p; char* comma = strchr(p, ','); if (comma == NULL) break; *comma = '\0'; p = comma + 1; } return count; } // Subgraph values of exactly 0 are this file's "no data on this bar" sentinel // (DrawZeros=0 then skips them). A real net position or COT Index can also be // exactly 0, so nudge a genuine zero by an imperceptible amount to keep the // two cases apart. static double snp_cot_nudge_zero(double v) { return (v == 0.0) ? 0.0001 : v; } // Is Y/M/D a real calendar date? A plain range check accepts 2026-02-31 and // 2026-04-31; building the date and reading it back rejects them, because // SetDateYMD normalises an impossible day into a different one. static bool snp_cot_is_real_date(int y, int m, int d) { if (y < 1900 || y > 2200 || m < 1 || m > 12 || d < 1 || d > 31) return false; SCDateTime probe; probe.SetDateYMD(y, m, d); int ry = 0, rm = 0, rd = 0; probe.GetDateYMD(ry, rm, rd); return (ry == y && rm == m && rd == d); } // FNV-1a. Persistent storage holds integers only, so the contract code is // compared by hash to detect that the rendered series, or the request in // flight, is still the current selection. static int snp_cot_hash(const char* s) { unsigned int h = 2166136261u; for (const char* p = s; p != NULL && *p != '\0'; ++p) { h ^= (unsigned char)*p; h *= 16777619u; } return (int)(h & 0x7FFFFFFF); } // --------------------------------------------------------------------------- // Release schedule. // // Positions are surveyed on Tuesday and published the following Friday at // 15:30 America/New_York. A US federal holiday on the Wednesday, Thursday or // Friday of that week moves the release to the Monday after. // // The schedule is computed rather than tabulated, so it does not need annual // maintenance. Holidays that always fall on a Monday (MLK, Presidents, // Memorial, Labor, Columbus) cannot sit inside a Wed-Fri window and never // delay a release. // // Unscheduled CFTC delays (government shutdowns, incident-driven // rescheduling) are not modelled; those reports will be dated earlier than // they were actually published. // --------------------------------------------------------------------------- static const int COT_PUBLICATION_HOUR_ET = 15; static const int COT_PUBLICATION_MINUTE_ET = 30; // Is this Y/M/D a US federal holiday as OBSERVED (Saturday holidays are // observed the preceding Friday, Sunday holidays the following Monday)? // Only the holidays that can land Tue-Fri need to be modelled; see above. static bool snp_cot_is_federal_holiday(int y, int m, int d) { SCDateTime probe; probe.SetDateYMD(y, m, d); const int dow = probe.GetDayOfWeek(); // A holiday is never OBSERVED on a weekend, so a Sat/Sun date cannot be // the observed day for anything. if (dow == SATURDAY || dow == SUNDAY) return false; // Fixed-date holidays, including the shifted observance that lands on this // very day: a Friday can be the observance of a Saturday holiday, and a // Monday that of a Sunday holiday. static const int fixedMonth[] = { 1, 6, 7, 11, 12 }; static const int fixedDay[] = { 1, 19, 4, 11, 25 }; for (int i = 0; i < 5; ++i) { // exact date if (m == fixedMonth[i] && d == fixedDay[i]) return true; // Friday observing a Saturday holiday (holiday falls tomorrow) if (dow == FRIDAY) { SCDateTime nxt = probe; nxt += SCDateTime::DAYS(1); int ny = 0, nm = 0, nd = 0; nxt.GetDateYMD(ny, nm, nd); if (nm == fixedMonth[i] && nd == fixedDay[i]) return true; } // Monday observing a Sunday holiday (holiday fell yesterday) if (dow == MONDAY) { SCDateTime prv = probe; prv -= SCDateTime::DAYS(1); int py = 0, pm = 0, pd = 0; prv.GetDateYMD(py, pm, pd); if (pm == fixedMonth[i] && pd == fixedDay[i]) return true; } } // Thanksgiving: 4th Thursday of November. if (m == 11 && dow == THURSDAY && d >= 22 && d <= 28) return true; return false; } // ============================================================================ // One weekly report, normalized. Also the cache-file layout. // ============================================================================ struct s_CotRow { int Y, M, D; double Lng[COT_MAX_CATS]; double Shrt[COT_MAX_CATS]; double Net(int k) const { return Lng[k] - Shrt[k]; } }; // The Tuesday the positions were held (what CFTC labels the report). static SCDateTime snp_cot_row_datetime(const s_CotRow& r) { SCDateTime dt; dt.SetDateYMD(r.Y, r.M, r.D); return dt; } // The moment the report became public, in the CHART's time zone so it can be // compared directly against sc.BaseDateTimeIn[]. Built as 15:30 New York and // converted through the timezone database, so daylight saving is handled. static SCDateTime snp_cot_publication_datetime(SCStudyGraphRef sc, const s_CotRow& r) { // Anchor on the actual Friday of the report's week, found by walking the // calendar rather than by adding a fixed number of days. The report date // is usually a Tuesday, but when a holiday falls on that Tuesday CFTC // moves the report date itself back to the Monday, and fixed-offset // arithmetic would then land on a weekend. SCDateTime friday; friday.SetDateYMD(r.Y, r.M, r.D); for (int guard = 0; guard < 7 && friday.GetDayOfWeek() != FRIDAY; ++guard) friday += SCDateTime::DAYS(1); // Only a holiday on the Wednesday, Thursday or Friday of that week delays // the release. A Tuesday holiday has already been absorbed by CFTC moving // the report date back, so it must not delay anything. bool holidayDelays = false; for (int back = 2; back >= 0 && !holidayDelays; --back) { SCDateTime probe = friday; probe -= SCDateTime::DAYS(back); int py = 0, pm = 0, pd = 0; probe.GetDateYMD(py, pm, pd); if (snp_cot_is_federal_holiday(py, pm, pd)) holidayDelays = true; } SCDateTime dt = friday; if (holidayDelays) dt += SCDateTime::DAYS(3); // slips to the following Monday dt.SetTimeHMS(COT_PUBLICATION_HOUR_ET, COT_PUBLICATION_MINUTE_ET, 0); return sc.ConvertDateTimeToChartTimeZone(dt, TIMEZONE_NEW_YORK); } static long snp_cot_stamp(const s_CotRow& r) { return (long)r.Y * 10000L + (long)r.M * 100L + (long)r.D; } // --------------------------------------------------------------------------- // Cache: "\COT__.csv", oldest row first. // Always 4 categories on disk (unused ones are zero) so the format is fixed. // YYYY-MM-DD,L0,S0,L1,S1,L2,S2,L3,S3 // --------------------------------------------------------------------------- static SCString snp_cot_cache_path(const SCString& folder, const SCString& code, const char* label) { SCString path; path.Format("%s\\COT_%s_%s.csv", folder.GetChars(), code.GetChars(), label); return path; } static bool snp_cot_read_cache(const SCString& path, std::vector& r_Rows) { r_Rows.clear(); FILE* fh = fopen(path.GetChars(), "r"); if (fh == NULL) return false; // The cache is validated as strictly as a network response. It is our own // file, but it can still be truncated by a crash, hand-edited, or left // over from an older format -- and a bad row here corrupts the index // window just as effectively as a bad row off the wire. char line[512]; long prevStamp = 0; while (fgets(line, sizeof(line), fh) != NULL) { char* fields[1 + COT_MAX_CATS * 2]; const int nf = snp_cot_split_csv_line(line, fields, 1 + COT_MAX_CATS * 2); if (nf != 1 + COT_MAX_CATS * 2) continue; // Trim the trailing newline off the last field. for (char* t = fields[nf - 1]; *t != '\0'; ++t) if (*t == '\r' || *t == '\n') { *t = '\0'; break; } int y = 0, m = 0, d = 0; if (sscanf(fields[0], "%d-%d-%d", &y, &m, &d) != 3) continue; if (!snp_cot_is_real_date(y, m, d)) continue; s_CotRow row; row.Y = y; row.M = m; row.D = d; bool good = true; for (int k = 0; k < COT_MAX_CATS && good; ++k) { if (!snp_cot_parse_number(fields[1 + k * 2], row.Lng[k])) good = false; if (!snp_cot_parse_number(fields[2 + k * 2], row.Shrt[k])) good = false; } if (!good) continue; // Reject out-of-order or duplicated dates instead of letting them // break the ascending-order assumption everything downstream relies on. const long stamp = snp_cot_stamp(row); if (stamp <= prevStamp) continue; prevStamp = stamp; r_Rows.push_back(row); } fclose(fh); return true; } // Create a directory chain. _mkdir only creates the last component, so a // user-supplied path with a missing parent would otherwise fail silently. static void snp_cot_make_dirs(const SCString& path) { SCString partial; const char* p = path.GetChars(); for (int i = 0; p[i] != '\0'; ++i) { partial += p[i]; if ((p[i] == '\\' || p[i] == '/') && i > 2) _mkdir(partial.GetChars()); } _mkdir(path.GetChars()); } // Atomic cache write: build a temporary file, verify every write and the // close, then replace the live file in one operation. The existing cache is // never truncated, so an interrupted write cannot leave a partial file that // still parses but is missing its newest weeks. static bool snp_cot_write_cache(const SCString& path, const SCString& tmpPath, const std::vector& rows) { if (rows.empty()) return false; FILE* fh = fopen(tmpPath.GetChars(), "w"); if (fh == NULL) return false; bool ok = true; for (size_t i = 0; i < rows.size() && ok; ++i) { const s_CotRow& r = rows[i]; const int written = fprintf(fh, "%04d-%02d-%02d,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f\n", r.Y, r.M, r.D, r.Lng[0], r.Shrt[0], r.Lng[1], r.Shrt[1], r.Lng[2], r.Shrt[2], r.Lng[3], r.Shrt[3]); if (written <= 0) ok = false; // disk full / write error } if (fflush(fh) != 0) ok = false; if (fclose(fh) != 0) ok = false; // close can surface a deferred write error if (!ok) { remove(tmpPath.GetChars()); return false; } // MoveFileEx replaces in a single operation, so the target file is never // momentarily absent (remove-then-rename would lose both files if the // rename failed). if (!MoveFileExA(tmpPath.GetChars(), path.GetChars(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { remove(tmpPath.GetChars()); return false; } return true; } // --------------------------------------------------------------------------- // Socrata request URL. Spaces must be percent-encoded: an unencoded space // terminates the path in the HTTP request line, and everything after it -- // including $order and $limit -- never reaches the server. // --------------------------------------------------------------------------- static SCString snp_cot_build_url(const s_CotSchema& sch, const SCString& code, int weeksWanted, const SCString& appToken) { SCString fields; for (int k = 0; k < sch.NumCats; ++k) { fields += ","; fields += sch.FieldLong[k]; fields += ","; fields += sch.FieldShort[k]; } SCString url; url.Format( "https://publicreporting.cftc.gov/resource/%s.csv?" "$select=report_date_as_yyyy_mm_dd%s" "&$where=cftc_contract_market_code='%s'" "&$order=report_date_as_yyyy_mm_dd%%20DESC" "&$limit=%d", sch.DatasetId, fields.GetChars(), code.GetChars(), weeksWanted + 12); if (appToken.GetLength() > 0) url += SCString().Format("&$$app_token=%s", appToken.GetChars()); return url; } // --------------------------------------------------------------------------- // Parse the CSV body. Column count is 1 + NumCats*2 and the header row is // skipped. The result is sorted ascending by date explicitly rather than // relying on the server having honoured $order; everything downstream assumes // that ordering. // --------------------------------------------------------------------------- static bool snp_cot_parse_response(const SCString& body, const s_CotSchema& sch, std::vector& r_Rows) { r_Rows.clear(); const size_t len = (size_t)body.GetLength(); if (len == 0) return false; const int expectCols = 1 + sch.NumCats * 2; std::vector buf(len + 1); memcpy(&buf[0], body.GetChars(), len + 1); char* linesContext = NULL; char* line = strtok_s(&buf[0], "\r\n", &linesContext); bool firstLine = true; while (line != NULL) { if (firstLine) { firstLine = false; line = strtok_s(NULL, "\r\n", &linesContext); continue; } char* fields[1 + COT_MAX_CATS * 2]; const int nf = snp_cot_split_csv_line(line, fields, expectCols); if (nf == expectCols) { for (int k = 0; k < nf; ++k) fields[k] = snp_cot_strip_quotes(fields[k]); int y = 0, m = 0, d = 0; if (sscanf(fields[0], "%d-%d-%d", &y, &m, &d) == 3 && snp_cot_is_real_date(y, m, d)) { s_CotRow row; row.Y = y; row.M = m; row.D = d; for (int k = 0; k < COT_MAX_CATS; ++k) { row.Lng[k] = 0.0; row.Shrt[k] = 0.0; } // All-or-nothing: one unparseable field drops the whole week // rather than silently contributing a fake zero. bool good = true; for (int k = 0; k < sch.NumCats && good; ++k) { if (!snp_cot_parse_number(fields[1 + k * 2], row.Lng[k])) good = false; if (!snp_cot_parse_number(fields[2 + k * 2], row.Shrt[k])) good = false; } if (good) r_Rows.push_back(row); } } line = strtok_s(NULL, "\r\n", &linesContext); } // Insertion sort ascending by date (list is ~180 rows and usually already // ordered, so this is effectively one validating pass). Hand-rolled because // Sierra's headers #define max/min, so std:: algorithms are avoided here. for (size_t i = 1; i < r_Rows.size(); ++i) { const s_CotRow key = r_Rows[i]; const long keyStamp = snp_cot_stamp(key); size_t j = i; while (j > 0 && snp_cot_stamp(r_Rows[j - 1]) > keyStamp) { r_Rows[j] = r_Rows[j - 1]; --j; } r_Rows[j] = key; } // Drop duplicate report dates, keeping the last one seen. The cache reader // rejects duplicates on the way in, so the network path must too -- // otherwise a duplicated week would occupy two slots in the index window // and skew its min/max. if (r_Rows.size() > 1) { size_t w = 1; for (size_t rd = 1; rd < r_Rows.size(); ++rd) { if (snp_cot_stamp(r_Rows[rd]) == snp_cot_stamp(r_Rows[w - 1])) r_Rows[w - 1] = r_Rows[rd]; // same date -> keep the later row else r_Rows[w++] = r_Rows[rd]; } r_Rows.resize(w); } return !r_Rows.empty(); } // --------------------------------------------------------------------------- // COT Index -- the standard net-position range oscillator: // // index = 100 * (net_now - min(net, N)) / (max(net, N) - min(net, N)) // // 100 = the most net-long this group has been in the last N weeks, // 0 = the most net-short. Verified against published descriptions of the // index (Williams / Briese lineage); 156 weeks (3 years) is the common // default. Larry Williams' own variant divides the net position by open // interest BEFORE running this oscillator; that refinement is not applied // here, so this is the plain net-position form of the index -- which is what // most charting packages label "COT Index". // // A value is produced ONLY once a FULL lookback window exists. Computing it // over a partial window is meaningless: with two weeks of history the result // can only ever be exactly 0 or 100, so the early part of a series would // flap between the extremes and look like noise. Rows without a full window // return COT_INDEX_NA and are simply not drawn. // --------------------------------------------------------------------------- static const double COT_INDEX_NA = -1.0; static void snp_cot_compute_index(const std::vector& rows, int numCats, int lookbackWeeks, std::vector< std::vector >& r_Index) { const size_t n = rows.size(); const size_t need = (lookbackWeeks > 1) ? (size_t)lookbackWeeks : 1; r_Index.assign(numCats, std::vector(n, COT_INDEX_NA)); for (int c = 0; c < numCats; ++c) { for (size_t i = 0; i < n; ++i) { if (i + 1 < need) continue; // not enough history yet const size_t start = i + 1 - need; double lo = rows[start].Net(c), hi = lo; for (size_t k = start; k <= i; ++k) { const double v = rows[k].Net(c); if (v < lo) lo = v; if (v > hi) hi = v; } const double range = hi - lo; r_Index[c][i] = (range > 0.0) ? 100.0 * (rows[i].Net(c) - lo) / range : 50.0; } } } // ============================================================================ // Study function // ============================================================================ SCSFExport scsf_COT_Prod(SCStudyGraphRef sc) { // Subgraphs 0-3 are the four trader categories; what they mean depends on // the report type, so their names are refreshed at runtime below. SCSubgraphRef Sg_Cat0 = sc.Subgraph[0]; SCSubgraphRef Sg_Cat1 = sc.Subgraph[1]; SCSubgraphRef Sg_Cat2 = sc.Subgraph[2]; SCSubgraphRef Sg_Cat3 = sc.Subgraph[3]; SCSubgraphRef Sg_ZeroLine = sc.Subgraph[4]; // Net mode only SCSubgraphRef Sg_Ref80 = sc.Subgraph[5]; // Index mode only SCSubgraphRef Sg_Ref50 = sc.Subgraph[6]; // Index mode only SCSubgraphRef Sg_Ref20 = sc.Subgraph[7]; // Index mode only // Subgraphs 8-15 are the MACHINE-READABLE outputs. They are never drawn // and never depend on the display mode -- see COT_SG_* in the header. // Subgraphs 0-3 hold whatever the user chose to LOOK at, so a trading // system reading them would silently get net positions or index values // depending on a UI setting. These fixed slots exist so a system always // reads the same thing. SCInputRef InInstrument = sc.Input[0]; SCInputRef InManualCode = sc.Input[1]; SCInputRef InReportType = sc.Input[2]; SCInputRef InDisplayMode = sc.Input[3]; SCInputRef InLookback = sc.Input[4]; SCInputRef InShowStudy = sc.Input[5]; SCInputRef InShowHUD = sc.Input[6]; SCInputRef InForceRefresh = sc.Input[7]; SCInputRef InRefreshDays = sc.Input[8]; SCInputRef InCacheFolder = sc.Input[9]; SCInputRef InAppToken = sc.Input[10]; SCInputRef InCoverage = sc.Input[11]; SCInputRef InHistoryYears = sc.Input[12]; if (sc.SetDefaults) { sc.GraphName = "COT v.Prod (Commitments of Traders)"; sc.StudyDescription = "Commitments of Traders positioning, downloaded directly from CFTC's public data " "portal. No account or API key is required and the contract is detected from the " "chart symbol. Report type: Legacy (Commercial / Non-Commercial / Non-Reportable), " "Disaggregated (Producer / Swap Dealer / Managed Money / Other, physical " "commodities only), or TFF (Dealer / Asset Manager / Leveraged Funds / Other, " "financial instruments only). Coverage: Futures Only or Futures and Options " "Combined; these are different figures and the active one is named in the info " "panel. Display: Net Positions in contracts, or COT Index 0-100 normalised over " "the lookback window, which counts reports rather than calendar weeks. Values " "appear at their CFTC publication time, so no bar shows unpublished data. Every " "mode is also switchable from the chart's right-click menu."; sc.AutoLoop = 0; // manual loop: network/cache handled once per call sc.GraphRegion = 1; // own sub-panel sc.ValueFormat = 0; // Category subgraphs. Default names list all three schemas' meanings // so they stay informative even if a chartbook keeps the stored name. // Value label on the right-hand scale is ON by default for the four // data lines: with several similar-coloured lines in one panel, the // scale label is the fastest way to tell which is which and read the // current number without hovering. The reference lines below leave it // off on purpose -- they are constants (80/50/20/zero) and labelling // them would just clutter the scale. const int cotValueLabel = LL_DISPLAY_VALUE | LL_VALUE_ALIGN_VALUES_SCALE; Sg_Cat0.Name = "1 Commercial / Producer / Dealer"; Sg_Cat0.DrawStyle = DRAWSTYLE_LINE; Sg_Cat0.PrimaryColor = RGB(0, 160, 255); Sg_Cat0.LineWidth = 2; Sg_Cat0.DrawZeros = 0; Sg_Cat0.LineLabel = cotValueLabel; Sg_Cat1.Name = "2 Non-Commercial / Swap Dealer / Asset Manager"; Sg_Cat1.DrawStyle = DRAWSTYLE_LINE; Sg_Cat1.PrimaryColor = RGB(0, 200, 120); Sg_Cat1.LineWidth = 2; Sg_Cat1.DrawZeros = 0; Sg_Cat1.LineLabel = cotValueLabel; Sg_Cat2.Name = "3 Non-Reportable / Managed Money / Leveraged Funds"; Sg_Cat2.DrawStyle = DRAWSTYLE_LINE; Sg_Cat2.PrimaryColor = RGB(255, 140, 0); Sg_Cat2.LineWidth = 2; Sg_Cat2.DrawZeros = 0; Sg_Cat2.LineLabel = cotValueLabel; Sg_Cat3.Name = "4 Other Reportable"; Sg_Cat3.DrawStyle = DRAWSTYLE_LINE; Sg_Cat3.PrimaryColor = RGB(190, 190, 190); Sg_Cat3.LineWidth = 1; Sg_Cat3.DrawZeros = 0; Sg_Cat3.LineLabel = cotValueLabel; Sg_ZeroLine.Name = "Zero (net long above / net short below)"; Sg_ZeroLine.DrawStyle = DRAWSTYLE_LINE; Sg_ZeroLine.PrimaryColor = RGB(130, 130, 130); Sg_ZeroLine.LineWidth = 1; Sg_ZeroLine.DrawZeros = 0; Sg_Ref80.Name = "Index 80 (crowded long)"; Sg_Ref80.DrawStyle = DRAWSTYLE_DASH; Sg_Ref80.PrimaryColor = RGB(130, 130, 130); Sg_Ref80.LineWidth = 1; Sg_Ref80.DrawZeros = 0; Sg_Ref50.Name = "Index 50"; Sg_Ref50.DrawStyle = DRAWSTYLE_DASH; Sg_Ref50.PrimaryColor = RGB(90, 90, 90); Sg_Ref50.LineWidth = 1; Sg_Ref50.DrawZeros = 0; Sg_Ref20.Name = "Index 20 (crowded short)"; Sg_Ref20.DrawStyle = DRAWSTYLE_DASH; Sg_Ref20.PrimaryColor = RGB(130, 130, 130); Sg_Ref20.LineWidth = 1; Sg_Ref20.DrawZeros = 0; // ---- Machine-readable outputs (never drawn, never mode-dependent) ---- // A trading system reads these with sc.GetStudyArrayUsingID(); they are // always populated regardless of what the chart is displaying. for (int k = 0; k < COT_MAX_CATS; ++k) { sc.Subgraph[COT_SG_NET_BASE + k].Name = SCString().Format("[API] Net %d (contracts)", k + 1); sc.Subgraph[COT_SG_NET_BASE + k].DrawStyle = DRAWSTYLE_IGNORE; sc.Subgraph[COT_SG_NET_BASE + k].DrawZeros = 0; sc.Subgraph[COT_SG_IDX_BASE + k].Name = SCString().Format("[API] Index %d (0-100, -1 = n/a)", k + 1); sc.Subgraph[COT_SG_IDX_BASE + k].DrawStyle = DRAWSTYLE_IGNORE; sc.Subgraph[COT_SG_IDX_BASE + k].DrawZeros = 0; } sc.Subgraph[COT_SG_VALID].Name = "[API] Data Valid (1/0) - check before using Net"; sc.Subgraph[COT_SG_VALID].DrawStyle = DRAWSTYLE_IGNORE; sc.Subgraph[COT_SG_VALID].DrawZeros = 0; sc.Subgraph[COT_SG_REPORTDATE].Name = "[API] Report Date (Sierra date serial)"; sc.Subgraph[COT_SG_REPORTDATE].DrawStyle = DRAWSTYLE_IGNORE; sc.Subgraph[COT_SG_REPORTDATE].DrawZeros = 0; // ---- Inputs: every default is chosen so the study works untouched ---- { SCString list = "Auto (from chart symbol)"; for (int i = 0; i < g_CotTableSize; ++i) { list += ";"; list += g_CotTable[i].Ticker; } list += ";Manual CFTC Code (see input below)"; InInstrument.Name = "[1] Instrument"; InInstrument.SetCustomInputStrings(list.GetChars()); InInstrument.SetCustomInputIndex(g_AutoInputIndex); } InManualCode.Name = "[2] Manual CFTC Contract Code (only if Instrument = 'Manual CFTC Code')"; InManualCode.SetString(""); InReportType.Name = "[3] Report Type"; InReportType.SetCustomInputStrings( "Auto (best for this instrument);" "Legacy - 3 groups (Commercial / Non-Comm / Small);" "Disaggregated - 4 groups (Producer / Swap / Managed Money / Other);" "TFF - 4 groups (Dealer / Asset Mgr / Leveraged / Other)"); InReportType.SetCustomInputIndex(RPT_AUTO); InDisplayMode.Name = "[4] Display"; InDisplayMode.SetCustomInputStrings( "Net Positions (contracts, long minus short);" "COT Index (0-100, normalized over lookback)"); InDisplayMode.SetCustomInputIndex(DISP_NET); // "reports", not "weeks": the window counts weekly REPORTS, and a thin // contract that skips reporting weeks spans more calendar time than // the number suggests. InLookback.Name = "[5] COT Index Lookback (reports; 156 = about 3 years)"; InLookback.SetInt(156); InLookback.SetIntLimits(4, 260); InShowStudy.Name = "[6] Show COT Study"; InShowStudy.SetYesNo(1); InShowHUD.Name = "[7] Show Info Panel"; InShowHUD.SetYesNo(1); InForceRefresh.Name = "[8] Force Refresh Now"; InForceRefresh.SetYesNo(0); InRefreshDays.Name = "[9] Re-check CFTC Every (days)"; InRefreshDays.SetInt(3); InRefreshDays.SetIntLimits(1, 14); // Left EMPTY on purpose: the default is resolved at runtime from // sc.DataFilesFolder(), so the study works on any machine and any // Sierra Chart installation without editing a hardcoded path. InCacheFolder.Name = "[10] Cache Folder (blank = \\COT_Cache)"; InCacheFolder.SetString(""); InAppToken.Name = "[11] CFTC App Token (optional; free, only needed if throttled)"; InAppToken.SetString(""); InCoverage.Name = "[12] Contract Coverage"; InCoverage.SetCustomInputStrings( "Futures Only;" "Futures + Options Combined"); InCoverage.SetCustomInputIndex(0); InHistoryYears.Name = "[13] History to Download (years)"; InHistoryYears.SetInt(10); InHistoryYears.SetIntLimits(1, 40); return; } // ------------------------------------------------------------------ // Persistent state (per study instance) // int 1 httpState 2 menuShowID 3 satisfiedWeeks // 4 lastFetchError 5 lastAskDate 6 warnedUnresolved // 7 renderSig 8 inflightSig 9 consecFails // 10 forcePending 11 loadedCacheSig 12 menuPanelID // 13 menuDispNetID 14 menuDispIdxID 15 menuRefreshID // 16 menuRepAutoID 17 menuRepLegacyID // 18 menuRepDisaggID 19 menuRepTffID 20 usableRows // 21 reportsOnChart 22 menuCovFutID 23 menuCovCombID // 28 inflightWeeks // SCDateTime 1 requestSentAt 2 nextRetryAt // ptr 0 cached rows 1 cached index 2 cached bar mapping // ------------------------------------------------------------------ enum { HTTP_IDLE = 0, HTTP_SENT = 1 }; enum { MAX_AUTO_RETRIES = 3 }; enum { HTTP_TIMEOUT_SECONDS = 90 }; // real elapsed time, not a call count int& httpState = sc.GetPersistentInt(1); int& menuShowID = sc.GetPersistentInt(2); // How many weeks the last SUCCESSFUL download asked for. This is what // terminates the "config wants more history" refetch: comparing against // rows.size() instead would loop forever whenever CFTC simply has fewer // reports than were requested (a contract listed only a few years ago), // because the request would keep succeeding and keep looking unsatisfied. int& satisfiedWeeks = sc.GetPersistentInt(3); int& lastFetchError = sc.GetPersistentInt(4); int& lastAskDate = sc.GetPersistentInt(5); int& warnedUnknown = sc.GetPersistentInt(6); int& renderSig = sc.GetPersistentInt(7); int& inflightSig = sc.GetPersistentInt(8); int& consecFails = sc.GetPersistentInt(9); // for the retry backoff below int& forcePending = sc.GetPersistentInt(10); int& loadedCacheSig = sc.GetPersistentInt(11); int& menuPanelID = sc.GetPersistentInt(12); int& menuDispNetID = sc.GetPersistentInt(13); int& menuDispIdxID = sc.GetPersistentInt(14); int& menuRefreshID = sc.GetPersistentInt(15); int& menuRepAutoID = sc.GetPersistentInt(16); int& menuRepLegacyID = sc.GetPersistentInt(17); int& menuRepDisaggID = sc.GetPersistentInt(18); int& menuRepTffID = sc.GetPersistentInt(19); int& menuCovFutID = sc.GetPersistentInt(22); int& menuCovCombID = sc.GetPersistentInt(23); int& inflightWeeks = sc.GetPersistentInt(28); // weeks the in-flight request asked for // Kept as SCDateTime rather than seconds in an int: SCDateTime counts days // from 1899-12-30, so an equivalent seconds value is around 3.99e9 and // overflows a 32-bit int. SCDateTime& requestSentAt = sc.GetPersistentSCDateTime(1); SCDateTime& nextRetryAt = sc.GetPersistentSCDateTime(2); // The parsed rows live across calls: re-reading and re-parsing the cache // file on every tick would be pure waste on a live chart. void*& rowsHolder = sc.GetPersistentPointer(0); // The panel occupies COT_HUD_MAX_LINES consecutive drawing ids, so each // study instance needs its own block of them -- otherwise two COT studies // on one chart would overwrite each other's rows. const int hudLineNumber = 88800 + sc.StudyGraphInstanceID * COT_HUD_MAX_LINES; if (sc.LastCallToFunction) { if (menuShowID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuShowID); if (menuPanelID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuPanelID); if (menuDispNetID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuDispNetID); if (menuDispIdxID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuDispIdxID); if (menuRepAutoID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuRepAutoID); if (menuRepLegacyID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuRepLegacyID); if (menuRepDisaggID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuRepDisaggID); if (menuRepTffID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuRepTffID); if (menuCovFutID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuCovFutID); if (menuCovCombID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuCovCombID); if (menuRefreshID > 0) sc.RemoveACSChartShortcutMenuItem(sc.ChartNumber, menuRefreshID); for (int L = 0; L < COT_HUD_MAX_LINES; ++L) sc.DeleteACSChartDrawing(sc.ChartNumber, TOOL_DELETE_CHARTDRAWING, hudLineNumber + L); if (rowsHolder != NULL) { delete (std::vector*)rowsHolder; rowsHolder = NULL; } void*& idxHolderCleanup = sc.GetPersistentPointer(1); if (idxHolderCleanup != NULL) { delete (std::vector< std::vector >*)idxHolderCleanup; idxHolderCleanup = NULL; } void*& rowBarHolderCleanup = sc.GetPersistentPointer(2); if (rowBarHolderCleanup != NULL) { delete (std::vector*)rowBarHolderCleanup; rowBarHolderCleanup = NULL; } return; } if (rowsHolder == NULL) rowsHolder = new std::vector(); std::vector& rows = *(std::vector*)rowsHolder; // ------------------------------------------------------------------ // Chart shortcut menu -- every mode switch is reachable with one // right-click, no Format Study dialog needed. The item TEXT is rewritten // each call to state what the click will do next, so nothing has to be // memorised. Ids are persistent-per-instance, not C++ statics: a static // would be shared by every chart using this DLL, so a second chart would // silently never register its own items. // ------------------------------------------------------------------ // Every option of both dropdowns gets its own line, with a checkmark on the // active one, mirroring the Format Study lists. // // No separators are used: sc.AddACSChartShortcutMenuSeparator() has no // removal counterpart and is not idempotent, so a study that is added, // removed and recalculated would accumulate dividers with no way to clean // them up. The "COT:", "COT view:", "COT report:" and "COT data:" prefixes // group the items instead. // // Adding items IS idempotent -- adding one whose MenuText already exists // returns the existing id rather than duplicating it -- so registration // simply runs on every full recalculation. That also makes it self-healing: // there is no stored "already registered" flag that could leave the menu // permanently empty. if (sc.UpdateStartIndex == 0) { menuShowID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT: study visible"); menuPanelID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT: info panel"); menuDispNetID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT view: Net Positions (contracts)"); menuDispIdxID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT view: COT Index (0-100)"); menuRepAutoID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT report: Auto"); menuRepLegacyID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT report: Legacy - 3 groups"); menuRepDisaggID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT report: Disaggregated - 4 groups"); menuRepTffID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT report: TFF - 4 groups"); menuCovFutID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT data: Futures Only"); menuCovCombID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT data: Futures + Options Combined"); menuRefreshID = sc.AddACSChartShortcutMenuItem(sc.ChartNumber, "COT: refresh from CFTC now"); } // Report-type picks need the instrument's category (a report type that // cannot hold this instrument must not be selectable), so they are handled // after the instrument is resolved, further below. if (sc.MenuEventID > 0) { if (sc.MenuEventID == menuShowID) InShowStudy.SetYesNo((InShowStudy.GetYesNo() != 0) ? 0 : 1); else if (sc.MenuEventID == menuPanelID) InShowHUD.SetYesNo((InShowHUD.GetYesNo() != 0) ? 0 : 1); else if (sc.MenuEventID == menuDispNetID) InDisplayMode.SetCustomInputIndex(DISP_NET); else if (sc.MenuEventID == menuDispIdxID) InDisplayMode.SetCustomInputIndex(DISP_INDEX); else if (sc.MenuEventID == menuCovFutID) InCoverage.SetCustomInputIndex(0); else if (sc.MenuEventID == menuCovCombID) InCoverage.SetCustomInputIndex(1); else if (sc.MenuEventID == menuRefreshID) InForceRefresh.SetYesNo(1); } // ------------------------------------------------------------------ // Resolve instrument -> CFTC code // ------------------------------------------------------------------ SCString code, resolvedTicker; int category = COT_CAT_UNKNOWN; bool resolved = false; const SCString chartRoot = snp_cot_root_from_symbol(sc.Symbol.GetChars()); const int instrumentIdx = InInstrument.GetIndex(); if (instrumentIdx == g_AutoInputIndex) { for (int i = 0; i < g_CotTableSize; ++i) { if (_stricmp(chartRoot.GetChars(), g_CotTable[i].Ticker) == 0) { code = g_CotTable[i].Code; category = g_CotTable[i].Category; resolvedTicker = g_CotTable[i].Ticker; resolved = true; break; } } } else if (instrumentIdx == g_ManualCodeInputIndex) { if (snp_cot_input_is_set(InManualCode.GetString())) { const SCString cleaned = snp_cot_sanitize_code(InManualCode.GetString()); if (cleaned.GetLength() > 0) { code = cleaned; category = COT_CAT_UNKNOWN; resolvedTicker = "Manual"; resolved = true; } } } else if (instrumentIdx >= 1 && instrumentIdx <= g_CotTableSize) { const s_CotEntry& e = g_CotTable[instrumentIdx - 1]; code = e.Code; category = e.Category; resolvedTicker = e.Ticker; resolved = true; } if (!resolved) { if (warnedUnknown == 0) { warnedUnknown = 1; if (instrumentIdx == g_AutoInputIndex) sc.AddMessageToLog(SCString().Format( "COT: chart symbol '%s' (root '%s') is not in the built-in CFTC table. " "Choose the instrument in [1], or set [1] to 'Manual CFTC Code' and enter a code in [2].", sc.Symbol.GetChars(), chartRoot.GetChars()), 1); else sc.AddMessageToLog("COT: [1] is set to 'Manual CFTC Code' but [2] is empty.", 1); } return; } warnedUnknown = 0; // Which report types can actually contain THIS instrument. CFTC only // publishes Disaggregated for physical commodities and TFF for financials; // Legacy covers everything. Asking for the wrong one is not an error to // report to the user, it is a combination that must simply not be issued. const bool canDisagg = (category != COT_CAT_FINANCIAL); // physical or unknown const bool canTFF = (category != COT_CAT_PHYSICAL); // financial or unknown // Report-type picks from the shortcut menu. A type that cannot hold this // instrument is simply ignored (its menu label already says so), so a // stray click can never produce a guaranteed-empty request. if (sc.MenuEventID > 0) { if (sc.MenuEventID == menuRepAutoID) InReportType.SetCustomInputIndex(RPT_AUTO); else if (sc.MenuEventID == menuRepLegacyID) InReportType.SetCustomInputIndex(RPT_LEGACY); else if (sc.MenuEventID == menuRepDisaggID && canDisagg) InReportType.SetCustomInputIndex(RPT_DISAGG); else if (sc.MenuEventID == menuRepTffID && canTFF) InReportType.SetCustomInputIndex(RPT_TFF); } // A contract's CFTC code is the SAME in all three datasets (verified live // for every code in the table), so Auto is free to pick the richest // breakdown: Disaggregated for physicals, TFF for financials. int reportType = InReportType.GetIndex(); if (reportType == RPT_AUTO) { if (category == COT_CAT_FINANCIAL) reportType = RPT_TFF; else if (category == COT_CAT_PHYSICAL) reportType = RPT_DISAGG; else reportType = RPT_LEGACY; } // If the report type was picked in the Format Study dialog (which cannot // be filtered), silently fall back to one that can hold this instrument // rather than issuing a request that is guaranteed to come back empty and // then reporting it as an error. The substitution is stated in the panel. const int requestedReport = reportType; if (reportType == RPT_DISAGG && !canDisagg) reportType = RPT_TFF; else if (reportType == RPT_TFF && !canTFF) reportType = RPT_DISAGG; const bool reportSubstituted = (reportType != requestedReport); const bool combined = (InCoverage.GetIndex() == 1); const s_CotSchema sch = snp_cot_schema(reportType, combined); const int displayMode = InDisplayMode.GetIndex(); const bool showStudy = InShowStudy.GetYesNo() != 0; // Identifies "which series is this" -- used both to validate an in-flight // response and to decide when the cache must be re-read from disk. // Coverage is part of it: futures-only and combined are different numbers // for the same contract and must never share a cache file or a response. const int datasetSig = snp_cot_hash(code.GetChars()) ^ (reportType * 7919) ^ (combined ? 0x5A5A5 : 0); // Name the category subgraphs for the schema actually in use, so the chart // legend says "Managed Money" rather than a generic placeholder. // (Indexed via sc.Subgraph[k] directly: SCSubgraphRef is a reference type, // and C++ has no arrays of references.) for (int k = 0; k < COT_MAX_CATS; ++k) { if (k < sch.NumCats) sc.Subgraph[k].Name = SCString().Format("%d %s%s", k + 1, sch.CatName[k], (displayMode == DISP_INDEX) ? " Index" : " Net"); else sc.Subgraph[k].Name = "(unused)"; } // ------------------------------------------------------------------ // Menu state. Checkmarks mark what is active; a report type that cannot // hold this instrument is GREYED OUT rather than relabelled. // // NOTHING HERE MAY RENAME A MENU ITEM. Sierra matches menu items by their // text: adding an item whose text already exists returns the existing id // instead of creating a duplicate, which is what makes re-registration // safe. Renaming an item at runtime breaks that match -- the next // registration no longer recognises the renamed item, creates a fresh one // and hands back a new id, and the renamed item is orphaned with its id // forgotten. It then survives every cleanup and stays in the chart's // right-click menu forever. That is exactly how a stale // "COT report: TFF - n/a for this instrument" line was left behind after // the study had been deleted from the chart. // // sc.SetACSChartShortcutMenuItemEnabled() conveys the same thing without // touching the text, which is what it exists for. // ------------------------------------------------------------------ if (menuShowID > 0) { const int pickedReport = InReportType.GetIndex(); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuShowID, showStudy); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuPanelID, InShowHUD.GetYesNo() != 0); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuDispNetID, displayMode == DISP_NET); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuDispIdxID, displayMode == DISP_INDEX); // Greyed out instead of relabelled; which report Auto resolved to is // stated in the info panel, so the menu does not need to say it. sc.SetACSChartShortcutMenuItemEnabled(sc.ChartNumber, menuRepDisaggID, canDisagg); sc.SetACSChartShortcutMenuItemEnabled(sc.ChartNumber, menuRepTffID, canTFF); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuRepAutoID, pickedReport == RPT_AUTO); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuRepLegacyID, pickedReport == RPT_LEGACY); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuRepDisaggID, pickedReport == RPT_DISAGG); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuRepTffID, pickedReport == RPT_TFF); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuCovFutID, !combined); sc.SetACSChartShortcutMenuItemChecked(sc.ChartNumber, menuCovCombID, combined); } // ------------------------------------------------------------------ // Cache (re-read from disk only when the series changed, not per tick) // ------------------------------------------------------------------ // Portable by default: sc.DataFilesFolder() is whatever Data folder THIS // installation uses, so the cache lands correctly regardless of where // Sierra Chart was installed or on whose machine. SCString folder = InCacheFolder.GetString(); if (!snp_cot_input_is_set(folder.GetChars())) folder.Format("%s\\COT_Cache", sc.DataFilesFolder().GetChars()); const SCString cachePath = snp_cot_cache_path(folder, code, sch.Label); // Unique per study instance: two instances downloading the same series // would otherwise write the same .tmp and corrupt each other's file. // StudyGraphInstanceID is unique only WITHIN a chart, so two charts can // both hold instance 1 and would collide on the same temp file while // downloading the same series into a shared cache folder. SCString cacheTmpPath; cacheTmpPath.Format("%s.c%d.s%d.tmp", cachePath.GetChars(), sc.ChartNumber, sc.StudyGraphInstanceID); // The folder is part of the cache key so that changing it reloads rather // than leaving the old rows in memory. const int cacheKey = datasetSig ^ snp_cot_hash(folder.GetChars()); if (loadedCacheSig != cacheKey) { snp_cot_make_dirs(folder); snp_cot_read_cache(cachePath, rows); loadedCacheSig = cacheKey; // Fetch and error state belong to the previous series; carrying them // over would misreport the new one and suppress its first refresh. lastFetchError = 0; consecFails = 0; lastAskDate = 0; satisfiedWeeks = 0; nextRetryAt.Clear(); } // ------------------------------------------------------------------ // Refresh gate and asynchronous fetch. // // The gate is based on when the data was last REQUESTED, not on how old // the data is. Age-based gating never stops for a contract whose data // legitimately ends in the past, producing a continuous request loop. // // Wall-clock time is used throughout, not chart time: // sc.GetCurrentDateTime() returns replay time during a replay, which // would let a backtest drive live network requests. // ------------------------------------------------------------------ const int refreshDays = InRefreshDays.GetInt(); const int todayDate = sc.CurrentSystemDateTime.GetDate(); const bool replaying = sc.IsReplayRunning(); if (InForceRefresh.GetYesNo() != 0) { forcePending = 1; consecFails = 0; nextRetryAt.Clear(); // a manual retry waits for nothing InForceRefresh.SetYesNo(0); // one-shot; honoured below even if a } // request is already in flight // How many weekly reports the configuration wants downloaded. const int weeksRequired = (InHistoryYears.GetInt() * 52 > InLookback.GetInt() + 12) ? InHistoryYears.GetInt() * 52 : InLookback.GetInt() + 12; // A backtest must never depend on the network, and a replay must never // rewrite the cache underneath itself. Serve whatever is already on disk. if (replaying) forcePending = 0; // The decision to fetch is made further down, immediately before the // request is sent and after the response block, because the response block // modifies the rows and the failure counters that decision depends on. // The response check is independent of any fetch decision, so a completed // response is never dropped just because the refresh gate happens to be // closed at that moment. bool rowsChanged = false; if (httpState == HTTP_SENT) { if (sc.HTTPResponse.GetLength() > 0) { // A request started BEFORE the replay began can still land during // it. Blocking new requests is not enough on its own -- parsing // this one would rewrite the cache in the middle of a backtest. if (replaying) { sc.AddMessageToLog("COT: discarding an HTTP response that arrived during replay.", 0); httpState = HTTP_IDLE; requestSentAt.Clear(); } // Discard a response whose request was built for a different // contract/report -- otherwise switching instrument mid-flight // writes one contract's data into another contract's cache. else if (inflightSig != datasetSig) { sc.AddMessageToLog("COT: discarding a response for a previous instrument/report selection.", 0); } else if (sc.HTTPResponse.CompareNoCase("HTTP_REQUEST_ERROR") == 0 || sc.HTTPResponse.CompareNoCase("ERROR") == 0) { sc.AddMessageToLog("COT: CFTC request failed -- check network.", 1); lastFetchError = 1; ++consecFails; } else { std::vector fetched; if (snp_cot_parse_response(sc.HTTPResponse, sch, fetched)) { if (snp_cot_write_cache(cachePath, cacheTmpPath, fetched)) { lastFetchError = 0; consecFails = 0; // Record what THIS request asked for, not what the // config wants right now: raising History Years while // a download was in flight would otherwise mark the // larger figure satisfied by the smaller download, and // the extra history would never be fetched. // // Recorded as the request's own value, not the row // count: CFTC may simply hold fewer reports than were // asked for, and comparing against rows.size() would // re-request the same download forever. satisfiedWeeks = inflightWeeks; } else { // Counts as a failure: rows are reloaded from disk when // the series changes, so a cache that never persists // would otherwise re-download forever. sc.AddMessageToLog(SCString().Format( "COT: downloaded OK but could not write the cache file '%s'. " "Check that [10] Cache Folder exists and is writable.", cachePath.GetChars()), 1); lastFetchError = 1; ++consecFails; } rows = fetched; rowsChanged = true; // A live contract reports weekly. Months-old data means a // dead code, not a transient gap -- say so rather than // silently drawing a flat line from stale positioning. const double ageDays = sc.CurrentSystemDateTime.GetAsDouble() - snp_cot_row_datetime(rows.back()).GetAsDouble(); if (ageDays > 60.0) sc.AddMessageToLog(SCString().Format( "COT: WARNING -- newest report for code %s is %04d-%02d-%02d (%.0f days old). " "That contract code is probably no longer reporting.", code.GetChars(), rows.back().Y, rows.back().M, rows.back().D, ageDays), 1); } else { sc.AddMessageToLog(SCString().Format( "COT: no usable rows for code %s in the %s report. That code may not exist " "in this report type -- try [3] Report Type = Legacy.", code.GetChars(), sch.Label), 1); lastFetchError = 1; ++consecFails; } } httpState = HTTP_IDLE; requestSentAt.Clear(); } else { // Watchdog on ELAPSED WALL TIME. Kept as SCDateTime arithmetic -- // converting to "seconds since epoch" in an int overflows, since // SCDateTime counts from 1899 and that value passed int32 range // long ago. if (requestSentAt.GetAsDouble() > 0.0) { const double elapsedSec = (sc.CurrentSystemDateTime.GetAsDouble() - requestSentAt.GetAsDouble()) * 86400.0; if (elapsedSec > (double)HTTP_TIMEOUT_SECONDS) { sc.AddMessageToLog("COT: request timed out with no response; will retry.", 1); httpState = HTTP_IDLE; requestSentAt.Clear(); lastFetchError = 1; ++consecFails; } } } // Schedule the next attempt with a bounded, growing delay so a // persistent failure cannot turn into a request storm. if (httpState == HTTP_IDLE && consecFails > 0) { const int backoffSec = (consecFails == 1) ? 15 : (consecFails == 2) ? 60 : 300; nextRetryAt = sc.CurrentSystemDateTime; nextRetryAt += SCDateTime::SECONDS(backoffSec); } } // ------------------------------------------------------------------ // Fetch gate. Evaluated here, immediately before acting on it, because // the response block above modifies the rows and failure counters it // depends on. Sierra's shipped HTTP example orders it the same way. // ------------------------------------------------------------------ bool needsFetch = false; if (forcePending) needsFetch = true; // A pending failure is a trigger in its own right; without it, an error on // a normal refresh day would leave nothing to re-open the gate and the // backoff would never be acted on. else if (consecFails > 0) needsFetch = true; else if (weeksRequired > satisfiedWeeks) needsFetch = true; // config wants more history else if (lastAskDate == 0) needsFetch = true; else if (todayDate < lastAskDate) needsFetch = true; // clock moved back else if (todayDate - lastAskDate >= refreshDays) needsFetch = true; else if (rows.empty()) needsFetch = true; if (replaying) needsFetch = false; // backtests never touch the network // Give up after repeated failures: the cache keeps being displayed and // [8] Force Refresh is the way back. if (consecFails >= MAX_AUTO_RETRIES && !forcePending) needsFetch = false; // Wait out the backoff. if (needsFetch && consecFails > 0 && nextRetryAt.GetAsDouble() > 0.0 && sc.CurrentSystemDateTime < nextRetryAt) needsFetch = false; if (httpState == HTTP_IDLE && needsFetch) { // History is sized independently of the COT Index lookback: in Net // Positions mode the lookback is meaningless, and letting it size the // download would silently truncate how much history exists at all. const int weeksWanted = weeksRequired; const SCString appToken = snp_cot_input_is_set(InAppToken.GetString()) ? SCString(InAppToken.GetString()) : SCString(""); const SCString url = snp_cot_build_url(sch, code, weeksWanted, appToken); // Commit the "we asked" state ONLY once the request actually started. // Sierra allows one outstanding request per chart, so MakeHTTPRequest // can legitimately fail to start; stamping beforehand consumed both the // refresh window and the user's Force Refresh click for a request that // was never made. // // consecFails is deliberately NOT reset here. It counts FAILURES, and // a request that merely started has not succeeded yet -- clearing it on // start made it oscillate and defeated the give-up limit entirely. // It is cleared only when a download completes AND persists. if (sc.MakeHTTPRequest(url)) { httpState = HTTP_SENT; lastAskDate = todayDate; forcePending = 0; inflightSig = datasetSig; inflightWeeks = weeksWanted; requestSentAt = sc.CurrentSystemDateTime; } else { // forcePending deliberately survives, so a Force Refresh that could // not start is honoured later instead of being lost -- the backoff // is what stops that from becoming a per-call retry. sc.AddMessageToLog("COT: could not start the HTTP request (another request may be in progress).", 1); lastFetchError = 1; ++consecFails; const int backoffSec = (consecFails == 1) ? 5 : (consecFails == 2) ? 15 : 60; nextRetryAt = sc.CurrentSystemDateTime; nextRetryAt += SCDateTime::SECONDS(backoffSec); } } // ------------------------------------------------------------------ // Derived data, cached across calls. // // Recomputing these on EVERY study call meant, on a tick-updating chart, // redoing a ~325,000-operation rolling min/max plus a full walk of the // chart's timestamps on every single tick. Both depend only on things // captured by the render signature below, so they are computed when that // changes and reused otherwise. // ------------------------------------------------------------------ void*& idxHolder = sc.GetPersistentPointer(1); void*& rowBarHolder = sc.GetPersistentPointer(2); if (idxHolder == NULL) idxHolder = new std::vector< std::vector >(); if (rowBarHolder == NULL) rowBarHolder = new std::vector(); std::vector< std::vector >& idx = *(std::vector< std::vector >*)idxHolder; std::vector& rowBar = *(std::vector*)rowBarHolder; // How many cached reports were already published as of the chart's last // bar. Declared here because the repaint decision below needs it. int& usableRowsP = sc.GetPersistentInt(20); int& reportsOnChart = sc.GetPersistentInt(21); // The index is computed whenever data exists, not only in Index display // mode, because subgraphs 12-15 must be readable by a trading system // regardless of what the chart is showing. const bool haveIndex = !rows.empty(); // Repaint signature. When new data arrives every bar must be rewritten, // not just the newest: the call delivering an HTTP response is not a full // recalculation, so sc.UpdateStartIndex is the last bar, and writing years // of history into that single bar would leave a one-point line that // DRAWSTYLE_LINE cannot draw. Anything else that changes what is drawn -- // display mode, report type, visibility, contract -- is folded into the // same signature. int sig = (int)rows.size() * 131; if (!rows.empty()) sig ^= (int)(snp_cot_stamp(rows.back()) & 0x7FFFFF); sig ^= datasetSig; sig ^= (displayMode ? 0x40000 : 0) ^ (showStudy ? 0x80000 : 0) ^ (InLookback.GetInt() << 9); // sc.ArraySize is deliberately not part of the signature: including it // would force a complete recompute on every new bar, which an ordinary new // bar does not require. A new bar only matters when it crosses the // publication time of the next report and makes it available; that case is // detected explicitly, and everything else is covered by the partial // repaint from sc.UpdateStartIndex. bool nextReportBecameAvailable = false; if (usableRowsP > 0 && usableRowsP < (int)rows.size() && sc.ArraySize > 0 && rowBar.size() == rows.size()) { const double nextPub = snp_cot_publication_datetime(sc, rows[usableRowsP]).GetAsDouble(); if (sc.BaseDateTimeIn[sc.ArraySize - 1].GetAsDouble() >= nextPub) nextReportBecameAvailable = true; } const bool fullRepaint = rowsChanged || (sig != renderSig) || nextReportBecameAvailable || (sc.UpdateStartIndex == 0); renderSig = sig; int startIdx = fullRepaint ? 0 : sc.UpdateStartIndex; if (startIdx < 0) startIdx = 0; // ------------------------------------------------------------------ // Causal bar mapping. // // Each report is mapped to the first bar at or after its publication // moment, by walking the chart's own timestamps. // // sc.GetNearestMatchForSCDateTime() must NOT be used for this: it clamps // a Date-Time past the end of the chart to sc.ArraySize - 1. On any chart // that does not run to the present, that would pin every future report // onto the last bar and silently display it there. // // Reports published after the final bar map to NOT_AVAILABLE and are // never drawn, which also makes replay correct: a bar can only carry // reports that were already public at that bar. // ------------------------------------------------------------------ enum { COT_BAR_NOT_AVAILABLE = -1 }; size_t cacheIdx = 0; if (fullRepaint) { usableRowsP = 0; reportsOnChart = 0; idx.clear(); rowBar.clear(); if (haveIndex) snp_cot_compute_index(rows, sch.NumCats, InLookback.GetInt(), idx); if (!rows.empty() && sc.ArraySize > 0) { rowBar.assign(rows.size(), COT_BAR_NOT_AVAILABLE); // Both sequences ascend, so one forward pass over each suffices. int barPtr = 0; for (size_t r = 0; r < rows.size(); ++r) { const double pub = snp_cot_publication_datetime(sc, rows[r]).GetAsDouble(); while (barPtr < sc.ArraySize && sc.BaseDateTimeIn[barPtr].GetAsDouble() < pub) ++barPtr; if (barPtr >= sc.ArraySize) break; // this and every later report are still in the future here rowBar[r] = barPtr; usableRowsP = (int)(r + 1); } // How many reports actually land INSIDE the chart rather than being // folded onto bar 0 (everything published before the chart begins // maps there). On a short or intraday chart that count is 1, every // bar inherits the same reading, and the plot is necessarily a // straight line -- correct, but useless, so it gets reported rather // than drawn without explanation. int prevBar = -1; for (int r = 0; r < usableRowsP; ++r) { if (rowBar[r] > 0 && rowBar[r] != prevBar) { ++reportsOnChart; prevBar = rowBar[r]; } } } } const size_t usableRows = (rowBar.size() >= (size_t)usableRowsP && usableRowsP > 0) ? (size_t)usableRowsP : 0; // Seed the forward pointer for a partial repaint. while (cacheIdx + 1 < usableRows && rowBar[cacheIdx + 1] <= startIdx) ++cacheIdx; for (int i = startIdx; i < sc.ArraySize; ++i) { // Clear everything first; only what this mode uses is filled back in. for (int k = 0; k < COT_MAX_CATS; ++k) { sc.Subgraph[k][i] = 0; sc.Subgraph[COT_SG_NET_BASE + k][i] = 0; sc.Subgraph[COT_SG_IDX_BASE + k][i] = (float)COT_INDEX_NA; } sc.Subgraph[COT_SG_VALID][i] = 0; sc.Subgraph[COT_SG_REPORTDATE][i] = 0; Sg_ZeroLine[i] = 0; Sg_Ref80[i] = 0; Sg_Ref50[i] = 0; Sg_Ref20[i] = 0; if (usableRows == 0) continue; while (cacheIdx + 1 < usableRows && rowBar[cacheIdx + 1] <= i) ++cacheIdx; if (rowBar[cacheIdx] > i) continue; // bar predates the first published report // ---- machine-readable slots: always filled, whatever is displayed ---- const bool idxReady = haveIndex && cacheIdx < idx[0].size() && idx[0][cacheIdx] != COT_INDEX_NA; for (int k = 0; k < sch.NumCats; ++k) { sc.Subgraph[COT_SG_NET_BASE + k][i] = (float)rows[cacheIdx].Net(k); sc.Subgraph[COT_SG_IDX_BASE + k][i] = idxReady ? (float)idx[k][cacheIdx] : (float)COT_INDEX_NA; } sc.Subgraph[COT_SG_VALID][i] = 1.0f; // Sierra's own date serial (~46,000), NOT YYYYMMDD. Subgraphs hold // float, which represents integers exactly only up to 16,777,216 -- // so 20260707 would silently store as 20260708 and report the wrong // day. The serial is far below that limit and converts back with // SCDateTime::SetDate(). sc.Subgraph[COT_SG_REPORTDATE][i] = (float)snp_cot_row_datetime(rows[cacheIdx]).GetDate(); if (!showStudy) continue; // In Index display mode a row without a full lookback window has no // defined value; leave the bar blank rather than drawing a fabricated // number (and leave the reference lines off too, so the empty stretch // is visibly empty rather than looking like a flat reading). if (displayMode == DISP_INDEX && !idxReady) continue; for (int k = 0; k < sch.NumCats; ++k) { const double v = (displayMode == DISP_INDEX) ? idx[k][cacheIdx] : rows[cacheIdx].Net(k); sc.Subgraph[k][i] = (float)snp_cot_nudge_zero(v); } // Reference lines belong to exactly one mode each -- drawing 80/50/20 // on a +/-160,000-contract scale is what produced the flat smear. if (displayMode == DISP_INDEX) { Sg_Ref80[i] = 80.0f; Sg_Ref50[i] = 50.0f; Sg_Ref20[i] = 20.0f; } else { Sg_ZeroLine[i] = (float)snp_cot_nudge_zero(0.0); } } // ------------------------------------------------------------------ // Info panel -- facts only (which contract, which report, how fresh, // current readings). No bullish/bearish narrative: the number is the // fact, reading direction into it is the trader's job. // ------------------------------------------------------------------ if (!showStudy || InShowHUD.GetYesNo() == 0) { for (int L = 0; L < COT_HUD_MAX_LINES; ++L) sc.DeleteACSChartDrawing(sc.ChartNumber, TOOL_DELETE_CHARTDRAWING, hudLineNumber + L); return; } // Build the panel as (text, colour) rows. Each becomes its own drawing // below, because a DRAWING_TEXT carries a single colour for all of its // contents and each category row must match its plotted line. SCString hudText[COT_HUD_MAX_LINES]; COLORREF hudColor[COT_HUD_MAX_LINES]; int hudCount = 0; hudText[hudCount] = SCString().Format(" COT v.Prod %s (CFTC %s) %s", resolvedTicker.GetChars(), code.GetChars(), sch.Label); hudColor[hudCount++] = COT_COLOR_TEXT; if (reportSubstituted && hudCount < COT_HUD_MAX_LINES) { hudText[hudCount] = SCString().Format(" %s has no data here -- using %s", (requestedReport == RPT_TFF) ? "TFF" : "Disaggregated", sch.Label); hudColor[hudCount++] = COT_COLOR_WARN; } if (rows.empty()) { if (httpState == HTTP_SENT) { hudText[hudCount] = " Downloading from CFTC..."; hudColor[hudCount++] = COT_COLOR_DIM; } else if (lastFetchError != 0) { hudText[hudCount] = " Download FAILED -- see Message Log"; hudColor[hudCount++] = COT_COLOR_ERROR; } else { hudText[hudCount] = " No data yet."; hudColor[hudCount++] = COT_COLOR_DIM; } } else if (usableRows == 0) { // Data is cached, but none of it was published by the chart's last // bar, so there is nothing this chart may show. hudText[hudCount] = " no report released yet at this chart's last bar"; hudColor[hudCount++] = COT_COLOR_DIM; } else { // The panel shows the same report the chart is allowed to plot, not // the newest row in the cache; on a chart that does not run to the // present the newest row is a report from after the last bar. const size_t hudRow = usableRows - 1; const s_CotRow& last = rows[hudRow]; // Age is measured against the chart's last bar rather than wall-clock, // which is the meaningful figure on a historical chart and identical // on a live one. const SCDateTime contextTime = (sc.ArraySize > 0) ? sc.BaseDateTimeIn[sc.ArraySize - 1] : sc.CurrentSystemDateTime; const double ageDays = contextTime.GetAsDouble() - snp_cot_row_datetime(last).GetAsDouble(); const bool bad = (lastFetchError != 0); const char* status = (httpState == HTTP_SENT) ? "updating..." : bad ? "ERROR - see Message Log" : "ok"; hudText[hudCount] = SCString().Format(" as of %04d-%02d-%02d (%.0fd ago) [%s] %s", last.Y, last.M, last.D, ageDays, status, sc.Symbol.GetChars()); hudColor[hudCount++] = bad ? COT_COLOR_ERROR : COT_COLOR_DIM; if (hudCount < COT_HUD_MAX_LINES) { hudText[hudCount] = SCString().Format(" %d reports held | %d on this chart", (int)rows.size(), reportsOnChart); hudColor[hudCount++] = COT_COLOR_DIM; } // COT is a WEEKLY series. On a chart whose visible range is shorter // than a few weeks (or an intraday chart), every bar inherits the same // single report and the line is necessarily dead straight. if (reportsOnChart < 3 && hudCount < COT_HUD_MAX_LINES) { hudText[hudCount] = " chart too short -- COT is weekly; use Daily, 1yr+"; hudColor[hudCount++] = COT_COLOR_WARN; } if (hudCount < COT_HUD_MAX_LINES) { hudText[hudCount] = (displayMode == DISP_INDEX) ? " -- COT Index (0-100) --" : " -- Net position, contracts --"; hudColor[hudCount++] = COT_COLOR_TEXT; } // One row per category, tinted to match that category's plotted line. for (int k = 0; k < sch.NumCats && hudCount < COT_HUD_MAX_LINES; ++k) { if (displayMode == DISP_INDEX) { const double v = (idx.empty() || hudRow >= idx[k].size()) ? COT_INDEX_NA : idx[k][hudRow]; hudText[hudCount] = (v == COT_INDEX_NA) ? SCString().Format(" %-17s n/a", sch.CatName[k]) : SCString().Format(" %-17s %6.0f", sch.CatName[k], v); } else { hudText[hudCount] = SCString().Format(" %-17s %+9.0f", sch.CatName[k], last.Net(k)); } hudColor[hudCount++] = COT_CAT_COLOR[k]; } // The index needs a full lookback window before it means anything -- // say so explicitly instead of leaving an unexplained blank stretch. if (displayMode == DISP_INDEX && !idx.empty() && hudRow < idx[0].size() && idx[0][hudRow] == COT_INDEX_NA && hudCount < COT_HUD_MAX_LINES) { hudText[hudCount] = SCString().Format(" needs %d reports, have %d", InLookback.GetInt(), (int)rows.size()); hudColor[hudCount++] = COT_COLOR_WARN; } } // Emit one drawing per row, then clear any rows left over from a longer // previous panel so stale text cannot linger underneath. // // Guarded against DownloadingHistoricalData only. A full recalculation must // NOT be excluded here: a settings change is a full recalculation, and the // panel has to reflect the new setting. The usual per-bar UseTool concern // does not apply, as this is a fixed handful of drawings per study call. if (!sc.DownloadingHistoricalData) { for (int L = 0; L < hudCount; ++L) { s_UseTool tool; tool.Clear(); tool.ChartNumber = sc.ChartNumber; tool.DrawingType = DRAWING_TEXT; tool.AddMethod = UTAM_ADD_OR_ADJUST; tool.AllowSaveToChartbook = 0; tool.Region = sc.GraphRegion; tool.LineNumber = hudLineNumber + L; tool.BeginDateTime = 3; tool.UseRelativeVerticalValues = 1; tool.BeginValue = COT_HUD_TOP_PCT - COT_HUD_STEP_PCT * L; tool.Text = hudText[L]; tool.FontSize = 9; tool.FontBold = 1; tool.TextAlignment = DT_LEFT | DT_TOP; tool.Color = hudColor[L]; tool.FontBackColor = RGB(18, 18, 18); sc.UseTool(tool); } for (int L = hudCount; L < COT_HUD_MAX_LINES; ++L) sc.DeleteACSChartDrawing(sc.ChartNumber, TOOL_DELETE_CHARTDRAWING, hudLineNumber + L); } }