diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..b864276 Binary files /dev/null and b/.DS_Store differ diff --git a/Dockerfile b/Dockerfile index b8de174..b188160 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,9 @@ RUN mkdir /intrinio COPY . /intrinio -WORKDIR /intrinio/IntrinioRealTimeSDK +WORKDIR /intrinio/SampleApp -RUN dotnet build IntrinioRealTimeSDK.csproj +RUN dotnet build SampleApp.csproj -CMD dotnet run IntrinioRealTimeSDK.csproj +CMD dotnet run SampleApp.csproj diff --git a/Intrinio.Realtime/CandleStick.cs b/Intrinio.Realtime/CandleStick.cs new file mode 100644 index 0000000..681d492 --- /dev/null +++ b/Intrinio.Realtime/CandleStick.cs @@ -0,0 +1,106 @@ +using System.Globalization; + +namespace Intrinio.Realtime; + +using System; + +public abstract class CandleStick +{ + private readonly string _contract; + private readonly double _openTimestamp; + private readonly double _closeTimestamp; + private readonly IntervalType _interval; + + public UInt32 Volume { get; set; } + public double High { get; set; } + public double Low { get; set; } + public double Close { get; set; } + public double Open { get; set; } + public double OpenTimestamp + { + get { return _openTimestamp; } + } + public double CloseTimestamp + { + get { return _closeTimestamp; } + } + public double FirstTimestamp { get; set; } + public double LastTimestamp { get; set; } + public bool Complete { get; set; } + public double Average { get; set; } + public double Change { get; set; } + public IntervalType Interval + { + get { return _interval; } + } + + public CandleStick(UInt32 volume, double price, double openTimestamp, double closeTimestamp, IntervalType interval, double eventTime) + { + Volume = volume; + High = price; + Low = price; + Close = price; + Open = price; + _openTimestamp = openTimestamp; + _closeTimestamp = closeTimestamp; + FirstTimestamp = eventTime; + LastTimestamp = eventTime; + Complete = false; + Average = price; + Change = 0.0; + _interval = interval; + } + + public CandleStick(UInt32 volume, double high, double low, double closePrice, double openPrice, double openTimestamp, double closeTimestamp, double firstTimestamp, double lastTimestamp, bool complete, double average, double change, IntervalType interval) + { + Volume = volume; + High = high; + Low = low; + Close = closePrice; + Open = openPrice; + _openTimestamp = openTimestamp; + _closeTimestamp = closeTimestamp; + FirstTimestamp = firstTimestamp; + LastTimestamp = lastTimestamp; + Complete = complete; + Average = average; + Change = change; + _interval = interval; + } + + public void Merge(CandleStick candle) + { + Average = ((Convert.ToDouble(Volume) * Average) + (Convert.ToDouble(candle.Volume) * candle.Average)) / (Convert.ToDouble(Volume + candle.Volume)); + Volume += candle.Volume; + High = High > candle.High ? High : candle.High; + Low = Low < candle.Low ? Low : candle.Low; + Close = LastTimestamp > candle.LastTimestamp ? Close : candle.Close; + Open = FirstTimestamp < candle.FirstTimestamp ? Open : candle.Open; + FirstTimestamp = candle.FirstTimestamp < FirstTimestamp ? candle.FirstTimestamp : FirstTimestamp; + LastTimestamp = candle.LastTimestamp > LastTimestamp ? candle.LastTimestamp : LastTimestamp; + Change = (Close - Open) / Open; + } + + internal void Update(UInt32 volume, double price, double time) + { + Average = ((Convert.ToDouble(Volume) * Average) + (Convert.ToDouble(volume) * price)) / (Convert.ToDouble(Volume + volume)); + Volume += volume; + High = price > High ? price : High; + Low = price < Low ? price : Low; + Close = time > LastTimestamp ? price : Close; + Open = time < FirstTimestamp ? price : Open; + FirstTimestamp = time < FirstTimestamp ? time : FirstTimestamp; + LastTimestamp = time > LastTimestamp ? time : LastTimestamp; + Change = (Close - Open) / Open; + } + + internal void MarkComplete() + { + Complete = true; + } + + internal void MarkIncomplete() + { + Complete = false; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/ClientStats.cs b/Intrinio.Realtime/ClientStats.cs new file mode 100644 index 0000000..c093386 --- /dev/null +++ b/Intrinio.Realtime/ClientStats.cs @@ -0,0 +1,47 @@ +namespace Intrinio.Realtime; + +using System; + +public class ClientStats +{ + private readonly UInt64 _socketDataMessages; + private readonly UInt64 _socketTextMessages; + private readonly int _queueDepth; + private readonly UInt64 _eventCount; + private readonly int _queueCapacity; + private readonly int _overflowQueueDepth; + private readonly int _overflowQueueCapacity; + private readonly int _droppedCount; + private readonly int _overflowCount; + + public ClientStats(UInt64 socketDataMessages, UInt64 socketTextMessages, int queueDepth, UInt64 eventCount, int queueCapacity, int overflowQueueDepth, int overflowQueueCapacity, int droppedCount, int overflowCount) + { + _socketDataMessages = socketDataMessages; + _socketTextMessages = socketTextMessages; + _queueDepth = queueDepth; + _eventCount = eventCount; + _queueCapacity = queueCapacity; + _overflowQueueDepth = overflowQueueDepth; + _overflowQueueCapacity = overflowQueueCapacity; + _droppedCount = droppedCount; + _overflowCount = overflowCount; + } + + public UInt64 SocketDataMessages { get { return _socketDataMessages; } } + + public UInt64 SocketTextMessages { get { return _socketTextMessages; } } + + public int QueueDepth { get { return _queueDepth; } } + + public int QueueCapacity { get { return _queueCapacity; } } + + public int OverflowQueueDepth { get { return _overflowQueueDepth; } } + + public int OverflowQueueCapacity { get { return _overflowQueueCapacity; } } + + public UInt64 EventCount { get { return _eventCount; } } + + public int DroppedCount { get { return _droppedCount; } } + + public int OverflowCount { get { return _overflowCount; } } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Composite/Delegates.cs b/Intrinio.Realtime/Composite/Delegates.cs new file mode 100644 index 0000000..6c4ae4c --- /dev/null +++ b/Intrinio.Realtime/Composite/Delegates.cs @@ -0,0 +1,20 @@ +namespace Intrinio.Realtime.Composite; + +using System; +using System.Threading.Tasks; + +public delegate Task OnSupplementalDatumUpdated(String key, double datum, IDataCache dataCache); +public delegate Task OnSecuritySupplementalDatumUpdated(String key, double datum, ISecurityData securityData, IDataCache dataCache); +public delegate Task OnOptionsContractSupplementalDatumUpdated(String key, double datum, IOptionsContractData optionsContractData, ISecurityData securityData, IDataCache dataCache); + +public delegate Task OnEquitiesTradeUpdated(ISecurityData securityData, IDataCache dataCache); +public delegate Task OnEquitiesQuoteUpdated(ISecurityData SecurityData, IDataCache DataCache); +public delegate Task OnEquitiesTradeCandleStickUpdated(ISecurityData securityData, IDataCache dataCache); +public delegate Task OnEquitiesQuoteCandleStickUpdated(ISecurityData SecurityData, IDataCache DataCache); + +public delegate Task OnOptionsTradeUpdated(IOptionsContractData optionsContractData, IDataCache dataCache, ISecurityData securityData); +public delegate Task OnOptionsQuoteUpdated(IOptionsContractData optionsContractData, IDataCache dataCache, ISecurityData securityData); +public delegate Task OnOptionsRefreshUpdated(IOptionsContractData optionsContractData, IDataCache dataCache, ISecurityData securityData); +public delegate Task OnOptionsUnusualActivityUpdated(IOptionsContractData optionsContractData, IDataCache dataCache, ISecurityData securityData); +public delegate Task OnOptionsTradeCandleStickUpdated(IOptionsContractData optionsContractData, IDataCache dataCache, ISecurityData securityData); +public delegate Task OnOptionsQuoteCandleStickUpdated(IOptionsContractData optionsContractData, IDataCache dataCache, ISecurityData securityData); \ No newline at end of file diff --git a/Intrinio.Realtime/Composite/IDataCache.cs b/Intrinio.Realtime/Composite/IDataCache.cs new file mode 100644 index 0000000..1c2b0c8 --- /dev/null +++ b/Intrinio.Realtime/Composite/IDataCache.cs @@ -0,0 +1,76 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Composite; + +using System; + +/// +/// Not for Use yet. Subject to change. +/// +public interface IDataCache +{ + double? GetsupplementaryDatum(string key); + Task SetsupplementaryDatum(string key, double? datum); + IReadOnlyDictionary AllSupplementaryData { get; } + + ISecurityData GetSecurityData(string tickerSymbol); + IReadOnlyDictionary AllSecurityData { get; } + + Intrinio.Realtime.Equities.Trade? GetLatestEquityTrade(string tickerSymbol); + Task SetEquityTrade(Intrinio.Realtime.Equities.Trade trade); + + Intrinio.Realtime.Equities.Quote? GetLatestEquityQuote(string tickerSymbol); + Task SetEquityQuote(Intrinio.Realtime.Equities.Quote quote); + + Intrinio.Realtime.Equities.TradeCandleStick? GetLatestEquityTradeCandleStick(string tickerSymbol); + Task SetEquityTradeCandleStick(Intrinio.Realtime.Equities.TradeCandleStick tradeCandleStick); + + Intrinio.Realtime.Equities.QuoteCandleStick? GetLatestEquityAskQuoteCandleStick(string tickerSymbol); + Intrinio.Realtime.Equities.QuoteCandleStick? GetLatestEquityBidQuoteCandleStick(string tickerSymbol); + Task SetEquityQuoteCandleStick(Intrinio.Realtime.Equities.QuoteCandleStick quoteCandleStick); + + IOptionsContractData GetOptionsContractData(string tickerSymbol, string contract); + + Intrinio.Realtime.Options.Trade? GetLatestOptionsTrade(string tickerSymbol, string contract); + Task SetOptionsTrade(Intrinio.Realtime.Options.Trade trade); + + Intrinio.Realtime.Options.Quote? GetLatestOptionsQuote(string tickerSymbol, string contract); + Task SetOptionsQuote(Intrinio.Realtime.Options.Quote quote); + + Intrinio.Realtime.Options.Refresh? GetLatestOptionsRefresh(string tickerSymbol, string contract); + Task SetOptionsRefresh(Intrinio.Realtime.Options.Refresh refresh); + + Intrinio.Realtime.Options.UnusualActivity? GetLatestOptionsUnusualActivity(string tickerSymbol, string contract); + Task SetOptionsUnusualActivity(Intrinio.Realtime.Options.UnusualActivity unusualActivity); + + Intrinio.Realtime.Options.TradeCandleStick? GetLatestOptionsTradeCandleStick(string tickerSymbol, string contract); + Task SetOptionsTradeCandleStick(Intrinio.Realtime.Options.TradeCandleStick tradeCandleStick); + + Intrinio.Realtime.Options.QuoteCandleStick? GetOptionsAskQuoteCandleStick(string tickerSymbol); + Intrinio.Realtime.Options.QuoteCandleStick? GetOptionsBidQuoteCandleStick(string tickerSymbol); + Task SetOptionsQuoteCandleStick(Intrinio.Realtime.Options.QuoteCandleStick quoteCandleStick); + + double? GetSecuritySupplementalDatum(string tickerSymbol, string key); + Task SetSecuritySupplementalDatum(string tickerSymbol, string key, double? datum); + + double? GetOptionsContractSupplementalDatum(string tickerSymbol, string contract, string key); + Task SetOptionSupplementalDatum(string tickerSymbol, string contract, string key, double? datum); + + void SetOnSupplementalDatumUpdated(OnSupplementalDatumUpdated onSupplementalDatumUpdated); + void SetOnSecuritySupplementalDatumUpdated(OnSecuritySupplementalDatumUpdated onSecuritySupplementalDatumUpdated); + void SetOnOptionSupplementalDatumUpdated(OnOptionsContractSupplementalDatumUpdated onOptionsContractSupplementalDatumUpdated); + + void SetOnEquitiesTradeUpdated(OnEquitiesTradeUpdated onEquitiesTradeUpdated); + void SetOnEquitiesQuoteUpdated(OnEquitiesQuoteUpdated onEquitiesQuoteUpdated); + void SetOnEquitiesTradeCandleStickUpdated(OnEquitiesTradeCandleStickUpdated onEquitiesTradeCandleStickUpdated); + void SetOnEquitiesQuoteCandleStickUpdated(OnEquitiesQuoteCandleStickUpdated onEquitiesQuoteCandleStickUpdated); + + void SetOnOptionsTradeUpdated(OnOptionsTradeUpdated onOptionsTradeUpdated); + void SetOnOptionsQuoteUpdated(OnOptionsQuoteUpdated onOptionsQuoteUpdated); + void SetOnOptionsRefreshUpdated(OnOptionsRefreshUpdated onOptionsRefreshUpdated); + void SetOnOptionsUnusualActivityUpdated(OnOptionsUnusualActivityUpdated onOptionsUnusualActivityUpdated); + void SetOnOptionsTradeCandleStickUpdated(OnOptionsTradeCandleStickUpdated onOptionsTradeCandleStickUpdated); + void SetOnOptionsQuoteCandleStickUpdated(OnOptionsQuoteCandleStickUpdated onOptionsQuoteCandleStickUpdated); +} diff --git a/Intrinio.Realtime/Composite/IOptionsContractData.cs b/Intrinio.Realtime/Composite/IOptionsContractData.cs new file mode 100644 index 0000000..416a8d8 --- /dev/null +++ b/Intrinio.Realtime/Composite/IOptionsContractData.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Composite; + +/// +/// Not for Use yet. Subject to change. +/// +public interface IOptionsContractData { + string Contract { get; } + + Intrinio.Realtime.Options.Trade? LatestTrade { get; } + Intrinio.Realtime.Options.Quote? LatestQuote { get; } + Intrinio.Realtime.Options.Refresh? LatestRefresh { get; } + Intrinio.Realtime.Options.UnusualActivity? LatestUnusualActivity { get; } + Intrinio.Realtime.Options.TradeCandleStick? LatestTradeCandleStick { get; } + Intrinio.Realtime.Options.QuoteCandleStick? LatestAskQuoteCandleStick { get; } + Intrinio.Realtime.Options.QuoteCandleStick? LatestBidQuoteCandleStick { get; } + + Task SetTrade(Intrinio.Realtime.Options.Trade trade); + Task SetQuote(Intrinio.Realtime.Options.Quote quote); + Task SetRefresh(Intrinio.Realtime.Options.Refresh refresh); + Task SetUnusualActivity(Intrinio.Realtime.Options.UnusualActivity unusualActivity); + Task SetTradeCandleStick(Intrinio.Realtime.Options.TradeCandleStick tradeCandleStick); + Task SetQuoteCandleStick(Intrinio.Realtime.Options.QuoteCandleStick quoteCandleStick); + + double? GetSupplementaryDatum(string key); + Task SetSupplementaryDatum(string key, double? datum); + IReadOnlyDictionary AllSupplementaryData { get; } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Composite/ISecurityData.cs b/Intrinio.Realtime/Composite/ISecurityData.cs new file mode 100644 index 0000000..bb65a54 --- /dev/null +++ b/Intrinio.Realtime/Composite/ISecurityData.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Composite; + +/// +/// Not for Use yet. Subject to change. +/// +public interface ISecurityData { + string TickerSymbol { get; } + + double? GetSupplementaryDatum(string key); + Task SetSupplementaryDatum(string key, double? datum); + IReadOnlyDictionary AllSupplementaryData { get; } + + Intrinio.Realtime.Equities.Trade? LatestEquitiesTrade { get; } + Intrinio.Realtime.Equities.Quote? LatestEquitiesQuote { get; } + + Intrinio.Realtime.Equities.TradeCandleStick? LatestEquitiesTradeCandleStick { get; } + Intrinio.Realtime.Equities.QuoteCandleStick? LatestEquitiesAskQuoteCandleStick { get; } + Intrinio.Realtime.Equities.QuoteCandleStick? LatestEquitiesBidQuoteCandleStick { get; } + + IOptionsContractData GetOptionsContractData(string contract); + IReadOnlyDictionary AllOptionsContractData { get; } + List GetContractNames(string ticker); + + Task SetEquitiesTrade(Intrinio.Realtime.Equities.Trade trade); + Task SetEquitiesQuote(Intrinio.Realtime.Equities.Quote quote); + + Task SetEquitiesTradeCandleStick(Intrinio.Realtime.Equities.TradeCandleStick tradeCandleStick); + Task SetEquitiesQuoteCandleStick(Intrinio.Realtime.Equities.QuoteCandleStick quoteCandleStick); + + Intrinio.Realtime.Options.Trade? GetLatestOptionsContractTrade(string contract); + Task SetOptionsContractTrade(Intrinio.Realtime.Options.Trade trade); + + Intrinio.Realtime.Options.Quote? GetLatestOptionsContractQuote(string contract); + Task SetOptionsContractQuote(Intrinio.Realtime.Options.Quote quote); + + Intrinio.Realtime.Options.Refresh? GetLatestOptionsContractRefresh(string contract); + Task SetOptionsContractRefresh(Intrinio.Realtime.Options.Refresh refresh); + + Intrinio.Realtime.Options.UnusualActivity? GetLatestOptionsContractUnusualActivity(string contract); + Task SetOptionsContractUnusualActivity(Intrinio.Realtime.Options.UnusualActivity unusualActivity); + + double? GetOptionsContractSupplementalDatum(string contract, string key); + Task SetOptionsContractSupplementalDatum(string contract, string key, double? datum); +} \ No newline at end of file diff --git a/Intrinio.Realtime/Composite/OptionsContractData.cs b/Intrinio.Realtime/Composite/OptionsContractData.cs new file mode 100644 index 0000000..57bcb4b --- /dev/null +++ b/Intrinio.Realtime/Composite/OptionsContractData.cs @@ -0,0 +1,263 @@ +using System; +using System.Threading.Tasks; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using Intrinio.Realtime.Options; + +namespace Intrinio.Realtime.Composite; + +/// +/// Not for Use yet. Subject to change. +/// +public class OptionsContractData : IOptionsContractData +{ + private readonly String contract; + private Intrinio.Realtime.Options.Trade? _latestTrade; + private Intrinio.Realtime.Options.Quote? _latestQuote; + private Intrinio.Realtime.Options.Refresh? _latestRefresh; + private Intrinio.Realtime.Options.UnusualActivity? _latestUnusualActivity; + private Intrinio.Realtime.Options.TradeCandleStick? _latestTradeCandleStick; + private Intrinio.Realtime.Options.QuoteCandleStick? _latestAskQuoteCandleStick; + private Intrinio.Realtime.Options.QuoteCandleStick? _latestBidQuoteCandleStick; + private readonly ConcurrentDictionary _supplementaryData; + private readonly IReadOnlyDictionary _readonlySupplementaryData; + + public OptionsContractData( String contract, + Intrinio.Realtime.Options.Trade latestTrade, + Intrinio.Realtime.Options.Quote latestQuote, + Intrinio.Realtime.Options.Refresh latestRefresh, + Intrinio.Realtime.Options.UnusualActivity latestUnusualActivity, + Intrinio.Realtime.Options.TradeCandleStick latestTradeCandleStick, + Intrinio.Realtime.Options.QuoteCandleStick latestAskQuoteCandleStick, + Intrinio.Realtime.Options.QuoteCandleStick latestBidQuoteCandleStick) + { + this.contract = contract; + this._latestTrade = latestTrade; + this._latestQuote = latestQuote; + this._latestRefresh = latestRefresh; + this._latestUnusualActivity = latestUnusualActivity; + this._latestTradeCandleStick = latestTradeCandleStick; + this._latestAskQuoteCandleStick = latestAskQuoteCandleStick; + this._latestBidQuoteCandleStick = latestBidQuoteCandleStick; + this._supplementaryData = new ConcurrentDictionary(); + this._readonlySupplementaryData = new ReadOnlyDictionary(_supplementaryData); + } + + public String Contract { get { return this.contract; } } + + public Intrinio.Realtime.Options.Trade? LatestTrade { get { return this._latestTrade; } } + + public Intrinio.Realtime.Options.Quote? LatestQuote { get { return this._latestQuote; } } + + public Intrinio.Realtime.Options.Refresh? LatestRefresh { get { return this._latestRefresh; } } + + public Intrinio.Realtime.Options.UnusualActivity? LatestUnusualActivity { get { return this._latestUnusualActivity; } } + + public Intrinio.Realtime.Options.TradeCandleStick? LatestTradeCandleStick { get { return this._latestTradeCandleStick; } } + + public Intrinio.Realtime.Options.QuoteCandleStick? LatestAskQuoteCandleStick { get { return this._latestAskQuoteCandleStick; } } + + public Intrinio.Realtime.Options.QuoteCandleStick? LatestBidQuoteCandleStick { get { return this._latestBidQuoteCandleStick; } } + + public Task SetTrade(Intrinio.Realtime.Options.Trade trade) + { + //dirty set + if ((!_latestTrade.HasValue) || (trade.Timestamp > _latestTrade.Value.Timestamp)) + { + _latestTrade = trade; + return Task.FromResult(true); + } + return Task.FromResult(false); + } + + internal async Task SetTrade(Intrinio.Realtime.Options.Trade trade, OnOptionsTradeUpdated onOptionsTradeUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool isSet = await SetTrade(trade); + if (isSet && onOptionsTradeUpdated != null) + { + try + { + await onOptionsTradeUpdated(this, dataCache, securityData); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in OnOptionsTradeUpdated Callback: {0}", e.Message); + } + } + return isSet; + } + + public Task SetQuote(Intrinio.Realtime.Options.Quote quote) + { + //dirty set + if ((!_latestQuote.HasValue) || (quote.Timestamp > _latestQuote.Value.Timestamp)) + { + _latestQuote = quote; + return Task.FromResult(true); + } + return Task.FromResult(false); + } + + internal async Task SetQuote(Intrinio.Realtime.Options.Quote quote, OnOptionsQuoteUpdated onOptionsQuoteUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool isSet = await this.SetQuote(quote); + if (isSet && onOptionsQuoteUpdated != null) + { + try + { + await onOptionsQuoteUpdated(this, dataCache, securityData); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in onOptionsQuoteUpdated Callback: {0}", e.Message); + } + } + return isSet; + } + + public Task SetRefresh(Intrinio.Realtime.Options.Refresh refresh) + { + _latestRefresh = refresh; + return Task.FromResult(true); + } + + internal async Task SetRefresh(Intrinio.Realtime.Options.Refresh refresh, OnOptionsRefreshUpdated onOptionsRefreshUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool isSet = await this.SetRefresh(refresh); + if (isSet && onOptionsRefreshUpdated != null) + { + try + { + await onOptionsRefreshUpdated(this, dataCache, securityData); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in onOptionsRefreshUpdated Callback: {0}", e.Message); + } + } + return isSet; + } + + public Task SetUnusualActivity(Intrinio.Realtime.Options.UnusualActivity unusualActivity) + { + _latestUnusualActivity = unusualActivity; + return Task.FromResult(true); + } + + internal async Task SetUnusualActivity(Intrinio.Realtime.Options.UnusualActivity unusualActivity, OnOptionsUnusualActivityUpdated onOptionsUnusualActivityUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool isSet = await this.SetUnusualActivity(unusualActivity); + if (isSet && onOptionsUnusualActivityUpdated != null) + { + try + { + await onOptionsUnusualActivityUpdated(this, dataCache, securityData); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in onOptionsUnusualActivityUpdated Callback: {0}", e.Message); + } + } + return isSet; + } + + public Task SetTradeCandleStick(Intrinio.Realtime.Options.TradeCandleStick tradeCandleStick) + { + //dirty set + if ((_latestTradeCandleStick == null) || (tradeCandleStick.OpenTimestamp > _latestTradeCandleStick.OpenTimestamp) || (tradeCandleStick.LastTimestamp > _latestTradeCandleStick.LastTimestamp)) + { + _latestTradeCandleStick = tradeCandleStick; + return Task.FromResult(true); + } + return Task.FromResult(false); + } + + internal async Task SetTradeCandleStick(Intrinio.Realtime.Options.TradeCandleStick tradeCandleStick, OnOptionsTradeCandleStickUpdated onOptionsTradeCandleStickUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool isSet = await SetTradeCandleStick(tradeCandleStick); + if (isSet && onOptionsTradeCandleStickUpdated != null) + { + try + { + await onOptionsTradeCandleStickUpdated(this, dataCache, securityData); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in OnOptionsTradeCandleStickUpdated Callback: {0}", e.Message); + } + } + return isSet; + } + + public Task SetQuoteCandleStick(Intrinio.Realtime.Options.QuoteCandleStick quoteCandleStick) + { + switch (quoteCandleStick.QuoteType) + { + case QuoteType.Ask: + //dirty set + if ((_latestAskQuoteCandleStick == null) || (quoteCandleStick.OpenTimestamp > _latestAskQuoteCandleStick.OpenTimestamp) || (quoteCandleStick.LastTimestamp > _latestAskQuoteCandleStick.LastTimestamp)) + { + _latestAskQuoteCandleStick = quoteCandleStick; + return Task.FromResult(true); + } + return Task.FromResult(false); + case QuoteType.Bid: + //dirty set + if ((_latestBidQuoteCandleStick == null) || (quoteCandleStick.OpenTimestamp > _latestBidQuoteCandleStick.OpenTimestamp) || (quoteCandleStick.LastTimestamp > _latestBidQuoteCandleStick.LastTimestamp)) + { + _latestBidQuoteCandleStick = quoteCandleStick; + return Task.FromResult(true); + } + return Task.FromResult(false); + default: + return Task.FromResult(false); + } + } + + internal async Task SetQuoteCandleStick(Intrinio.Realtime.Options.QuoteCandleStick quoteCandleStick, OnOptionsQuoteCandleStickUpdated onOptionsQuoteCandleStickUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool isSet = await this.SetQuoteCandleStick(quoteCandleStick); + if (isSet && onOptionsQuoteCandleStickUpdated != null) + { + try + { + await onOptionsQuoteCandleStickUpdated(this, dataCache, securityData); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in onOptionsQuoteCandleStickUpdated Callback: {0}", e.Message); + } + } + return isSet; + } + + public double? GetSupplementaryDatum(String key) + { + return _supplementaryData.GetValueOrDefault(key, null); + } + + public Task SetSupplementaryDatum(String key, double? datum) + { + return Task.FromResult(datum == _supplementaryData.AddOrUpdate(key, datum, (key, oldValue) => datum)); + } + + internal async Task SetSupplementaryDatum(String key, double datum, OnOptionsContractSupplementalDatumUpdated onOptionsContractSupplementalDatumUpdated, ISecurityData securityData, IDataCache dataCache) + { + bool result = await SetSupplementaryDatum(key, datum); + if (result && onOptionsContractSupplementalDatumUpdated != null) + { + try + { + await onOptionsContractSupplementalDatumUpdated(key, datum, this, securityData, dataCache); + } + catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Error in onOptionsContractSupplementalDatumUpdated Callback: {0}", e.Message); + } + } + return result; + } + + public IReadOnlyDictionary AllSupplementaryData{ get { return _readonlySupplementaryData; } } +} \ No newline at end of file diff --git a/Intrinio.Realtime/DropOldestRingBuffer.cs b/Intrinio.Realtime/DropOldestRingBuffer.cs new file mode 100644 index 0000000..038f263 --- /dev/null +++ b/Intrinio.Realtime/DropOldestRingBuffer.cs @@ -0,0 +1,118 @@ +using System; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Intrinio.Realtime; + +internal class DropOldestRingBuffer +{ + #region Data Members + private readonly byte[] _data; + private uint _blockNextReadIndex; + private uint _blockNextWriteIndex; + private readonly object _readLock; + private readonly object _writeLock; + private ulong _count; + private readonly uint _blockSize; + private readonly uint _blockCapacity; + private ulong _dropCount; + + public ulong Count { get { return Interlocked.Read(ref _count); } } + public uint BlockSize { get { return _blockSize; } } + public uint BlockCapacity { get { return _blockCapacity; } } + public ulong DropCount { get { return Interlocked.Read(ref _dropCount); } } + + public bool IsEmpty + { + get + { + return IsEmptyNoLock(); + } + } + + public bool IsFull + { + get + { + return IsFullNoLock(); + } + } + #endregion //Data Members + + #region Constructors + + public DropOldestRingBuffer(uint blockSize, uint blockCapacity) + { + _blockSize = blockSize; + _blockCapacity = blockCapacity; + _blockNextReadIndex = 0u; + _blockNextWriteIndex = 0u; + _count = 0u; + _dropCount = 0UL; + _readLock = new object(); + _writeLock = new object(); + _data = new byte[blockSize * blockCapacity]; + } + + #endregion //Constructors + + /// + /// blockToWrite MUST be of length BlockSize! + /// + /// + public void Enqueue(ReadOnlySpan blockToWrite) + { + lock (_writeLock) + { + if (IsFullNoLock()) + { + lock (_readLock) + { + if (IsFullNoLock()) + { + _blockNextReadIndex = (++_blockNextReadIndex) % BlockCapacity; + Interlocked.Increment(ref _dropCount); + } + } + } + + Span target = new Span(_data, Convert.ToInt32(_blockNextWriteIndex * BlockSize), Convert.ToInt32(BlockSize)); + blockToWrite.CopyTo(target); + + _blockNextWriteIndex = (++_blockNextWriteIndex) % BlockCapacity; + Interlocked.Increment(ref _count); + } + } + + /// + /// blockBuffer MUST be of length BlockSize! + /// + /// + public bool TryDequeue(Span blockBuffer) + { + lock (_readLock) + { + if (IsEmptyNoLock()) + return false; + + Span target = new Span(_data, Convert.ToInt32(_blockNextReadIndex * BlockSize), Convert.ToInt32(BlockSize)); + target.CopyTo(blockBuffer); + + _blockNextReadIndex = (++_blockNextReadIndex) % BlockCapacity; + Interlocked.Decrement(ref _count); + return true; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsFullNoLock() + { + return Interlocked.Read(ref _count) == _blockCapacity; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsEmptyNoLock() + { + return Interlocked.Read(ref _count) == 0UL; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/CandleStickClient.cs b/Intrinio.Realtime/Equities/CandleStickClient.cs new file mode 100644 index 0000000..f6d707f --- /dev/null +++ b/Intrinio.Realtime/Equities/CandleStickClient.cs @@ -0,0 +1,720 @@ +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Equities; + +using Intrinio; +using Serilog; +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Runtime.CompilerServices; + +public delegate TradeCandleStick FetchHistoricalTradeCandleStick(string symbol, double openTimestamp, double closeTimestamp, IntervalType interval); +public delegate QuoteCandleStick FetchHistoricalQuoteCandleStick(string symbol, double openTimestamp, double closeTimestamp, QuoteType quoteType, IntervalType interval); + +public class CandleStickClient +{ + #region Data Members + private readonly IntervalType _interval; + private readonly bool _broadcastPartialCandles; + private readonly double _sourceDelaySeconds; + private readonly bool _useTradeFiltering; + private readonly CancellationTokenSource _ctSource; + private const int InitialDictionarySize = 31_387;//3_601_579; //a close prime number greater than 2x the max expected size. There are usually around 1.5m option contracts. + private readonly object _symbolBucketsLock; + private readonly object _lostAndFoundLock; + private readonly Dictionary _symbolBuckets; + private readonly Dictionary _lostAndFound; + private const double FlushBufferSeconds = 30.0; + private readonly Thread _lostAndFoundThread; + private readonly Thread _flushThread; + + private bool UseOnTradeCandleStick { get { return !ReferenceEquals(OnTradeCandleStick, null); } } + private bool UseOnQuoteCandleStick { get { return !ReferenceEquals(OnQuoteCandleStick, null); } } + private bool UseGetHistoricalTradeCandleStick { get { return !ReferenceEquals(GetHistoricalTradeCandleStick,null); } } + private bool UseGetHistoricalQuoteCandleStick { get { return !ReferenceEquals(GetHistoricalQuoteCandleStick,null); } } + + /// + /// The callback used for broadcasting trade candles. + /// + public Action OnTradeCandleStick { get; set; } + + /// + /// The callback used for broadcasting quote candles. + /// + private Action OnQuoteCandleStick { get; set; } + + /// + /// Fetch a previously broadcasted trade candlestick from the given unique parameters. + /// + public FetchHistoricalTradeCandleStick GetHistoricalTradeCandleStick { get; set; } + + /// + /// Fetch a previously broadcasted quote candlestick from the given unique parameters. + /// + public FetchHistoricalQuoteCandleStick GetHistoricalQuoteCandleStick { get; set; } + #endregion //Data Members + + #region Constructors + + /// + /// Creates an equities CandleStickClient that creates trade and quote candlesticks from a stream of trades and quotes. + /// + /// + /// + /// + /// + /// + /// + /// + /// + public CandleStickClient( + Action onTradeCandleStick, + Action onQuoteCandleStick, + IntervalType interval, + bool broadcastPartialCandles, + FetchHistoricalTradeCandleStick getHistoricalTradeCandleStick, + FetchHistoricalQuoteCandleStick getHistoricalQuoteCandleStick, + double sourceDelaySeconds, + bool useTradeFiltering) + { + this.OnTradeCandleStick = onTradeCandleStick; + this.OnQuoteCandleStick = onQuoteCandleStick; + this._interval = interval; + this._broadcastPartialCandles = broadcastPartialCandles; + this.GetHistoricalTradeCandleStick = getHistoricalTradeCandleStick; + this.GetHistoricalQuoteCandleStick = getHistoricalQuoteCandleStick; + this._sourceDelaySeconds = sourceDelaySeconds; + this._useTradeFiltering = useTradeFiltering; + _ctSource = new CancellationTokenSource(); + _symbolBucketsLock = new object(); + _lostAndFoundLock = new object(); + _symbolBuckets = new Dictionary(InitialDictionarySize); + _lostAndFound = new Dictionary(InitialDictionarySize); + _lostAndFoundThread = new Thread(new ThreadStart(LostAndFoundFn)); + _flushThread = new Thread(new ThreadStart(FlushFn)); + } + #endregion //Constructors + + #region Public Methods + + public void OnTrade(Trade trade) + { + try + { + if (UseOnTradeCandleStick && (!ShouldFilterTrade(trade, _useTradeFiltering))) + { + SymbolBucket bucket = GetSlot(trade.Symbol, _symbolBuckets, _symbolBucketsLock); + + lock (bucket.Locker) + { + double ts = ConvertToTimestamp(trade.Timestamp); + + if (bucket.TradeCandleStick != null) + { + if (bucket.TradeCandleStick.CloseTimestamp < ts) + { + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = CreateNewTradeCandle(trade, ts); + } + else if (bucket.TradeCandleStick.OpenTimestamp <= ts) + { + bucket.TradeCandleStick.Update(trade.Size, trade.Price, ts); + if (_broadcastPartialCandles) + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + } + else //This is a late trade. We already shipped the candle, so add to lost and found + { + AddTradeToLostAndFound(trade); + } + } + else + { + bucket.TradeCandleStick = CreateNewTradeCandle(trade, ts); + if (_broadcastPartialCandles) + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + } + } + } + } + catch (Exception e) + { + Log.Warning("Error on handling trade in CandleStick Client: {0}", e.Message); + } + } + + public void OnQuote(Quote quote) + { + try + { + if (UseOnQuoteCandleStick && (!(ShouldFilterQuote(quote, _useTradeFiltering)))) + { + SymbolBucket bucket = GetSlot(quote.Symbol, _symbolBuckets, _symbolBucketsLock); + + lock (bucket.Locker) + { + if (quote.Type == QuoteType.Ask) + { + OnAsk(quote, bucket); + } + + if (quote.Type == QuoteType.Bid) + { + OnBid(quote, bucket); + } + } + } + } + catch (Exception e) + { + Log.Warning("Error on handling trade in CandleStick Client: {0}", e.Message); + } + } + + public void Start() + { + if (!_flushThread.IsAlive) + { + _flushThread.Start(); + } + + if (!_lostAndFoundThread.IsAlive) + { + _lostAndFoundThread.Start(); + } + } + + public void Stop() + { + _ctSource.Cancel(); + } + #endregion //Public Methods + + #region Private Methods + + private TradeCandleStick CreateNewTradeCandle(Trade trade, double timestamp) + { + double start = GetNearestModInterval(timestamp, _interval); + TradeCandleStick freshCandle = new TradeCandleStick(trade.Symbol, trade.Size, trade.Price, start, (start + System.Convert.ToDouble((int)_interval)), _interval, timestamp); + + if (UseGetHistoricalTradeCandleStick && UseOnTradeCandleStick) + { + try + { + TradeCandleStick historical = GetHistoricalTradeCandleStick(freshCandle.Symbol, freshCandle.OpenTimestamp, freshCandle.CloseTimestamp, freshCandle.Interval); + if (ReferenceEquals(historical,null)) + return freshCandle; + historical.MarkIncomplete(); + return MergeTradeCandles(historical, freshCandle); + } + catch (Exception e) + { + Log.Error("Error retrieving historical TradeCandleStick: {0}; trade: {1}", e.Message, trade); + return freshCandle; + } + } + else + { + return freshCandle; + } + } + + private QuoteCandleStick CreateNewQuoteCandle(Quote quote, double timestamp) + { + double start = GetNearestModInterval(timestamp, _interval); + QuoteCandleStick freshCandle = new QuoteCandleStick(quote.Symbol, quote.Size, quote.Price, quote.Type, start, (start + System.Convert.ToDouble((int)_interval)), _interval, timestamp); + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(freshCandle.Symbol, freshCandle.OpenTimestamp, freshCandle.CloseTimestamp, freshCandle.QuoteType, freshCandle.Interval); + if (ReferenceEquals(historical,null)) + return freshCandle; + historical.MarkIncomplete(); + return MergeQuoteCandles(historical, freshCandle); + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}; quote: {1}", e.Message, quote); + return freshCandle; + } + } + else + { + return freshCandle; + } + } + + private void AddAskToLostAndFound(Quote ask) + { + double ts = ConvertToTimestamp(ask.Timestamp); + string key = String.Format("{0}|{1}|{2}", ask.Symbol, GetNearestModInterval(ts, _interval), _interval); + SymbolBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + try + { + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + lock (bucket.Locker) + { + if (bucket.AskCandleStick != null) + { + bucket.AskCandleStick.Update(ask.Size, ask.Price, ts); + } + else + { + double start = GetNearestModInterval(ts, _interval); + bucket.AskCandleStick = new QuoteCandleStick(ask.Symbol, ask.Size, ask.Price, QuoteType.Ask, start, (start + System.Convert.ToDouble((int)_interval)), _interval, ts); + } + } + } + } + catch (Exception ex) + { + Log.Warning("Error on handling late ask in CandleStick Client: {0}", ex.Message); + } + } + + private void AddBidToLostAndFound(Quote bid) + { + double ts = ConvertToTimestamp(bid.Timestamp); + string key = String.Format("{0}|{1}|{2}", bid.Symbol, GetNearestModInterval(ts, _interval), _interval); + SymbolBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + try + { + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + lock (bucket.Locker) + { + if (bucket.BidCandleStick != null) + { + bucket.BidCandleStick.Update(bid.Size, bid.Price, ts); + } + else + { + double start = GetNearestModInterval(ts, _interval); + bucket.BidCandleStick = new QuoteCandleStick(bid.Symbol, bid.Size, bid.Price, QuoteType.Bid, start, (start + System.Convert.ToDouble((int) _interval)), _interval, ts); + } + } + } + } + catch (Exception ex) + { + Log.Warning("Error on handling late bid in CandleStick Client: {0}", ex.Message); + } + } + + private void AddTradeToLostAndFound(Trade trade) + { + double ts = ConvertToTimestamp(trade.Timestamp); + string key = String.Format("{0}|{1}|{2}", trade.Symbol, GetNearestModInterval(ts, _interval), _interval); + SymbolBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + try + { + if (UseGetHistoricalTradeCandleStick && UseOnTradeCandleStick) + { + lock (bucket.Locker) + { + if (bucket.TradeCandleStick != null) + { + bucket.TradeCandleStick.Update(trade.Size, trade.Price, ts); + } + else + { + double start = GetNearestModInterval(ts, _interval); + bucket.TradeCandleStick = new TradeCandleStick(trade.Symbol, trade.Size, trade.Price, start, (start + System.Convert.ToDouble((int)_interval)), _interval, ts); + } + } + } + } + catch (Exception ex) + { + Log.Warning("Error on handling late trade in CandleStick Client: {0}", ex.Message); + } + } + + private void OnAsk(Quote quote, SymbolBucket bucket) + { + double ts = ConvertToTimestamp(quote.Timestamp); + + if (bucket.AskCandleStick != null && !Double.IsNaN(quote.Price)) + { + if (bucket.AskCandleStick.CloseTimestamp < ts) + { + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = CreateNewQuoteCandle(quote, ts); + } + else if (bucket.AskCandleStick.OpenTimestamp <= ts) + { + bucket.AskCandleStick.Update(quote.Size, quote.Price, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + } + else //This is a late event. We already shipped the candle, so add to lost and found + { + AddAskToLostAndFound(quote); + } + } + else if (bucket.AskCandleStick == null && !Double.IsNaN(quote.Price)) + { + bucket.AskCandleStick = CreateNewQuoteCandle(quote, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + } + } + + private void OnBid(Quote quote, SymbolBucket bucket) + { + double ts = ConvertToTimestamp(quote.Timestamp); + + if (bucket.BidCandleStick != null && !Double.IsNaN(quote.Price)) + { + if (bucket.BidCandleStick.CloseTimestamp < ts) + { + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = CreateNewQuoteCandle(quote, ts); + } + else if(bucket.BidCandleStick.OpenTimestamp <= ts) + { + bucket.BidCandleStick.Update(quote.Size, quote.Price, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + } + else //This is a late event. We already shipped the candle, so add to lost and found + { + AddBidToLostAndFound(quote); + } + } + else if (bucket.BidCandleStick == null && !Double.IsNaN(quote.Price)) + { + bucket.BidCandleStick = CreateNewQuoteCandle(quote, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + } + } + + private void FlushFn() + { + Log.Information("Starting candlestick expiration watcher..."); + CancellationToken ct = _ctSource.Token; + System.Threading.Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; + List keys = new List(); + + while (!ct.IsCancellationRequested) + { + try + { + lock (_symbolBucketsLock) + { + foreach (string key in _symbolBuckets.Keys) + keys.Add(key); + } + + foreach (string key in keys) + { + SymbolBucket bucket = GetSlot(key, _symbolBuckets, _symbolBucketsLock); + double flushThresholdTime = GetCurrentTimestamp(_sourceDelaySeconds) - FlushBufferSeconds; + + lock (bucket.Locker) + { + if (UseOnTradeCandleStick && bucket.TradeCandleStick != null && (bucket.TradeCandleStick.CloseTimestamp < flushThresholdTime)) + { + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + + if (UseOnQuoteCandleStick && bucket.AskCandleStick != null && (bucket.AskCandleStick.CloseTimestamp < flushThresholdTime)) + { + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + + if (UseOnQuoteCandleStick && bucket.BidCandleStick != null && (bucket.BidCandleStick.CloseTimestamp < flushThresholdTime)) + { + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + } + } + keys.Clear(); + + if (!(ct.IsCancellationRequested)) + Thread.Sleep(1000); + } + catch (OperationCanceledException) + { + } + } + + Log.Information("Stopping candlestick expiration watcher..."); + } + + private async void LostAndFoundFn() + { + Log.Information("Starting candlestick late event watcher..."); + CancellationToken ct = _ctSource.Token; + System.Threading.Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; + List keys = new List(); + + while (!ct.IsCancellationRequested) + { + try + { + lock (_lostAndFoundLock) + { + foreach (string key in _lostAndFound.Keys) + keys.Add(key); + } + + foreach (string key in keys) + { + SymbolBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + + lock (bucket.Locker) + { + if (UseGetHistoricalTradeCandleStick && UseOnTradeCandleStick && bucket.TradeCandleStick != null) + { + try + { + TradeCandleStick historical = GetHistoricalTradeCandleStick.Invoke(bucket.TradeCandleStick.Symbol, bucket.TradeCandleStick.OpenTimestamp, bucket.TradeCandleStick.CloseTimestamp, bucket.TradeCandleStick.Interval); + if (ReferenceEquals(historical,null)) + { + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + else + { + bucket.TradeCandleStick = MergeTradeCandles(historical, bucket.TradeCandleStick); + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + } + catch (Exception e) + { + Log.Error("Error retrieving historical TradeCandleStick: {0}", e.Message); + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + } + else + { + bucket.TradeCandleStick = null; + } + + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick && bucket.AskCandleStick != null) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(bucket.AskCandleStick.Symbol, bucket.AskCandleStick.OpenTimestamp, bucket.AskCandleStick.CloseTimestamp, bucket.AskCandleStick.QuoteType, bucket.AskCandleStick.Interval); + if (ReferenceEquals(historical,null)) + { + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + else + { + bucket.AskCandleStick = MergeQuoteCandles(historical, bucket.AskCandleStick); + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}", e.Message); + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + } + else + { + bucket.AskCandleStick = null; + } + + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick && bucket.BidCandleStick != null) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(bucket.BidCandleStick.Symbol, bucket.BidCandleStick.OpenTimestamp, bucket.BidCandleStick.CloseTimestamp, bucket.BidCandleStick.QuoteType, bucket.BidCandleStick.Interval); + if (ReferenceEquals(historical,null)) + { + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + else + { + bucket.BidCandleStick = MergeQuoteCandles(historical, bucket.BidCandleStick); + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}", e.Message); + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + } + else + { + bucket.BidCandleStick = null; + } + + if (bucket.TradeCandleStick == null && bucket.AskCandleStick == null && bucket.BidCandleStick == null) + RemoveSlot(key, _lostAndFound, _lostAndFoundLock); + } + } + keys.Clear(); + + if (!ct.IsCancellationRequested) + Thread.Sleep(1000); + } + catch (OperationCanceledException) + { + } + } + + Log.Information("Stopping candlestick late event watcher..."); + } + + #endregion //Private Methods + + private class SymbolBucket + { + public TradeCandleStick TradeCandleStick; + public QuoteCandleStick AskCandleStick; + public QuoteCandleStick BidCandleStick; + public object Locker; + + public SymbolBucket(TradeCandleStick tradeCandleStick, QuoteCandleStick askCandleStick, QuoteCandleStick bidCandleStick) + { + TradeCandleStick = tradeCandleStick; + AskCandleStick = askCandleStick; + BidCandleStick = bidCandleStick; + Locker = new object(); + } + } + + #region Private Static Methods + // [SkipLocalsInit] + // [MethodImpl(MethodImplOptions.AggressiveInlining)] + // private static Span StackAlloc(int length) where T : unmanaged + // { + // unsafe + // { + // Span p = stackalloc T[length]; + // return p; + // } + // } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static double GetCurrentTimestamp(double delay) + { + return (DateTime.UtcNow - DateTime.UnixEpoch.ToUniversalTime()).TotalSeconds - delay; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static double GetNearestModInterval(double timestamp, IntervalType interval) + { + return Convert.ToDouble(Convert.ToUInt64(timestamp) / Convert.ToUInt64((int)interval)) * Convert.ToDouble(((int)interval)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TradeCandleStick MergeTradeCandles(TradeCandleStick a, TradeCandleStick b) + { + a.Merge(b); + return a; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static QuoteCandleStick MergeQuoteCandles(QuoteCandleStick a, QuoteCandleStick b) + { + a.Merge(b); + return a; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsAnomalous(char marketCenter, string condition) + { + return marketCenter.Equals('L') && (condition.Equals("@ Zo", StringComparison.InvariantCultureIgnoreCase) || + condition.Equals("@ To", StringComparison.InvariantCultureIgnoreCase) || + condition.Equals("@ TW", StringComparison.InvariantCultureIgnoreCase)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsDarkPoolMarketCenter(char marketCenter) + { + return marketCenter.Equals((char)0) || Char.IsWhiteSpace(marketCenter) || marketCenter.Equals('D') || marketCenter.Equals('E'); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ShouldFilterTrade(Trade incomingTrade, bool useFiltering) + { + return useFiltering && (IsDarkPoolMarketCenter(incomingTrade.MarketCenter) || IsAnomalous(incomingTrade.MarketCenter, incomingTrade.Condition)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ShouldFilterQuote(Quote incomingQuote, bool useFiltering) + { + return useFiltering && IsDarkPoolMarketCenter(incomingQuote.MarketCenter); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static double ConvertToTimestamp(DateTime input) + { + return (input.ToUniversalTime() - DateTime.UnixEpoch.ToUniversalTime()).TotalSeconds; + } + + private static SymbolBucket GetSlot(string key, Dictionary dict, object locker) + { + SymbolBucket value; + if (dict.TryGetValue(key, out value)) + { + return value; + } + + lock (locker) + { + if (dict.TryGetValue(key, out value)) + { + return value; + } + + SymbolBucket bucket = new SymbolBucket(null, null, null); + dict.Add(key, bucket); + return bucket; + } + } + + private static void RemoveSlot(string key, Dictionary dict, object locker) + { + if (dict.ContainsKey(key)) + { + lock (locker) + { + if (dict.ContainsKey(key)) + { + dict.Remove(key); + } + } + } + } + #endregion //Private Static Methods +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/Config.cs b/Intrinio.Realtime/Equities/Config.cs new file mode 100644 index 0000000..1b0a031 --- /dev/null +++ b/Intrinio.Realtime/Equities/Config.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; + +namespace Intrinio.Realtime.Equities; + +using System; +using Serilog; +using System.IO; +using Microsoft.Extensions.Configuration; + +public class Config +{ + public string ApiKey { get; set; } + + public Provider Provider { get; set; } + + public string IPAddress { get; set; } + + public string[] Symbols { get; set; } + + public bool TradesOnly { get; set; } + + public int NumThreads { get; set; } + + public int BufferSize { get; set; } + + public int OverflowBufferSize { get; set; } + + /// + /// The configuration for The Equities Websocket Client. + /// + public Config() + { + ApiKey = String.Empty; + Provider = Provider.NONE; + IPAddress = String.Empty; + Symbols = Array.Empty(); + TradesOnly = false; + NumThreads = 2; + BufferSize = 2048; + OverflowBufferSize = 2048; + } + + public void Validate() + { + if (String.IsNullOrWhiteSpace(ApiKey)) + { + throw new ArgumentException("You must provide a valid API key"); + } + + if (Provider == Provider.NONE) + { + throw new ArgumentException("You must specify a valid 'provider'"); + } + + if ((Provider == Provider.MANUAL) && (String.IsNullOrWhiteSpace(IPAddress))) + { + throw new ArgumentException("You must specify an IP address for manual configuration"); + } + + if (NumThreads <= 0) + { + throw new ArgumentException("You must specify a valid 'NumThreads'"); + } + + if (BufferSize < 2048) + { + throw new ArgumentException("'BufferSize' must be greater than or equal to 2048."); + } + + if (OverflowBufferSize < 2048) + { + throw new ArgumentException("'OverflowBufferSize' must be greater than or equal to 2048."); + } + } + + public static Config LoadConfig() + { + Log.Information("Loading application configuration"); + var rawConfig = new ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("config.json").Build(); + Log.Logger = new LoggerConfiguration().ReadFrom.Configuration(rawConfig).CreateLogger(); + Config config = new Config(); + + foreach (KeyValuePair kvp in rawConfig.AsEnumerable()) + { + Log.Debug("Key: {0}, Value:{1}", kvp.Key, kvp.Value); + } + + rawConfig.Bind("Config", config); + config.Validate(); + return config; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/EquitiesWebSocketClient.cs b/Intrinio.Realtime/Equities/EquitiesWebSocketClient.cs new file mode 100644 index 0000000..9617c98 --- /dev/null +++ b/Intrinio.Realtime/Equities/EquitiesWebSocketClient.cs @@ -0,0 +1,367 @@ +using System; +using System.Text; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Intrinio.Realtime.Equities; + +public class EquitiesWebSocketClient : WebSocketClient, IEquitiesWebSocketClient +{ + #region Data Members + + private const string LobbyName = "lobby"; + private bool _useOnTrade; + private bool _useOnQuote; + private Action _onTrade; + + /// + /// The callback for when a trade event occurs. + /// + public Action OnTrade + { + set + { + _useOnTrade = !ReferenceEquals(value, null); + _onTrade = value; + } + } + + private Action _onQuote; + + /// + /// The callback for when a quote event occurs. + /// + public Action OnQuote + { + set + { + _useOnQuote = !ReferenceEquals(value, null); + _onQuote = value; + } + } + + private readonly Config _config; + private UInt64 _dataTradeCount = 0UL; + private UInt64 _dataQuoteCount = 0UL; + public UInt64 TradeCount { get { return Interlocked.Read(ref _dataTradeCount); } } + public UInt64 QuoteCount { get { return Interlocked.Read(ref _dataQuoteCount); } } + + private readonly string _logPrefix; + private const string MessageVersionHeaderKey = "UseNewEquitiesFormat"; + private const string MessageVersionHeaderValue = "v2"; + private const uint MaxMessageSize = 64u; + private const string ChannelFormat = "{0}|TradesOnly|{1}"; + #endregion //Data Members + + #region Constuctors + /// + /// Create a new Equities websocket client. + /// + /// + /// + /// + public EquitiesWebSocketClient(Action onTrade, Action onQuote, Config config) + : base(Convert.ToUInt32(config.NumThreads), Convert.ToUInt32(config.BufferSize), Convert.ToUInt32(config.OverflowBufferSize), MaxMessageSize) + { + OnTrade = onTrade; + OnQuote = onQuote; + _config = config; + + if (ReferenceEquals(null, _config)) + throw new ArgumentException("Config may not be null."); + _config.Validate(); + _logPrefix = String.Format("{0}: ", _config?.Provider.ToString()); + } + + /// + /// Create a new Equities websocket client. + /// + /// + public EquitiesWebSocketClient(Action onTrade) : this(onTrade, null, Config.LoadConfig()) + { + } + + /// + /// Create a new Equities websocket client. + /// + /// + public EquitiesWebSocketClient(Action onQuote) : this(null, onQuote, Config.LoadConfig()) + { + } + + /// + /// Create a new Equities websocket client. + /// + /// + /// + public EquitiesWebSocketClient(Action onTrade, Action onQuote) : this(onTrade, onQuote, Config.LoadConfig()) + { + } + #endregion //Constructors + + #region Public Methods + public async Task Join() + { + while (!IsReady()) + await Task.Delay(1000); + HashSet channelsToAdd = _config.Symbols.Select(s => GetChannel(s, _config.TradesOnly)).ToHashSet(); + channelsToAdd.ExceptWith(Channels); + foreach (string channel in channelsToAdd) + await JoinImpl(channel); + } + + public async Task Join(string symbol, bool? tradesOnly) + { + bool t = tradesOnly.HasValue ? tradesOnly.Value || _config.TradesOnly : false || _config.TradesOnly; + while (!IsReady()) + await Task.Delay(1000); + if (!Channels.Contains(GetChannel(symbol, t))) + await JoinImpl(GetChannel(symbol, t)); + } + + public async Task JoinLobby(bool? tradesOnly) + { + await Join(LobbyName, tradesOnly); + } + + public async Task Join(string[] symbols, bool? tradesOnly) + { + bool t = tradesOnly.HasValue ? tradesOnly.Value || _config.TradesOnly : false || _config.TradesOnly; + while (!IsReady()) + await Task.Delay(1000); + HashSet symbolsToAdd = symbols.Select(s => GetChannel(s, t)).ToHashSet(); + symbolsToAdd.ExceptWith(Channels); + foreach (string channel in symbolsToAdd) + await JoinImpl(channel); + } + + public async Task Leave() + { + await LeaveImpl(); + } + + public async Task Leave(string symbol) + { + foreach (string channel in Channels.Where(c => symbol == GetSymbolFromChannel(c))) + await LeaveImpl(channel); + } + + public async Task LeaveLobby() + { + await Leave(LobbyName); + } + + public async Task Leave(string[] symbols) + { + HashSet hashSymbols = new HashSet(symbols); + foreach (string channel in Channels.Where(c => hashSymbols.Contains(GetSymbolFromChannel(c)))) + await LeaveImpl(channel); + } + #endregion //Public Methods + + #region Private Methods + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private string GetChannel(string symbol, bool tradesOnly) + { + return String.Format(ChannelFormat, symbol, _config.TradesOnly.ToString()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private string GetSymbolFromChannel(string channel) + { + return channel.Split('|')[0]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool GetTradesOnlyFromChannel(string channel) + { + return Boolean.Parse(channel.Split('|')[2]); + } + + protected override string GetLogPrefix() + { + return _logPrefix; + } + + protected override string GetAuthUrl() + { + switch (_config.Provider) + { + case Provider.REALTIME: + return $"https://realtime-mx.intrinio.com/auth?api_key={_config.ApiKey}"; + break; + case Provider.DELAYED_SIP: + return $"https://realtime-delayed-sip.intrinio.com/auth?api_key={_config.ApiKey}"; + break; + case Provider.NASDAQ_BASIC: + return $"https://realtime-nasdaq-basic.intrinio.com/auth?api_key={_config.ApiKey}"; + break; + case Provider.MANUAL: + return $"http://{_config.IPAddress}/auth?api_key={_config.ApiKey}"; + break; + default: + throw new ArgumentException("Provider not specified!"); + break; + } + } + + protected override string GetWebSocketUrl(string token) + { + switch (_config.Provider) + { + case Provider.REALTIME: + return $"wss://realtime-mx.intrinio.com/socket/websocket?vsn=1.0.0&token={token}"; + break; + case Provider.DELAYED_SIP: + return $"wss://realtime-delayed-sip.intrinio.com/socket/websocket?vsn=1.0.0&token={token}"; + break; + case Provider.NASDAQ_BASIC: + return $"wss://realtime-nasdaq-basic.intrinio.com/socket/websocket?vsn=1.0.0&token={token}"; + break; + case Provider.MANUAL: + return $"ws://{_config.IPAddress}/socket/websocket?vsn=1.0.0&token={token}"; + break; + default: + throw new ArgumentException("Provider not specified!"); + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected override int GetNextChunkLength(ReadOnlySpan bytes) + { + return Convert.ToInt32(bytes[1]); + } + + protected override List> GetCustomSocketHeaders() + { + List> headers = new List>(); + headers.Add(new KeyValuePair(MessageVersionHeaderKey, MessageVersionHeaderValue)); + return headers; + } + + private Trade ParseTrade(ReadOnlySpan bytes) + { + int symbolLength = Convert.ToInt32(bytes[2]); + int conditionLength = Convert.ToInt32(bytes[26 + symbolLength]); + string symbol = Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)); + double price = Convert.ToDouble(BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4))); + UInt32 size = BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)); + DateTime timestamp = DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)); + SubProvider subProvider = (SubProvider)((int)bytes[3 + symbolLength]); + char marketCenter = BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)); + string condition = conditionLength > 0 ? Encoding.ASCII.GetString(bytes.Slice(27 + symbolLength, conditionLength)) : String.Empty; + UInt64 totalVolume = Convert.ToUInt64(BitConverter.ToUInt32(bytes.Slice(22 + symbolLength, 4))); + + return new Trade(symbol, price, size, totalVolume, timestamp, subProvider, marketCenter, condition); + } + + private Quote ParseQuote(ReadOnlySpan bytes) + { + int symbolLength = Convert.ToInt32(bytes[2]); + int conditionLength = Convert.ToInt32(bytes[22 + symbolLength]); + QuoteType type = (QuoteType)((int)(bytes[0])); + string symbol = Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)); + double price = Convert.ToDouble(BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4))); + UInt32 size = BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)); + DateTime timestamp = DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)); + SubProvider subProvider = (SubProvider)((int)(bytes[3 + symbolLength])); + char marketCenter = BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)); + string condition = (conditionLength > 0) ? Encoding.ASCII.GetString(bytes.Slice(23 + symbolLength, conditionLength)) : String.Empty; + + return new Quote(type, symbol, price, size, timestamp, subProvider, marketCenter, condition); + } + + protected override void HandleMessage(ReadOnlySpan bytes) + { + MessageType msgType = (MessageType)Convert.ToInt32(bytes[0]); + switch (msgType) + { + case MessageType.Trade: + { + if (_useOnTrade) + { + Trade trade = ParseTrade(bytes); + Interlocked.Increment(ref _dataTradeCount); + try { _onTrade.Invoke(trade); } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while invoking user supplied OnTrade: {0}; {1}", new object[]{e.Message, e.StackTrace}); + } + } + break; + } + case MessageType.Ask: + case MessageType.Bid: + { + if (_useOnQuote) + { + Quote quote = ParseQuote(bytes); + Interlocked.Increment(ref _dataQuoteCount); + try { _onQuote.Invoke(quote); } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while invoking user supplied OnQuote: {0}; {1}", new object[]{e.Message, e.StackTrace}); + } + } + break; + } + default: + LogMessage(LogLevel.WARNING, "Invalid MessageType: {0}", new object[] {Convert.ToInt32(bytes[0])}); + break; + } + } + + protected override byte[] MakeJoinMessage(string channel) + { + string symbol = GetSymbolFromChannel(channel); + bool tradesOnly = GetTradesOnlyFromChannel(channel); + switch (symbol) + { + case LobbyName: + { + byte[] message = new byte[11]; //1 + 1 + 9 + message[0] = Convert.ToByte(74); //type: join (74uy) or leave (76uy) + message[1] = tradesOnly ? Convert.ToByte(1) : Convert.ToByte(0); + Encoding.ASCII.GetBytes("$FIREHOSE").CopyTo(message, 2); + return message; + } + default: + { + byte[] message = new byte[2 + symbol.Length]; //1 + 1 + symbol.Length + message[0] = Convert.ToByte(74); //type: join (74uy) or leave (76uy) + message[1] = tradesOnly ? Convert.ToByte(1) : Convert.ToByte(0); + Encoding.ASCII.GetBytes(symbol).CopyTo(message, 2); + return message; + } + } + } + + protected override byte[] MakeLeaveMessage(string channel) + { + string symbol = GetSymbolFromChannel(channel); + bool tradesOnly = GetTradesOnlyFromChannel(channel); + switch (symbol) + { + case LobbyName: + { + byte[] message = new byte[10]; // 1 (type = join) + 9 (symbol = $FIREHOSE) + message[0] = Convert.ToByte(76); //type: join (74uy) or leave (76uy) + Encoding.ASCII.GetBytes("$FIREHOSE").CopyTo(message, 1); + return message; + } + default: + { + byte[] message = new byte[1 + symbol.Length]; //1 + symbol.Length + message[0] = Convert.ToByte(76); //type: join (74uy) or leave (76uy) + Encoding.ASCII.GetBytes(symbol).CopyTo(message, 1); + return message; + } + } + } + #endregion //Private Methods +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/IEquitiesWebSocketClient.cs b/Intrinio.Realtime/Equities/IEquitiesWebSocketClient.cs new file mode 100644 index 0000000..7d9cb2f --- /dev/null +++ b/Intrinio.Realtime/Equities/IEquitiesWebSocketClient.cs @@ -0,0 +1,24 @@ +using System; +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Equities; + +public interface IEquitiesWebSocketClient +{ + public Action OnTrade { set; } + public Action OnQuote { set; } + public Task Join(); + public Task Join(string channel, bool? tradesOnly); + public Task JoinLobby(bool? tradesOnly); + public Task Join(string[] channels, bool? tradesOnly); + public Task Leave(); + public Task Leave(string channel); + public Task LeaveLobby(); + public Task Leave(string[] channels); + public Task Stop(); + public Task Start(); + public ClientStats GetStats(); + public UInt64 TradeCount { get; } + public UInt64 QuoteCount { get; } + public void LogMessage(LogLevel logLevel, string messageTemplate, params object[] propertyValues); +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/MessageType.cs b/Intrinio.Realtime/Equities/MessageType.cs new file mode 100644 index 0000000..e331724 --- /dev/null +++ b/Intrinio.Realtime/Equities/MessageType.cs @@ -0,0 +1,8 @@ +namespace Intrinio.Realtime.Equities; + +public enum MessageType +{ + Trade = 0, + Ask = 1, + Bid = 2 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/Provider.cs b/Intrinio.Realtime/Equities/Provider.cs new file mode 100644 index 0000000..428f414 --- /dev/null +++ b/Intrinio.Realtime/Equities/Provider.cs @@ -0,0 +1,10 @@ +namespace Intrinio.Realtime.Equities; + +public enum Provider +{ + NONE = 0, + REALTIME = 1, + MANUAL = 2, + DELAYED_SIP = 3, + NASDAQ_BASIC = 4 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/Quote.cs b/Intrinio.Realtime/Equities/Quote.cs new file mode 100644 index 0000000..5a86f2f --- /dev/null +++ b/Intrinio.Realtime/Equities/Quote.cs @@ -0,0 +1,38 @@ +namespace Intrinio.Realtime.Equities; + +using System; + +public struct Quote +{ + public readonly QuoteType Type; + public readonly string Symbol; + public readonly double Price; + public readonly UInt32 Size; + public readonly DateTime Timestamp; + public readonly SubProvider SubProvider; + public readonly char MarketCenter; + public readonly string Condition; + + /// Type: the type of the quote (can be 'ask' or 'bid') + /// Symbol: the 'ticker' symbol + /// Price: the dollar price of the quote + /// Size: the number of shares that were offered as part of the quote + /// Timestamp: the time that the quote was placed (a unix timestamp representing the number of milliseconds (or better) since the unix epoch) + /// SubProvider: the specific provider this trade came from under the parent provider grouping. + public Quote(QuoteType type, string symbol, double price, UInt32 size, DateTime timestamp, SubProvider subProvider, char marketCenter, string condition) + { + Type = type; + Symbol = symbol; + Price = price; + Size = size; + Timestamp = timestamp; + SubProvider = subProvider; + MarketCenter = marketCenter; + Condition = condition; + } + + public override string ToString() + { + return $"Quote (Type: {Type}, Symbol: {Symbol}, Price: {Price}, Size: {Size}, Timestamp: {Timestamp}, SubProvider: {SubProvider}, MarketCenter: {MarketCenter}, Condition: {Condition})"; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/QuoteCandleStick.cs b/Intrinio.Realtime/Equities/QuoteCandleStick.cs new file mode 100644 index 0000000..856e71a --- /dev/null +++ b/Intrinio.Realtime/Equities/QuoteCandleStick.cs @@ -0,0 +1,129 @@ +namespace Intrinio.Realtime.Equities; + +using System; + +public class QuoteCandleStick : CandleStick, IEquatable, IComparable, IComparable +{ + private readonly string _symbol; + private readonly QuoteType _quoteType; + + public string Symbol + { + get { return _symbol; } + } + public QuoteType QuoteType + { + get { return _quoteType; } + } + + public QuoteCandleStick(string symbol, UInt32 volume, double price, QuoteType quoteType, double openTimestamp, double closeTimestamp, IntervalType interval, double quoteTime) + : base(volume, price, openTimestamp, closeTimestamp, interval, quoteTime) + { + _symbol = symbol; + _quoteType = quoteType; + } + + public QuoteCandleStick(string symbol, UInt32 volume, double high, double low, double closePrice, double openPrice, QuoteType quoteType, double openTimestamp, double closeTimestamp, double firstTimestamp, double lastTimestamp, bool complete, double average, double change, IntervalType interval) + : base(volume, high, low, closePrice, openPrice, openTimestamp, closeTimestamp, firstTimestamp, lastTimestamp, complete, average, change, interval) + { + _symbol = symbol; + _quoteType = quoteType; + } + + public override bool Equals(object other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (other is QuoteCandleStick) + && (Symbol.Equals(((QuoteCandleStick)other).Symbol)) + && (Interval.Equals(((QuoteCandleStick)other).Interval)) + && (QuoteType.Equals(((QuoteCandleStick)other).QuoteType)) + && (OpenTimestamp.Equals(((QuoteCandleStick)other).OpenTimestamp)) + ); + } + + public override int GetHashCode() + { + return Symbol.GetHashCode() ^ Interval.GetHashCode() ^ OpenTimestamp.GetHashCode() ^ QuoteType.GetHashCode(); + } + + public bool Equals(QuoteCandleStick other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (Symbol.Equals(other.Symbol)) + && (Interval.Equals(other.Interval)) + && (QuoteType.Equals(other.QuoteType)) + && (OpenTimestamp.Equals(other.OpenTimestamp)) + ); + } + + public int CompareTo(object other) + { + return Equals(other) switch + { + true => 0, + false => ReferenceEquals(other, null) switch + { + true => 1, + false => (other is QuoteCandleStick) switch + { + true => Symbol.CompareTo(((QuoteCandleStick)other).Symbol) switch + { + < 0 => -1, + > 0 => 1, + 0 => Interval.CompareTo(((QuoteCandleStick)other).Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => QuoteType.CompareTo(((QuoteCandleStick)other).QuoteType) switch + { + < 0 => -1, + > 0 => 1, + 0 => OpenTimestamp.CompareTo(((QuoteCandleStick)other).OpenTimestamp) + } + } + }, + false => 1 + } + } + }; + } + + public int CompareTo(QuoteCandleStick other) + { + return Equals(other) switch + { + true => 0, + false => Object.ReferenceEquals(other, null) switch + { + true => 1, + false => Symbol.CompareTo(other.Symbol) switch + { + < 0 => -1, + > 0 => 1, + 0 => Interval.CompareTo(other.Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => QuoteType.CompareTo(other.QuoteType) switch + { + < 0 => -1, + > 0 => 1, + 0 => OpenTimestamp.CompareTo(other.OpenTimestamp) + } + } + } + } + }; + } + + public override string ToString() + { + return $"QuoteCandleStick (Symbol: {Symbol}, QuoteType: {QuoteType.ToString()}, High: {High.ToString("f3")}, Low: {Low.ToString("f3")}, Close: {Close.ToString("f3")}, Open: {Open.ToString("f3")}, OpenTimestamp: {OpenTimestamp.ToString("f6")}, CloseTimestamp: {CloseTimestamp.ToString("f6")}, Change: {Change.ToString("f6")}, Complete: {Complete.ToString()})"; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/QuoteType.cs b/Intrinio.Realtime/Equities/QuoteType.cs new file mode 100644 index 0000000..2697287 --- /dev/null +++ b/Intrinio.Realtime/Equities/QuoteType.cs @@ -0,0 +1,7 @@ +namespace Intrinio.Realtime.Equities; + +public enum QuoteType +{ + Ask = 1, + Bid = 2 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/ReplayClient.cs b/Intrinio.Realtime/Equities/ReplayClient.cs new file mode 100644 index 0000000..a255b29 --- /dev/null +++ b/Intrinio.Realtime/Equities/ReplayClient.cs @@ -0,0 +1,693 @@ +using System.Linq; + +namespace Intrinio.Realtime.Equities; + +using Intrinio.SDK.Model; +using System; +using System.IO; +using System.Net.Http; +using System.Text; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +public class ReplayClient : IEquitiesWebSocketClient +{ + #region Data Members + private const string LobbyName = "lobby"; + public Action OnTrade { get; set; } + public Action OnQuote { get; set; } + private readonly Config _config; + private readonly DateTime _date; + private readonly bool _withSimulatedDelay; + private readonly bool _deleteFileWhenDone; + private readonly bool _writeToCsv; + private readonly string _csvFilePath; + private ulong _dataMsgCount; + private ulong _dataEventCount; + private ulong _dataTradeCount; + private ulong _dataQuoteCount; + private ulong _textMsgCount; + private readonly HashSet _channels; + private readonly CancellationTokenSource _ctSource; + private readonly ConcurrentQueue _data; + private bool _useOnTrade { get {return !(ReferenceEquals(OnTrade, null));} } + private bool _useOnQuote { get {return !(ReferenceEquals(OnQuote, null));} } + + private readonly string _logPrefix; + private readonly object _csvLock; + private readonly Thread[] _threads; + private readonly Thread _replayThread; + public UInt64 TradeCount { get { return Interlocked.Read(ref _dataTradeCount); } } + public UInt64 QuoteCount { get { return Interlocked.Read(ref _dataQuoteCount); } } + #endregion //Data Members + + #region Constructors + public ReplayClient(Action onTrade, Action onQuote, Config config, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) + { + this.OnTrade = onTrade; + this.OnQuote = onQuote; + this._config = config; + this._date = date; + this._withSimulatedDelay = withSimulatedDelay; + this._deleteFileWhenDone = deleteFileWhenDone; + this._writeToCsv = writeToCsv; + this._csvFilePath = csvFilePath; + + _dataMsgCount = 0UL; + _dataEventCount = 0UL; + _dataTradeCount = 0UL; + _dataQuoteCount = 0UL; + _textMsgCount = 0UL; + _channels = new HashSet(); + _ctSource = new CancellationTokenSource(); + _data = new ConcurrentQueue(); + + _logPrefix = _logPrefix = String.Format("{0}: ", config.Provider.ToString()); + _csvLock = new Object(); + _threads = new Thread[config.NumThreads]; + for (int i = 0; i < _threads.Length; i++) + _threads[i] = new Thread(ThreadFn); + _replayThread = new Thread(ReplayThreadFn); + + config.Validate(); + } + + public ReplayClient(Action onTrade, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) : this(onTrade, null, Config.LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) + { + + } + + public ReplayClient(Action onQuote, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) : this(null, onQuote, Config.LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) + { + + } + + public ReplayClient(Action onTrade, Action onQuote, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) : this(onTrade, onQuote, Config.LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) + { + + } + #endregion //Constructors + + #region Public Methods + + public Task Join() + { + HashSet symbolsToAdd = _config.Symbols.Select(s => new Channel(s, _config.TradesOnly)).ToHashSet(); + symbolsToAdd.ExceptWith(_channels); + + foreach (Channel channel in symbolsToAdd) + Join(channel.ticker, channel.tradesOnly); + + return Task.CompletedTask; + } + + public Task Join(string symbol, bool? tradesOnly) + { + bool t = tradesOnly.HasValue + ? tradesOnly.Value || _config.TradesOnly + : _config.TradesOnly; + if (!_channels.Contains(new Channel(symbol, t))) + Join(symbol, t); + + return Task.CompletedTask; + } + + public async Task JoinLobby(bool? tradesOnly) + { + await Join(LobbyName, tradesOnly); + } + + public Task Join(string[] symbols, bool? tradesOnly) + { + bool t = tradesOnly.HasValue + ? tradesOnly.Value || _config.TradesOnly + : _config.TradesOnly; + HashSet symbolsToAdd = symbols.Select(s => new Channel(s, t)).ToHashSet(); + symbolsToAdd.ExceptWith(_channels); + foreach (Channel channel in symbolsToAdd) + Join(channel.ticker, channel.tradesOnly); + return Task.CompletedTask; + } + + public Task Leave() + { + foreach (Channel channel in _channels) + Leave(channel.ticker, channel.tradesOnly); + return Task.CompletedTask; + } + + public Task Leave(string symbol) + { + IEnumerable matchingChannels = _channels.Where(c => c.ticker == symbol); + foreach (Channel channel in matchingChannels) + Leave(channel.ticker, channel.tradesOnly); + return Task.CompletedTask; + } + + public async Task LeaveLobby() + { + await Leave(LobbyName); + } + + public Task Leave(string[] symbols) + { + HashSet _symbols = new HashSet(symbols); + IEnumerable matchingChannels = _channels.Where(c => _symbols.Contains(c.ticker)); + foreach (Channel channel in matchingChannels) + Leave(channel.ticker, channel.tradesOnly); + return Task.CompletedTask; + } + + public Task Start() + { + foreach (Thread thread in _threads) + thread.Start(); + if (_writeToCsv) + WriteHeaderRow(); + _replayThread.Start(); + + return Task.CompletedTask; + } + + public Task Stop() + { + foreach (Channel channel in _channels) + Leave(channel.ticker, channel.tradesOnly); + + _ctSource.Cancel(); + LogMessage(LogLevel.INFORMATION, "Websocket - Closing..."); + + foreach (Thread thread in _threads) + thread.Join(); + + _replayThread.Join(); + + LogMessage(LogLevel.INFORMATION, "Stopped"); + return Task.CompletedTask; + } + + public ClientStats GetStats() + { + return new ClientStats( + Interlocked.Read(ref _dataMsgCount), + Interlocked.Read(ref _textMsgCount), + _data.Count, + Interlocked.Read(ref _dataEventCount), + Int32.MaxValue, + 0, + Int32.MaxValue, + 0, + 0 + ); + } + + [Serilog.Core.MessageTemplateFormatMethod("messageTemplate")] + public void LogMessage(LogLevel logLevel, string messageTemplate, params object[] propertyValues) + { + switch (logLevel) + { + case LogLevel.DEBUG: + Serilog.Log.Debug(_logPrefix + messageTemplate, propertyValues); + break; + case LogLevel.INFORMATION: + Serilog.Log.Information(_logPrefix + messageTemplate, propertyValues); + break; + case LogLevel.WARNING: + Serilog.Log.Warning(_logPrefix + messageTemplate, propertyValues); + break; + case LogLevel.ERROR: + Serilog.Log.Error(_logPrefix + messageTemplate, propertyValues); + break; + default: + throw new ArgumentException("LogLevel not specified!"); + break; + } + } + #endregion //Public Methods + + #region Private Methods + private DateTime ParseTimeReceived(ReadOnlySpan bytes) + { + return DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes) / 100UL)); + } + + private Trade ParseTrade(ReadOnlySpan bytes) + { + int symbolLength = Convert.ToInt32(bytes[2]); + int conditionLength = Convert.ToInt32(bytes[26 + symbolLength]); + Trade trade = new Trade( + Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)), + Convert.ToDouble(BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4))), + BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)), + BitConverter.ToUInt32(bytes.Slice(22 + symbolLength, 4)), + DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)), + (SubProvider)Convert.ToInt32(bytes[3 + symbolLength]), + BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)), + conditionLength > 0 ? Encoding.ASCII.GetString(bytes.Slice(27 + symbolLength, conditionLength)) : String.Empty + ); + + return trade; + } + + private Quote ParseQuote(ReadOnlySpan bytes) + { + int symbolLength = Convert.ToInt32(bytes[2]); + int conditionLength = Convert.ToInt32(bytes[22 + symbolLength]); + + Quote quote = new Quote( + (QuoteType)(Convert.ToInt32(bytes[0])), + Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)), + (Convert.ToDouble(BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4)))), + BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)), + DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)), + (SubProvider)(Convert.ToInt32(bytes[3 + symbolLength])), + BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)), + conditionLength > 0 ? Encoding.ASCII.GetString(bytes.Slice(23 + symbolLength, conditionLength)) : String.Empty + ); + + return quote; + } + + private void WriteRowToOpenCsvWithoutLock(IEnumerable row) + { + bool first = true; + using (FileStream fs = new FileStream(_csvFilePath, FileMode.Append)) + using (TextWriter tw = new StreamWriter(fs)) + { + foreach (string s in row) + { + if (!first) + tw.Write(","); + else + first = false; + tw.Write($"\"{s}\""); + } + + tw.WriteLine(); + } + } + + private void WriteRowToOpenCsvWithLock(IEnumerable row) + { + lock (_csvLock) + { + WriteRowToOpenCsvWithoutLock(row); + } + } + + private string DoubleRoundSecRule612(double value) + { + if (value >= 1.0D) + return value.ToString("0.00"); + + return value.ToString("0.0000"); + } + + private IEnumerable MapTradeToRow(Trade trade) + { + yield return MessageType.Trade.ToString(); + yield return trade.Symbol; + yield return DoubleRoundSecRule612(trade.Price); + yield return trade.Size.ToString(); + yield return trade.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffK"); + yield return trade.SubProvider.ToString(); + yield return trade.MarketCenter.ToString(); + yield return trade.Condition; + yield return trade.TotalVolume.ToString(); + } + + private void WriteTradeToCsv(Trade trade) + { + WriteRowToOpenCsvWithLock(MapTradeToRow(trade)); + } + + private IEnumerable MapQuoteToRow(Quote quote) + { + yield return quote.Type.ToString(); + yield return quote.Symbol; + yield return DoubleRoundSecRule612(quote.Price); + yield return quote.Size.ToString(); + yield return quote.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffK"); + yield return quote.SubProvider.ToString(); + yield return quote.MarketCenter.ToString(); + yield return quote.Condition; + } + + private void WriteQuoteToCsv(Quote quote) + { + WriteRowToOpenCsvWithLock(MapQuoteToRow(quote)); + } + + private void WriteHeaderRow() + { + WriteRowToOpenCsvWithLock(new string[]{"Type", "Symbol", "Price", "Size", "Timestamp", "SubProvider", "MarketCenter", "Condition", "TotalVolume"}); + } + + private void ThreadFn() + { + CancellationToken ct = _ctSource.Token; + while (!ct.IsCancellationRequested) + { + try + { + if (_data.TryDequeue(out Tick datum)) + { + if (datum.IsTrade()) + { + if (_useOnTrade) + { + Interlocked.Increment(ref _dataTradeCount); + OnTrade.Invoke(datum.Trade); + } + } + else + { + if (_useOnQuote) + { + Interlocked.Increment(ref _dataQuoteCount); + OnQuote.Invoke(datum.Quote); + } + } + } + else + Thread.Sleep(1); + } + catch (OperationCanceledException) + { + } + catch (Exception exn) + { + LogMessage(LogLevel.ERROR, "Error parsing message: {0}; {1}", exn.Message, exn.StackTrace); + } + } + } + + /// + /// The results of this should be streamed and not ToList-ed. + /// + /// + /// + /// + private IEnumerable ReplayTickFileWithoutDelay(string fullFilePath, int byteBufferSize, CancellationToken ct) + { + if (File.Exists(fullFilePath)) + { + using (FileStream fRead = new FileStream(fullFilePath, FileMode.Open, FileAccess.Read, FileShare.None)) + { + if (fRead.CanRead) + { + int readResult = fRead.ReadByte(); //This is message type + + while (readResult != -1) + { + if (!ct.IsCancellationRequested) + { + byte[] eventBuffer = new byte[byteBufferSize]; + byte[] timeReceivedBuffer = new byte[8]; + ReadOnlySpan eventSpanBuffer = new ReadOnlySpan(eventBuffer); + ReadOnlySpan timeReceivedSpanBuffer = new ReadOnlySpan(timeReceivedBuffer); + eventBuffer[0] = (byte)readResult; //This is message type + eventBuffer[1] = (byte)(fRead.ReadByte()); //This is message length, including this and the previous byte. + int bytesRead = fRead.Read(eventBuffer, 2, (System.Convert.ToInt32(eventBuffer[1]) - 2)); //read the rest of the message + int timeBytesRead = fRead.Read(timeReceivedBuffer, 0, 8); //get the time received + DateTime timeReceived = ParseTimeReceived(timeReceivedSpanBuffer); + + switch ((MessageType)(Convert.ToInt32(eventBuffer[0]))) + { + case MessageType.Trade: + Trade trade = ParseTrade(eventSpanBuffer); + if (_channels.Contains(new Channel(LobbyName, true)) + || _channels.Contains(new Channel(LobbyName, false)) + || _channels.Contains(new Channel(trade.Symbol, true)) + || _channels.Contains(new Channel(trade.Symbol, false))) + { + if (_writeToCsv) + WriteTradeToCsv(trade); + yield return new Tick(timeReceived, trade, null); + } + break; + case MessageType.Ask: + case MessageType.Bid: + Quote quote = ParseQuote(eventSpanBuffer); + if (_channels.Contains (new Channel(LobbyName, false)) || _channels.Contains (new Channel(quote.Symbol, false))) + { + if (_writeToCsv) + WriteQuoteToCsv(quote); + yield return new Tick(timeReceived, null, quote); + } + break; + default: + LogMessage(LogLevel.ERROR, "Invalid MessageType: {0}", eventBuffer[0]); + break; + } + + //Set up the next iteration + readResult = fRead.ReadByte(); + } + else + readResult = -1; + } + } + else + throw new FileLoadException("Unable to read replay file."); + } + } + else + { + yield break; + } + } + + /// + /// The results of this should be streamed and not ToList-ed. + /// + /// + /// + /// returns + private IEnumerable ReplayTickFileWithDelay(string fullFilePath, int byteBufferSize, CancellationToken ct) + { + long start = DateTime.UtcNow.Ticks; + long offset = 0L; + foreach (Tick tick in ReplayTickFileWithoutDelay(fullFilePath, byteBufferSize, ct)) + { + if (offset == 0L) + offset = start - tick.TimeReceived().Ticks; + + if (!ct.IsCancellationRequested) + { + SpinWait.SpinUntil(() => (tick.TimeReceived().Ticks + offset) <= DateTime.UtcNow.Ticks); + yield return tick; + } + } + } + + private string MapSubProviderToApiValue(SubProvider subProvider) + { + switch (subProvider) + { + case SubProvider.IEX: return "iex"; + case SubProvider.UTP: return "utp_delayed"; + case SubProvider.CTA_A: return "cta_a_delayed"; + case SubProvider.CTA_B: return "cta_b_delayed"; + case SubProvider.OTC: return "otc_delayed"; + case SubProvider.NASDAQ_BASIC: return "nasdaq_basic"; + default: return "iex"; + } + } + + private SubProvider[] MapProviderToSubProviders(Intrinio.Realtime.Equities.Provider provider) + { + switch (provider) + { + case Provider.NONE: return Array.Empty(); + case Provider.MANUAL: return Array.Empty(); + case Provider.REALTIME: return new SubProvider[]{SubProvider.IEX}; + case Provider.DELAYED_SIP: return new SubProvider[]{SubProvider.UTP, SubProvider.CTA_A, SubProvider.CTA_B, SubProvider.OTC}; + case Provider.NASDAQ_BASIC: return new SubProvider[]{SubProvider.NASDAQ_BASIC}; + default: return new SubProvider[0]; + } + } + + private string FetchReplayFile(SubProvider subProvider) + { + Intrinio.SDK.Api.SecurityApi api = new Intrinio.SDK.Api.SecurityApi(); + + if (!api.Configuration.ApiKey.ContainsKey("api_key")) + api.Configuration.ApiKey.Add("api_key", _config.ApiKey); + + try + { + SecurityReplayFileResult result = api.GetSecurityReplayFile(MapSubProviderToApiValue(subProvider), _date); + string decodedUrl = result.Url.Replace(@"\u0026", "&"); + string tempDir = System.IO.Path.GetTempPath(); + string fileName = Path.Combine(tempDir, result.Name); + + using (FileStream outputFile = new FileStream(fileName,System.IO.FileMode.Create)) + using (HttpClient httpClient = new HttpClient()) + { + httpClient.Timeout = TimeSpan.FromHours(1); + httpClient.BaseAddress = new Uri(decodedUrl); + using (HttpResponseMessage response = httpClient.GetAsync(decodedUrl, HttpCompletionOption.ResponseHeadersRead).Result) + using (Stream streamToReadFrom = response.Content.ReadAsStreamAsync().Result) + { + streamToReadFrom.CopyTo(outputFile); + } + } + + return fileName; + } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while fetching {0} file: {1}", subProvider.ToString(), e.Message); + return null; + } + } + + private void FillNextTicks(IEnumerator[] enumerators, Tick[] nextTicks) + { + for (int i = 0; i < nextTicks.Length; i++) + if (nextTicks[i] == null && enumerators[i].MoveNext()) + nextTicks[i] = enumerators[i].Current; + } + + private Tick PullNextTick(Tick[] nextTicks) + { + int pullIndex = 0; + DateTime t = DateTime.MaxValue; + for (int i = 0; i < nextTicks.Length; i++) + { + if (nextTicks[i] != null && nextTicks[i].TimeReceived() < t) + { + pullIndex = i; + t = nextTicks[i].TimeReceived(); + } + } + + Tick pulledTick = nextTicks[pullIndex]; + nextTicks[pullIndex] = null; + return pulledTick; + } + + private bool HasAnyValue(Tick[] nextTicks) + { + bool hasValue = false; + + for (int i = 0; i < nextTicks.Length; i++) + if (nextTicks[i] != null) + hasValue = true; + + return hasValue; + } + + private IEnumerable ReplayFileGroupWithoutDelay(IEnumerable[] tickGroup, CancellationToken ct) + { + Tick[] nextTicks = new Tick[tickGroup.Length]; + IEnumerator[] enumerators = new IEnumerator[tickGroup.Length]; + for (int i = 0; i < tickGroup.Length; i++) + { + enumerators[i] = tickGroup[i].GetEnumerator(); + } + + FillNextTicks(enumerators, nextTicks); + while (HasAnyValue(nextTicks)) + { + Tick nextTick = PullNextTick(nextTicks); + if (nextTick != null) + yield return nextTick; + + FillNextTicks(enumerators, nextTicks); + } + } + + private IEnumerable ReplayFileGroupWithDelay(IEnumerable[] tickGroup, CancellationToken ct) + { + Int64 start = DateTime.UtcNow.Ticks; + Int64 offset = 0L; + + foreach (Tick tick in ReplayFileGroupWithoutDelay(tickGroup, ct)) + { + if (offset == 0L) + { + offset = start - tick.TimeReceived().Ticks; + } + + if (!ct.IsCancellationRequested) + { + System.Threading.SpinWait.SpinUntil(() => (tick.TimeReceived().Ticks + offset) <= DateTime.UtcNow.Ticks); + yield return tick; + } + } + } + + private void ReplayThreadFn() + { + CancellationToken ct = _ctSource.Token; + SubProvider[] subProviders = MapProviderToSubProviders(_config.Provider); + string[] replayFiles = new string[subProviders.Length]; + IEnumerable[] allTicks = new IEnumerable[subProviders.Length]; + + try + { + for (int i = 0; i < subProviders.Length; i++) + { + LogMessage(LogLevel.INFORMATION, "Downloading Replay file for {0} on {1}...", subProviders[i].ToString(), _date.Date.ToString()); + replayFiles[i] = FetchReplayFile(subProviders[i]); + LogMessage(LogLevel.INFORMATION, "Downloaded Replay file to: {0}", replayFiles[i]); + allTicks[i] = ReplayTickFileWithoutDelay(replayFiles[i], 100, ct); + } + + IEnumerable aggregatedTicks = _withSimulatedDelay + ? ReplayFileGroupWithDelay(allTicks, ct) + : ReplayFileGroupWithoutDelay(allTicks, ct); + + foreach (Tick tick in aggregatedTicks) + { + if (!ct.IsCancellationRequested) + { + Interlocked.Increment(ref _dataEventCount); + Interlocked.Increment(ref _dataMsgCount); + _data.Enqueue(tick); + } + } + } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while replaying file: {0}", e.Message); + } + + if (_deleteFileWhenDone) + { + foreach (string deleteFilePath in replayFiles) + { + if (File.Exists(deleteFilePath)) + { + LogMessage(LogLevel.INFORMATION, "Deleting Replay file: {0}", deleteFilePath); + File.Delete(deleteFilePath); + } + } + } + } + + private void Join(string symbol, bool tradesOnly) + { + string lastOnly = tradesOnly ? "true" : "false"; + if (_channels.Add(new (symbol, tradesOnly))) + { + LogMessage(LogLevel.INFORMATION, "Websocket - Joining channel: {0} (trades only = {1})", symbol, lastOnly); + } + } + + private void Leave(string symbol, bool tradesOnly) + { + string lastOnly = tradesOnly ? "true" : "false"; + if (_channels.Remove(new (symbol, tradesOnly))) + { + LogMessage(LogLevel.INFORMATION, "Websocket - Leaving channel: {0} (trades only = {1})", symbol, lastOnly); + } + } + #endregion //Private Methods + + private record Channel(string ticker, bool tradesOnly); +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/SubProvider.cs b/Intrinio.Realtime/Equities/SubProvider.cs new file mode 100644 index 0000000..8033299 --- /dev/null +++ b/Intrinio.Realtime/Equities/SubProvider.cs @@ -0,0 +1,12 @@ +namespace Intrinio.Realtime.Equities; + +public enum SubProvider +{ + NONE = 0, + CTA_A = 1, + CTA_B = 2, + UTP = 3, + OTC = 4, + NASDAQ_BASIC = 5, + IEX = 6 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/Tick.cs b/Intrinio.Realtime/Equities/Tick.cs new file mode 100644 index 0000000..0b96060 --- /dev/null +++ b/Intrinio.Realtime/Equities/Tick.cs @@ -0,0 +1,131 @@ +namespace Intrinio.Realtime.Equities; + +using System; +using System.Text; + +internal class Tick +{ + private readonly DateTime _timeReceived; + private readonly Trade? _trade; + private readonly Quote? _quote; + + public Tick(DateTime timeReceived, Trade? trade, Quote? quote) + { + _timeReceived = timeReceived; + _trade = trade; + _quote = quote; + } + + public byte[] getTradeBytes(Trade trade) + { + byte[] symbolBytes = Encoding.ASCII.GetBytes(trade.Symbol); + byte symbolLength = Convert.ToByte(symbolBytes.Length); + int symbolLengthInt32 = Convert.ToInt32(symbolLength); + byte[] marketCenterBytes = BitConverter.GetBytes(trade.MarketCenter); + byte[] tradePrice = BitConverter.GetBytes(Convert.ToSingle(trade.Price)); + byte[] tradeSize = BitConverter.GetBytes(trade.Size); + byte[] timeStamp = BitConverter.GetBytes(Convert.ToUInt64((trade.Timestamp - DateTime.UnixEpoch).Ticks) * 100UL); + byte[] tradeTotalVolume = BitConverter.GetBytes(trade.TotalVolume); + byte[] condition = Encoding.ASCII.GetBytes(trade.Condition); + byte conditionLength = Convert.ToByte(condition.Length); + byte messageLength = Convert.ToByte(symbolLength + conditionLength + 27); + + byte[] bytes = GC.AllocateUninitializedArray(System.Convert.ToInt32(messageLength)); + bytes[0] = System.Convert.ToByte((int)(MessageType.Trade)); + bytes[1] = messageLength; + bytes[2] = symbolLength; + Array.Copy(symbolBytes, 0, bytes, 3, symbolLengthInt32); + bytes[3 + symbolLengthInt32] = Convert.ToByte((int)(trade.SubProvider)); + Array.Copy(marketCenterBytes, 0, bytes, 4 + symbolLengthInt32, marketCenterBytes.Length); + Array.Copy(tradePrice, 0, bytes, 6 + symbolLengthInt32, tradePrice.Length); + Array.Copy(tradeSize, 0, bytes, 10 + symbolLengthInt32, tradeSize.Length); + Array.Copy(timeStamp, 0, bytes, 14 + symbolLengthInt32, timeStamp.Length); + Array.Copy(tradeTotalVolume, 0, bytes, 22 + symbolLengthInt32, tradeTotalVolume.Length); + bytes[26 + symbolLengthInt32] = conditionLength; + Array.Copy(condition, 0, bytes, 27 + symbolLengthInt32, System.Convert.ToInt32(conditionLength)); + + // byte 0: message type (hasn't changed) + // byte 1: message length (in bytes, including bytes 0 and 1) + // byte 2: symbol length (in bytes) + // bytes[3...]: symbol string (ascii) + // next byte: source + // next 2 bytes: market center (as 1 char) + // next 4 bytes: trade price (float) + // next 4 bytes: trade size (uint) + // next 8 bytes: timestamp (uint64) + // next 4 bytes: trade total volume ((uint) + // next byte: condition len + // next bytes: condition string (ascii) + + return bytes; + } + + public byte[] getQuoteBytes(Quote quote) + { + byte[] symbolBytes = Encoding.ASCII.GetBytes(quote.Symbol); + byte symbolLength = Convert.ToByte(symbolBytes.Length); + int symbolLengthInt32 = Convert.ToInt32(symbolLength); + byte[] marketCenterBytes = BitConverter.GetBytes(quote.MarketCenter); + byte[] tradePrice = BitConverter.GetBytes(Convert.ToSingle(quote.Price)); + byte[] tradeSize = BitConverter.GetBytes(quote.Size); + byte[] timeStamp = BitConverter.GetBytes(Convert.ToUInt64((quote.Timestamp - DateTime.UnixEpoch).Ticks) * 100UL); + byte[] condition = Encoding.ASCII.GetBytes(quote.Condition); + byte conditionLength = Convert.ToByte(condition.Length); + byte messageLength = Convert.ToByte(23 + symbolLength + conditionLength); + + byte[] bytes = System.GC.AllocateUninitializedArray(System.Convert.ToInt32(messageLength)); + bytes[0] = System.Convert.ToByte((int)quote.Type); + bytes[1] = messageLength; + bytes[2] = symbolLength; + Array.Copy(symbolBytes, 0, bytes, 3, symbolLengthInt32); + bytes[3 + symbolLengthInt32] = System.Convert.ToByte((int)(quote.SubProvider)); + Array.Copy(marketCenterBytes, 0, bytes, 4 + symbolLengthInt32, marketCenterBytes.Length); + Array.Copy(tradePrice, 0, bytes, 6 + symbolLengthInt32, tradePrice.Length); + Array.Copy(tradeSize, 0, bytes, 10 + symbolLengthInt32, tradeSize.Length); + Array.Copy(timeStamp, 0, bytes, 14 + symbolLengthInt32, timeStamp.Length); + bytes[22 + symbolLengthInt32] = conditionLength; + Array.Copy(condition, 0, bytes, 23 + symbolLengthInt32, System.Convert.ToInt32(conditionLength)); + + // byte 0: message type (hasn't changed) + // byte 1: message length (in bytes, including bytes 0 and 1) + // byte 2: symbol length (in bytes) + // bytes[3...]: symbol string (ascii) + // next byte: source + // next 2 bytes: market center (as 1 char) + // next 4 bytes: ask/bid price (float) + // next 4 bytes: ask/bid size (uint) + // next 8 bytes: timestamp (uint64) + // next byte: condition len + // next bytes: condition string (ascii) + + return bytes; + } + + public DateTime TimeReceived() + { + return _timeReceived; + } + + public bool IsTrade() + { + return _trade.HasValue; + } + + public Trade Trade { get {return _trade ?? default;} } + + public Quote Quote { get { return _quote ?? default; } } + + public byte[] GetTimeReceivedBytes() + { + return BitConverter.GetBytes(Convert.ToUInt64((_timeReceived - DateTime.UnixEpoch).Ticks) * 100UL); + } + + public byte[] GetEventBytes() + { + return _trade.HasValue + ? getTradeBytes(_trade.Value) + : _quote.HasValue + ? getQuoteBytes(_quote.Value) + : Array.Empty(); + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/Trade.cs b/Intrinio.Realtime/Equities/Trade.cs new file mode 100644 index 0000000..404a326 --- /dev/null +++ b/Intrinio.Realtime/Equities/Trade.cs @@ -0,0 +1,43 @@ +namespace Intrinio.Realtime.Equities; + +using System; + +public readonly struct Trade +{ + public readonly string Symbol; + public readonly double Price; + public readonly UInt32 Size; + public readonly DateTime Timestamp; + public readonly SubProvider SubProvider; + public readonly char MarketCenter; + public readonly string Condition; + public readonly UInt64 TotalVolume; + + /// Symbol: the 'ticker' symbol + /// Price: the dollar price of the last trade + /// Size: the number of shares that were exchanged in the last trade + /// TotalVolume: the total number of shares that have been traded since market open + /// Timestamp: the time that the trade was executed (a unix timestamp representing the number of milliseconds (or better) since the unix epoch) + /// SubProvider: the specific provider this trade came from under the parent provider grouping. + public Trade(string symbol, double price, UInt32 size, UInt64 totalVolume, DateTime timestamp, SubProvider subProvider, char marketCenter, string condition) + { + Symbol = symbol; + Price = price; + Size = size; + Timestamp = timestamp; + SubProvider = subProvider; + MarketCenter = marketCenter; + Condition = condition; + TotalVolume = totalVolume; + } + + public override string ToString() + { + return $"Trade (Symbol: {Symbol}, Price: {Price}, Size: {Size}, TotalVolume: {TotalVolume}, Timestamp: {Timestamp}, SubProvider: {SubProvider}, MarketCenter: {MarketCenter}, Condition: {Condition})"; + } + + public bool IsDarkpool() + { + return MarketCenter.Equals((char)0) || MarketCenter.Equals('D') || MarketCenter.Equals('E') || Char.IsWhiteSpace(MarketCenter); + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Equities/TradeCandleStick.cs b/Intrinio.Realtime/Equities/TradeCandleStick.cs new file mode 100644 index 0000000..ecc89b7 --- /dev/null +++ b/Intrinio.Realtime/Equities/TradeCandleStick.cs @@ -0,0 +1,110 @@ +namespace Intrinio.Realtime.Equities; + +using System; + +public class TradeCandleStick : CandleStick, IEquatable, IComparable, IComparable +{ + private readonly string _symbol; + + public string Symbol + { + get { return _symbol; } + } + + public TradeCandleStick(string symbol, UInt32 volume, double price, double openTimestamp, double closeTimestamp, IntervalType interval, double tradeTime) + : base(volume, price, openTimestamp, closeTimestamp, interval, tradeTime) + { + _symbol = symbol; + } + + public TradeCandleStick(string symbol, UInt32 volume, double high, double low, double closePrice, double openPrice, double openTimestamp, double closeTimestamp, double firstTimestamp, double lastTimestamp, bool complete, double average, double change, IntervalType interval) + : base(volume, high, low, closePrice, openPrice, openTimestamp, closeTimestamp, firstTimestamp, lastTimestamp, complete, average, change, interval) + { + _symbol = symbol; + } + + public override bool Equals(object other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (other is TradeCandleStick) + && (Symbol.Equals(((TradeCandleStick)other).Symbol)) + && (Interval.Equals(((TradeCandleStick)other).Interval)) + && (OpenTimestamp.Equals(((TradeCandleStick)other).OpenTimestamp)) + ); + } + + public override int GetHashCode() + { + return Symbol.GetHashCode() ^ Interval.GetHashCode() ^ OpenTimestamp.GetHashCode(); + } + + public bool Equals(TradeCandleStick other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (Symbol.Equals(other.Symbol)) + && (Interval.Equals(other.Interval)) + && (OpenTimestamp.Equals(other.OpenTimestamp)) + ); + } + + public int CompareTo(object other) + { + return Equals(other) switch + { + true => 0, + false => ReferenceEquals(other, null) switch + { + true => 1, + false => (other is TradeCandleStick) switch + { + true => Symbol.CompareTo(((TradeCandleStick)other).Symbol) switch + { + < 0 => -1, + > 0 => 1, + 0 => Interval.CompareTo(((TradeCandleStick)other).Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => this.OpenTimestamp.CompareTo(((TradeCandleStick)other).OpenTimestamp) + } + }, + false => 1 + } + } + }; + } + + public int CompareTo(TradeCandleStick other) + { + return Equals(other) switch + { + true => 0, + false => Object.ReferenceEquals(other, null) switch + { + true => 1, + false => this.Symbol.CompareTo(other.Symbol) switch + { + < 0 => -1, + > 0 => 1, + 0 => this.Interval.CompareTo(other.Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => this.OpenTimestamp.CompareTo(other.OpenTimestamp) + } + } + } + }; + } + + public override string ToString() + { + return $"TradeCandleStick (Symbol: {Symbol}, Volume: {Volume.ToString()}, High: {High.ToString("f3")}, Low: {Low.ToString("f3")}, Close: {Close.ToString("f3")}, Open: {Open.ToString("f3")}, OpenTimestamp: {OpenTimestamp.ToString("f6")}, CloseTimestamp: {CloseTimestamp.ToString("f6")}, AveragePrice: {Average.ToString("f3")}, Change: {Change.ToString("f6")}, Complete: {Complete.ToString()})"; + } +} \ No newline at end of file diff --git a/IntrinioRealtimeMultiExchange/config.json b/Intrinio.Realtime/Equities/config.json similarity index 71% rename from IntrinioRealtimeMultiExchange/config.json rename to Intrinio.Realtime/Equities/config.json index abec0c4..5851eeb 100644 --- a/IntrinioRealtimeMultiExchange/config.json +++ b/Intrinio.Realtime/Equities/config.json @@ -1,14 +1,16 @@ { "Config": { - "ApiKey": "", - "NumThreads": 2, + "ApiKey": "API_KEY_HERE", + "NumThreads": 4, "Provider": "REALTIME", //"Provider": "DELAYED_SIP", //"Provider": "NASDAQ_BASIC", //"Provider": "MANUAL", - "Symbols": [ "AAPL", "MSFT", "GOOG" ] + //"IPAddress": "1.2.3.4", + "BufferSize": 2048, + "OverflowBufferSize": 2048, + "Symbols": [ "AAPL", "MSFT", "TSLA" ] //"Symbols": [ "lobby" ] - //"IPAddress": "1.2.3.4" }, "Serilog": { "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], diff --git a/Intrinio.Realtime/IntervalType.cs b/Intrinio.Realtime/IntervalType.cs new file mode 100644 index 0000000..ea7fc3c --- /dev/null +++ b/Intrinio.Realtime/IntervalType.cs @@ -0,0 +1,14 @@ +namespace Intrinio.Realtime; + +public enum IntervalType +{ + OneMinute = 60, + TwoMinute = 120, + ThreeMinute = 180, + FourMinute = 240, + FiveMinute = 300, + TenMinute = 600, + FifteenMinute = 900, + ThirtyMinute = 1800, + SixtyMinute = 3600 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Intrinio.Realtime.csproj b/Intrinio.Realtime/Intrinio.Realtime.csproj new file mode 100644 index 0000000..84f8182 --- /dev/null +++ b/Intrinio.Realtime/Intrinio.Realtime.csproj @@ -0,0 +1,29 @@ + + + + net8.0 + disable + enable + true + true + + + + + PreserveNewest + + + + + + + + + + + + + + + + diff --git a/Intrinio.Realtime/LogLevel.cs b/Intrinio.Realtime/LogLevel.cs new file mode 100644 index 0000000..01dc904 --- /dev/null +++ b/Intrinio.Realtime/LogLevel.cs @@ -0,0 +1,9 @@ +namespace Intrinio.Realtime; + +public enum LogLevel +{ + DEBUG = 0, + INFORMATION = 1, + WARNING = 2, + ERROR = 3 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Logging.cs b/Intrinio.Realtime/Logging.cs new file mode 100644 index 0000000..48ce8d9 --- /dev/null +++ b/Intrinio.Realtime/Logging.cs @@ -0,0 +1,29 @@ +using System; + +namespace Intrinio.Realtime; + +public static class Logging +{ + [Serilog.Core.MessageTemplateFormatMethod("messageTemplate")] + public static void Log(LogLevel logLevel, string messageTemplate, params object[] propertyValues) + { + switch (logLevel) + { + case LogLevel.DEBUG: + Serilog.Log.Debug(messageTemplate, propertyValues); + break; + case LogLevel.INFORMATION: + Serilog.Log.Information(messageTemplate, propertyValues); + break; + case LogLevel.WARNING: + Serilog.Log.Warning(messageTemplate, propertyValues); + break; + case LogLevel.ERROR: + Serilog.Log.Error(messageTemplate, propertyValues); + break; + default: + throw new ArgumentException("LogLevel not specified!"); + break; + } + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/CandleStickClient.cs b/Intrinio.Realtime/Options/CandleStickClient.cs new file mode 100644 index 0000000..05e4e1b --- /dev/null +++ b/Intrinio.Realtime/Options/CandleStickClient.cs @@ -0,0 +1,711 @@ +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Options; + +using Intrinio; +using Serilog; +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Runtime.CompilerServices; + +public delegate TradeCandleStick FetchHistoricalTradeCandleStick(string contract, double openTimestamp, double closeTimestamp, IntervalType interval); +public delegate QuoteCandleStick FetchHistoricalQuoteCandleStick(string contract, double openTimestamp, double closeTimestamp, QuoteType quoteType, IntervalType interval); + +public class CandleStickClient +{ + #region Data Members + private readonly IntervalType _interval; + private readonly bool _broadcastPartialCandles; + private readonly double _sourceDelaySeconds; + private readonly bool _useTradeFiltering; + private readonly CancellationTokenSource _ctSource; + private const int InitialDictionarySize = 3_601_579; //a close prime number greater than 2x the max expected size. There are usually around 1.5m option contracts. + private readonly object _contractBucketsLock; + private readonly object _lostAndFoundLock; + private readonly Dictionary _contractBuckets; + private readonly Dictionary _lostAndFound; + private const double FlushBufferSeconds = 30.0; + private readonly Thread _lostAndFoundThread; + private readonly Thread _flushThread; + + private bool UseOnTradeCandleStick { get { return !ReferenceEquals(OnTradeCandleStick, null); } } + private bool UseOnQuoteCandleStick { get { return !ReferenceEquals(OnQuoteCandleStick, null); } } + private bool UseGetHistoricalTradeCandleStick { get { return !ReferenceEquals(GetHistoricalTradeCandleStick,null); } } + private bool UseGetHistoricalQuoteCandleStick { get { return !ReferenceEquals(GetHistoricalQuoteCandleStick,null); } } + + /// + /// The callback used for broadcasting trade candles. + /// + public Action OnTradeCandleStick { get; set; } + + /// + /// The callback used for broadcasting quote candles. + /// + private Action OnQuoteCandleStick { get; set; } + + /// + /// Fetch a previously broadcasted trade candlestick from the given unique parameters. + /// + public FetchHistoricalTradeCandleStick GetHistoricalTradeCandleStick { get; set; } + + /// + /// Fetch a previously broadcasted quote candlestick from the given unique parameters. + /// + public FetchHistoricalQuoteCandleStick GetHistoricalQuoteCandleStick { get; set; } + #endregion //Data Members + + #region Constructors + + /// + /// Creates an equities CandleStickClient that creates trade and quote candlesticks from a stream of trades and quotes. + /// + /// + /// + /// + /// + /// + /// + /// + /// + public CandleStickClient( + Action onTradeCandleStick, + Action onQuoteCandleStick, + IntervalType interval, + bool broadcastPartialCandles, + FetchHistoricalTradeCandleStick getHistoricalTradeCandleStick, + FetchHistoricalQuoteCandleStick getHistoricalQuoteCandleStick, + double sourceDelaySeconds) + { + this.OnTradeCandleStick = onTradeCandleStick; + this.OnQuoteCandleStick = onQuoteCandleStick; + this._interval = interval; + this._broadcastPartialCandles = broadcastPartialCandles; + this.GetHistoricalTradeCandleStick = getHistoricalTradeCandleStick; + this.GetHistoricalQuoteCandleStick = getHistoricalQuoteCandleStick; + this._sourceDelaySeconds = sourceDelaySeconds; + _ctSource = new CancellationTokenSource(); + _contractBucketsLock = new object(); + _lostAndFoundLock = new object(); + _contractBuckets = new Dictionary(InitialDictionarySize); + _lostAndFound = new Dictionary(InitialDictionarySize); + _lostAndFoundThread = new Thread(new ThreadStart(LostAndFoundFn)); + _flushThread = new Thread(new ThreadStart(FlushFn)); + } + #endregion //Constructors + + #region Public Methods + + public void OnTrade(Trade trade) + { + try + { + if (UseOnTradeCandleStick) + { + ContractBucket bucket = GetSlot(trade.Contract, _contractBuckets, _contractBucketsLock); + + lock (bucket.Locker) + { + double ts = ConvertToUnixTimestamp(trade.Timestamp); + + if (bucket.TradeCandleStick != null) + { + if (bucket.TradeCandleStick.CloseTimestamp < ts) + { + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = CreateNewTradeCandle(trade, ts); + } + else if (bucket.TradeCandleStick.OpenTimestamp <= ts) + { + bucket.TradeCandleStick.Update(trade.Size, trade.Price, ts); + if (_broadcastPartialCandles) + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + } + else //This is a late trade. We already shipped the candle, so add to lost and found + { + AddTradeToLostAndFound(trade); + } + } + else + { + bucket.TradeCandleStick = CreateNewTradeCandle(trade, ts); + if (_broadcastPartialCandles) + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + } + } + } + } + catch (Exception e) + { + Log.Warning("Error on handling trade in CandleStick Client: {0}", e.Message); + } + } + + public void OnQuote(Quote quote) + { + try + { + if (UseOnQuoteCandleStick) + { + ContractBucket bucket = GetSlot(quote.Contract, _contractBuckets, _contractBucketsLock); + + lock (bucket.Locker) + { + OnAsk(quote, bucket); + OnBid(quote, bucket); + } + } + } + catch (Exception e) + { + Log.Warning("Error on handling trade in CandleStick Client: {0}", e.Message); + } + } + + public void Start() + { + if (!_flushThread.IsAlive) + { + _flushThread.Start(); + } + + if (!_lostAndFoundThread.IsAlive) + { + _lostAndFoundThread.Start(); + } + } + + public void Stop() + { + _ctSource.Cancel(); + } + #endregion //Public Methods + + #region Private Methods + + private TradeCandleStick CreateNewTradeCandle(Trade trade, double timestamp) + { + double start = GetNearestModInterval(timestamp, _interval); + TradeCandleStick freshCandle = new TradeCandleStick(trade.Contract, trade.Size, trade.Price, start, (start + System.Convert.ToDouble((int)_interval)), _interval, timestamp); + + if (UseGetHistoricalTradeCandleStick && UseOnTradeCandleStick) + { + try + { + TradeCandleStick historical = GetHistoricalTradeCandleStick(freshCandle.Contract, freshCandle.OpenTimestamp, freshCandle.CloseTimestamp, freshCandle.Interval); + if (ReferenceEquals(historical,null)) + return freshCandle; + historical.MarkIncomplete(); + return MergeTradeCandles(historical, freshCandle); + } + catch (Exception e) + { + Log.Error("Error retrieving historical TradeCandleStick: {0}; trade: {1}", e.Message, trade); + return freshCandle; + } + } + else + { + return freshCandle; + } + } + + private QuoteCandleStick CreateNewAskCandle(Quote quote, double timestamp) + { + double start = GetNearestModInterval(timestamp, _interval); + QuoteCandleStick freshCandle = new QuoteCandleStick(quote.Contract, quote.AskSize, quote.AskPrice, QuoteType.Ask, start, (start + System.Convert.ToDouble((int)_interval)), _interval, timestamp); + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(freshCandle.Contract, freshCandle.OpenTimestamp, freshCandle.CloseTimestamp, freshCandle.QuoteType, freshCandle.Interval); + if (ReferenceEquals(historical,null)) + return freshCandle; + historical.MarkIncomplete(); + return MergeQuoteCandles(historical, freshCandle); + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}; quote: {1}", e.Message, quote); + return freshCandle; + } + } + else + { + return freshCandle; + } + } + + private QuoteCandleStick CreateNewBidCandle(Quote quote, double timestamp) + { + double start = GetNearestModInterval(timestamp, _interval); + QuoteCandleStick freshCandle = new QuoteCandleStick(quote.Contract, quote.BidSize, quote.BidPrice, QuoteType.Bid, start, (start + System.Convert.ToDouble((int)_interval)), _interval, timestamp); + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(freshCandle.Contract, freshCandle.OpenTimestamp, freshCandle.CloseTimestamp, freshCandle.QuoteType, freshCandle.Interval); + if (ReferenceEquals(historical,null)) + return freshCandle; + historical.MarkIncomplete(); + return MergeQuoteCandles(historical, freshCandle); + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}; quote: {1}", e.Message, quote); + return freshCandle; + } + } + else + { + return freshCandle; + } + } + + private void AddAskToLostAndFound(Quote ask) + { + double ts = ConvertToUnixTimestamp(ask.Timestamp); + string key = String.Format("{0}|{1}|{2}", ask.Contract, GetNearestModInterval(ts, _interval), _interval); + ContractBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + try + { + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + lock (bucket.Locker) + { + if (bucket.AskCandleStick != null) + { + bucket.AskCandleStick.Update(ask.AskSize, ask.AskPrice, ts); + } + else + { + double start = GetNearestModInterval(ts, _interval); + bucket.AskCandleStick = new QuoteCandleStick(ask.Contract, ask.AskSize, ask.AskPrice, QuoteType.Ask, start, (start + System.Convert.ToDouble((int)_interval)), _interval, ts); + } + } + } + } + catch (Exception ex) + { + Log.Warning("Error on handling late ask in CandleStick Client: {0}", ex.Message); + } + } + + private void AddBidToLostAndFound(Quote bid) + { + double ts = ConvertToUnixTimestamp(bid.Timestamp); + string key = String.Format("{0}|{1}|{2}", bid.Contract, GetNearestModInterval(ts, _interval), _interval); + ContractBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + try + { + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick) + { + lock (bucket.Locker) + { + if (bucket.BidCandleStick != null) + { + bucket.BidCandleStick.Update(bid.BidSize, bid.AskPrice, ts); + } + else + { + double start = GetNearestModInterval(ts, _interval); + bucket.BidCandleStick = new QuoteCandleStick(bid.Contract, bid.BidSize, bid.AskPrice, QuoteType.Bid, start, (start + System.Convert.ToDouble((int) _interval)), _interval, ts); + } + } + } + } + catch (Exception ex) + { + Log.Warning("Error on handling late bid in CandleStick Client: {0}", ex.Message); + } + } + + private void AddTradeToLostAndFound(Trade trade) + { + double ts = ConvertToUnixTimestamp(trade.Timestamp); + string key = String.Format("{0}|{1}|{2}", trade.Contract, GetNearestModInterval(ts, _interval), _interval); + ContractBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + try + { + if (UseGetHistoricalTradeCandleStick && UseOnTradeCandleStick) + { + lock (bucket.Locker) + { + if (bucket.TradeCandleStick != null) + { + bucket.TradeCandleStick.Update(trade.Size, trade.Price, ts); + } + else + { + double start = GetNearestModInterval(ts, _interval); + bucket.TradeCandleStick = new TradeCandleStick(trade.Contract, trade.Size, trade.Price, start, (start + System.Convert.ToDouble((int)_interval)), _interval, ts); + } + } + } + } + catch (Exception ex) + { + Log.Warning("Error on handling late trade in CandleStick Client: {0}", ex.Message); + } + } + + private void OnAsk(Quote quote, ContractBucket bucket) + { + double ts = ConvertToUnixTimestamp(quote.Timestamp); + + if (bucket.AskCandleStick != null && !Double.IsNaN(quote.AskPrice)) + { + if (bucket.AskCandleStick.CloseTimestamp < ts) + { + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = CreateNewAskCandle(quote, ts); + } + else if (bucket.AskCandleStick.OpenTimestamp <= ts) + { + bucket.AskCandleStick.Update(quote.AskSize, quote.AskPrice, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + } + else //This is a late event. We already shipped the candle, so add to lost and found + { + AddAskToLostAndFound(quote); + } + } + else if (bucket.AskCandleStick == null && !Double.IsNaN(quote.AskPrice)) + { + bucket.AskCandleStick = CreateNewAskCandle(quote, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + } + } + + private void OnBid(Quote quote, ContractBucket bucket) + { + double ts = ConvertToUnixTimestamp(quote.Timestamp); + + if (bucket.BidCandleStick != null && !Double.IsNaN(quote.AskPrice)) + { + if (bucket.BidCandleStick.CloseTimestamp < ts) + { + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = CreateNewBidCandle(quote, ts); + } + else if(bucket.BidCandleStick.OpenTimestamp <= ts) + { + bucket.BidCandleStick.Update(quote.BidSize, quote.AskPrice, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + } + else //This is a late event. We already shipped the candle, so add to lost and found + { + AddBidToLostAndFound(quote); + } + } + else if (bucket.BidCandleStick == null && !Double.IsNaN(quote.AskPrice)) + { + bucket.BidCandleStick = CreateNewBidCandle(quote, ts); + if (_broadcastPartialCandles) + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + } + } + + private void FlushFn() + { + Log.Information("Starting candlestick expiration watcher..."); + CancellationToken ct = _ctSource.Token; + System.Threading.Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; + List keys = new List(); + + while (!ct.IsCancellationRequested) + { + try + { + lock (_contractBucketsLock) + { + foreach (string key in _contractBuckets.Keys) + keys.Add(key); + } + + foreach (string key in keys) + { + ContractBucket bucket = GetSlot(key, _contractBuckets, _contractBucketsLock); + double flushThresholdTime = GetCurrentTimestamp(_sourceDelaySeconds) - FlushBufferSeconds; + + lock (bucket.Locker) + { + if (UseOnTradeCandleStick && bucket.TradeCandleStick != null && (bucket.TradeCandleStick.CloseTimestamp < flushThresholdTime)) + { + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + + if (UseOnQuoteCandleStick && bucket.AskCandleStick != null && (bucket.AskCandleStick.CloseTimestamp < flushThresholdTime)) + { + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + + if (UseOnQuoteCandleStick && bucket.BidCandleStick != null && (bucket.BidCandleStick.CloseTimestamp < flushThresholdTime)) + { + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + } + } + keys.Clear(); + + if (!(ct.IsCancellationRequested)) + Thread.Sleep(1000); + } + catch (OperationCanceledException) + { + } + } + + Log.Information("Stopping candlestick expiration watcher..."); + } + + private async void LostAndFoundFn() + { + Log.Information("Starting candlestick late event watcher..."); + CancellationToken ct = _ctSource.Token; + System.Threading.Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; + List keys = new List(); + + while (!ct.IsCancellationRequested) + { + try + { + lock (_lostAndFoundLock) + { + foreach (string key in _lostAndFound.Keys) + keys.Add(key); + } + + foreach (string key in keys) + { + ContractBucket bucket = GetSlot(key, _lostAndFound, _lostAndFoundLock); + + lock (bucket.Locker) + { + if (UseGetHistoricalTradeCandleStick && UseOnTradeCandleStick && bucket.TradeCandleStick != null) + { + try + { + TradeCandleStick historical = GetHistoricalTradeCandleStick.Invoke(bucket.TradeCandleStick.Contract, bucket.TradeCandleStick.OpenTimestamp, bucket.TradeCandleStick.CloseTimestamp, bucket.TradeCandleStick.Interval); + if (ReferenceEquals(historical,null)) + { + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + else + { + bucket.TradeCandleStick = MergeTradeCandles(historical, bucket.TradeCandleStick); + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + } + catch (Exception e) + { + Log.Error("Error retrieving historical TradeCandleStick: {0}", e.Message); + bucket.TradeCandleStick.MarkComplete(); + OnTradeCandleStick.Invoke(bucket.TradeCandleStick); + bucket.TradeCandleStick = null; + } + } + else + { + bucket.TradeCandleStick = null; + } + + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick && bucket.AskCandleStick != null) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(bucket.AskCandleStick.Contract, bucket.AskCandleStick.OpenTimestamp, bucket.AskCandleStick.CloseTimestamp, bucket.AskCandleStick.QuoteType, bucket.AskCandleStick.Interval); + if (ReferenceEquals(historical,null)) + { + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + else + { + bucket.AskCandleStick = MergeQuoteCandles(historical, bucket.AskCandleStick); + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}", e.Message); + bucket.AskCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.AskCandleStick); + bucket.AskCandleStick = null; + } + } + else + { + bucket.AskCandleStick = null; + } + + if (UseGetHistoricalQuoteCandleStick && UseOnQuoteCandleStick && bucket.BidCandleStick != null) + { + try + { + QuoteCandleStick historical = GetHistoricalQuoteCandleStick.Invoke(bucket.BidCandleStick.Contract, bucket.BidCandleStick.OpenTimestamp, bucket.BidCandleStick.CloseTimestamp, bucket.BidCandleStick.QuoteType, bucket.BidCandleStick.Interval); + if (ReferenceEquals(historical,null)) + { + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + else + { + bucket.BidCandleStick = MergeQuoteCandles(historical, bucket.BidCandleStick); + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + } + catch (Exception e) + { + Log.Error("Error retrieving historical QuoteCandleStick: {0}", e.Message); + bucket.BidCandleStick.MarkComplete(); + OnQuoteCandleStick.Invoke(bucket.BidCandleStick); + bucket.BidCandleStick = null; + } + } + else + { + bucket.BidCandleStick = null; + } + + if (bucket.TradeCandleStick == null && bucket.AskCandleStick == null && bucket.BidCandleStick == null) + RemoveSlot(key, _lostAndFound, _lostAndFoundLock); + } + } + keys.Clear(); + + if (!ct.IsCancellationRequested) + Thread.Sleep(1000); + } + catch (OperationCanceledException) + { + } + } + + Log.Information("Stopping candlestick late event watcher..."); + } + + #endregion //Private Methods + + private class ContractBucket + { + public TradeCandleStick TradeCandleStick; + public QuoteCandleStick AskCandleStick; + public QuoteCandleStick BidCandleStick; + public object Locker; + + public ContractBucket(TradeCandleStick tradeCandleStick, QuoteCandleStick askCandleStick, QuoteCandleStick bidCandleStick) + { + TradeCandleStick = tradeCandleStick; + AskCandleStick = askCandleStick; + BidCandleStick = bidCandleStick; + Locker = new object(); + } + } + + #region Private Static Methods + // [SkipLocalsInit] + // [MethodImpl(MethodImplOptions.AggressiveInlining)] + // private static Span StackAlloc(int length) where T : unmanaged + // { + // unsafe + // { + // Span p = stackalloc T[length]; + // return p; + // } + // } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static double GetCurrentTimestamp(double delay) + { + return (DateTime.UtcNow - DateTime.UnixEpoch.ToUniversalTime()).TotalSeconds - delay; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static double GetNearestModInterval(double timestamp, IntervalType interval) + { + return Convert.ToDouble(Convert.ToUInt64(timestamp) / Convert.ToUInt64((int)interval)) * Convert.ToDouble(((int)interval)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TradeCandleStick MergeTradeCandles(TradeCandleStick a, TradeCandleStick b) + { + a.Merge(b); + return a; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static QuoteCandleStick MergeQuoteCandles(QuoteCandleStick a, QuoteCandleStick b) + { + a.Merge(b); + return a; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static double ConvertToUnixTimestamp(double input) + { + return input; + } + + private static ContractBucket GetSlot(string key, Dictionary dict, object locker) + { + ContractBucket value; + if (dict.TryGetValue(key, out value)) + { + return value; + } + + lock (locker) + { + if (dict.TryGetValue(key, out value)) + { + return value; + } + + ContractBucket bucket = new ContractBucket(null, null, null); + dict.Add(key, bucket); + return bucket; + } + } + + private static void RemoveSlot(string key, Dictionary dict, object locker) + { + if (dict.ContainsKey(key)) + { + lock (locker) + { + if (dict.ContainsKey(key)) + { + dict.Remove(key); + } + } + } + } + #endregion //Private Static Methods +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Conditions.cs b/Intrinio.Realtime/Options/Conditions.cs new file mode 100644 index 0000000..cf1bc22 --- /dev/null +++ b/Intrinio.Realtime/Options/Conditions.cs @@ -0,0 +1,9 @@ +using System.Runtime.CompilerServices; + +namespace Intrinio.Realtime.Options; + +[InlineArray(4)] +public struct Conditions +{ + public byte _a; +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Config.cs b/Intrinio.Realtime/Options/Config.cs new file mode 100644 index 0000000..4f6c094 --- /dev/null +++ b/Intrinio.Realtime/Options/Config.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Intrinio.Realtime.Options; + +using System; +using Serilog; +using System.IO; +using Microsoft.Extensions.Configuration; + +public class Config +{ + public string ApiKey { get; set; } + + public Provider Provider { get; set; } + + public string IPAddress { get; set; } + + public string[] Symbols { get; set; } + + public bool TradesOnly { get; set; } + + public int NumThreads { get; set; } + + public int BufferSize { get; set; } + + public int OverflowBufferSize { get; set; } + + public bool Delayed { get; set; } + + /// + /// The configuration for The Options Websocket Client. + /// + public Config() + { + ApiKey = String.Empty; + Provider = Provider.NONE; + IPAddress = String.Empty; + Symbols = Array.Empty(); + TradesOnly = false; + NumThreads = 2; + BufferSize = 2048; + OverflowBufferSize = 2048; + Delayed = false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static string TranslateContract(string contract) + { + if ((contract.Length <= 9) || (contract.IndexOf(".")>=9)) + { + return contract; + } + + //this is of the old format and we need to translate it to what the server understands. input: AAPL__220101C00140000, TSLA__221111P00195000 + string symbol = contract.Substring(0, 6).TrimEnd('_'); + string date = contract.Substring(6, 6); + char callPut = contract[12]; + string wholePrice = contract.Substring(13, 5).TrimStart('0'); + if (wholePrice == String.Empty) + { + wholePrice = "0"; + } + + string decimalPrice = contract.Substring(18); + + if (decimalPrice[2] == '0') + decimalPrice = decimalPrice.Substring(0, 2); + + return String.Format($"{symbol}_{date}{callPut}{wholePrice}.{decimalPrice}"); + } + + public void Validate() + { + if (String.IsNullOrWhiteSpace(ApiKey)) + { + throw new ArgumentException("You must provide a valid API key"); + } + + if (Provider == Provider.NONE) + { + throw new ArgumentException("You must specify a valid 'provider'"); + } + + if ((Provider == Provider.MANUAL) && (String.IsNullOrWhiteSpace(IPAddress))) + { + throw new ArgumentException("You must specify an IP address for manual configuration"); + } + + if (NumThreads <= 0) + { + throw new ArgumentException("You must specify a valid 'NumThreads'"); + } + + if (BufferSize < 2048) + { + throw new ArgumentException("'BufferSize' must be greater than or equal to 2048."); + } + + if (OverflowBufferSize < 2048) + { + throw new ArgumentException("'OverflowBufferSize' must be greater than or equal to 2048."); + } + + for (int i = 0; i < Symbols.Length; i++) + { + Symbols[i] = TranslateContract(Symbols[i]); + } + } + + public static Config LoadConfig() + { + Log.Information("Loading application configuration"); + var rawConfig = new ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("config.json").Build(); + Log.Logger = new LoggerConfiguration().ReadFrom.Configuration(rawConfig).CreateLogger(); + Config config = new Config(); + + foreach (KeyValuePair kvp in rawConfig.AsEnumerable()) + { + Log.Debug("Key: {0}, Value:{1}", kvp.Key, kvp.Value); + } + + rawConfig.Bind("Config", config); + config.Validate(); + return config; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Exchange.cs b/Intrinio.Realtime/Options/Exchange.cs new file mode 100644 index 0000000..53bcac5 --- /dev/null +++ b/Intrinio.Realtime/Options/Exchange.cs @@ -0,0 +1,24 @@ +namespace Intrinio.Realtime.Options; + +public enum Exchange +{ + UNKNOWN = '?', + NYSE_AMERICAN = 'A', + BOSTON = 'B', + CBOE = 'C', + MIAMI_EMERALD = 'D', + BATS_EDGX = 'E', + ISE_GEMINI = 'H', + ISE = 'I', + MERCURY = 'J', + MIAMI = 'M', + NYSE_ARCA = 'N', + MIAMI_PEARL = 'P', + NASDAQ = 'Q', + MIAX_SAPPHIRE = 'S', + NASDAQ_BX = 'T', + MEMX = 'U', + CBOE_C2 = 'W', + PHLX = 'X', + BATS_BZX = 'Z' +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Helpers.cs b/Intrinio.Realtime/Options/Helpers.cs new file mode 100644 index 0000000..e323bd2 --- /dev/null +++ b/Intrinio.Realtime/Options/Helpers.cs @@ -0,0 +1,50 @@ +using System; +using System.Runtime.CompilerServices; + +namespace Intrinio.Realtime.Options; + +internal static class Helpers +{ + internal static readonly double[] _priceTypeDivisorTable = new double[] + { + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 512, + 0.0, + 0.0, + 0.0, + 0.0, + Double.NaN + }; + + // [] + // let inline internal stackalloc<'a when 'a: unmanaged> (length: int): Span<'a> = + // let p = NativePtr.stackalloc<'a> length |> NativePtr.toVoidPtr + // Span<'a>(p, length) + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static double ScaleUInt64Price(UInt64 price, byte priceType) + { + return ((double)price) / _priceTypeDivisorTable[(int)priceType]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static double ScaleInt32Price(int price, byte priceType) + { + return ((double)price) / _priceTypeDivisorTable[(int)priceType]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static double ScaleTimestampToSeconds(UInt64 nanoseconds) + { + return ((double) nanoseconds) / 1_000_000_000.0; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/IOptionsWebSocketClient.cs b/Intrinio.Realtime/Options/IOptionsWebSocketClient.cs new file mode 100644 index 0000000..0d451a3 --- /dev/null +++ b/Intrinio.Realtime/Options/IOptionsWebSocketClient.cs @@ -0,0 +1,28 @@ +using System; +using System.Threading.Tasks; + +namespace Intrinio.Realtime.Options; + +public interface IOptionsWebSocketClient +{ + public Action OnTrade { set; } + public Action OnQuote { set; } + public Action OnRefresh { set; } + public Action OnUnusualActivity { set; } + public Task Join(); + public Task Join(string channel, bool? tradesOnly); + public Task JoinLobby(bool? tradesOnly); + public Task Join(string[] channels, bool? tradesOnly); + public Task Leave(); + public Task Leave(string channel); + public Task LeaveLobby(); + public Task Leave(string[] channels); + public Task Stop(); + public Task Start(); + public ClientStats GetStats(); + public UInt64 TradeCount { get; } + public UInt64 QuoteCount { get; } + public UInt64 RefreshCount { get; } + public UInt64 UnusualActivityCount { get; } + public void LogMessage(LogLevel logLevel, string messageTemplate, params object[] propertyValues); +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/MessageType.cs b/Intrinio.Realtime/Options/MessageType.cs new file mode 100644 index 0000000..f5787da --- /dev/null +++ b/Intrinio.Realtime/Options/MessageType.cs @@ -0,0 +1,9 @@ +namespace Intrinio.Realtime.Options; + +public enum MessageType +{ + Trade = 0, + Quote = 1, + Refresh = 2, + UnusualActivity = 3 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/OptionsWebSocketClient.cs b/Intrinio.Realtime/Options/OptionsWebSocketClient.cs new file mode 100644 index 0000000..d27b4f3 --- /dev/null +++ b/Intrinio.Realtime/Options/OptionsWebSocketClient.cs @@ -0,0 +1,582 @@ +using System; +using System.Text; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Intrinio.Realtime.Options; + +public class OptionsWebSocketClient : WebSocketClient, IOptionsWebSocketClient +{ + #region Data Members + + private const string LobbyName = "lobby"; + private const uint MaxMessageSize = 75u; + private const int MessageTypeIndex = 22; + private const int TradeMessageSize = 72; + private const int QuoteMessageSize = 52; + private const int RefreshMessageSize = 52; + private const int UnusualActivityMessageSize = 74; + private bool _useOnTrade; + private bool _useOnQuote; + private bool _useOnRefresh; + private bool _useOnUnusualActivity; + private Action _onTrade; + + /// + /// The callback for when a trade event occurs. + /// + public Action OnTrade + { + set + { + _useOnTrade = !ReferenceEquals(value, null); + _onTrade = value; + } + } + + private Action _onQuote; + + /// + /// The callback for when a quote event occurs. + /// + public Action OnQuote + { + set + { + _useOnQuote = !ReferenceEquals(value, null); + _onQuote = value; + } + } + + private Action _onRefresh; + + /// + /// The callback for when a refresh event occurs. + /// + public Action OnRefresh + { + set + { + _useOnRefresh = !ReferenceEquals(value, null); + _onRefresh = value; + } + } + + private Action _onUnusualActivity; + + /// + /// The callback for when an unusual activity event occurs. + /// + public Action OnUnusualActivity + { + set + { + _useOnUnusualActivity = !ReferenceEquals(value, null); + _onUnusualActivity = value; + } + } + + private readonly Config _config; + private UInt64 _dataTradeCount = 0UL; + private UInt64 _dataQuoteCount = 0UL; + private UInt64 _dataRefreshCount = 0UL; + private UInt64 _dataUnusualActivityCount = 0UL; + public UInt64 TradeCount { get { return Interlocked.Read(ref _dataTradeCount); } } + public UInt64 QuoteCount { get { return Interlocked.Read(ref _dataQuoteCount); } } + public UInt64 RefreshCount { get { return Interlocked.Read(ref _dataRefreshCount); } } + public UInt64 UnusualActivityCount { get { return Interlocked.Read(ref _dataUnusualActivityCount); } } + + private readonly string _logPrefix; + private const string MessageVersionHeaderKey = "UseNewOptionsFormat"; + private const string MessageVersionHeaderValue = "v2"; + private const string DelayHeaderKey = "delay"; + private const string DelayHeaderValue = "true"; + private const string ChannelFormat = "{0}|TradesOnly|{1}"; + #endregion //Data Members + + #region Constuctors + /// + /// Create a new Options websocket client. + /// + /// + /// + /// + /// + /// + public OptionsWebSocketClient(Action onTrade, Action onQuote, Action onRefresh, Action onUnusualActivity, Config config) + : base(Convert.ToUInt32(config.NumThreads), Convert.ToUInt32(config.BufferSize), Convert.ToUInt32(config.OverflowBufferSize), MaxMessageSize) + { + OnTrade = onTrade; + OnQuote = onQuote; + OnRefresh = onRefresh; + OnUnusualActivity = onUnusualActivity; + _config = config; + + if (ReferenceEquals(null, _config)) + throw new ArgumentException("Config may not be null."); + _config.Validate(); + _logPrefix = String.Format("{0}: ", _config?.Provider.ToString()); + } + + /// + /// Create a new Options websocket client. + /// + /// + public OptionsWebSocketClient(Action onTrade) : this(onTrade, null, null, null, Config.LoadConfig()) + { + } + + /// + /// Create a new Options websocket client. + /// + /// + public OptionsWebSocketClient(Action onQuote) : this(null, onQuote, null, null, Config.LoadConfig()) + { + } + + /// + /// Create a new Options websocket client. + /// + /// + /// + public OptionsWebSocketClient(Action onTrade, Action onQuote) : this(onTrade, onQuote, null, null, Config.LoadConfig()) + { + } + + /// + /// Create a new Options websocket client. + /// + /// + /// + /// + /// + public OptionsWebSocketClient(Action onTrade, Action onQuote, Action onRefresh, Action onUnusualActivity) : this(onTrade, onQuote, onRefresh, onUnusualActivity, Config.LoadConfig()) + { + } + #endregion //Constructors + + #region Public Methods + public async Task Join() + { + while (!IsReady()) + await Task.Delay(1000); + HashSet channelsToAdd = _config.Symbols.Select(s => GetChannel(s, _config.TradesOnly)).ToHashSet(); + channelsToAdd.ExceptWith(Channels); + foreach (string channel in channelsToAdd) + await JoinImpl(channel); + } + + public async Task Join(string symbol, bool? tradesOnly) + { + bool t = tradesOnly.HasValue ? tradesOnly.Value || _config.TradesOnly : false || _config.TradesOnly; + while (!IsReady()) + await Task.Delay(1000); + if (!Channels.Contains(GetChannel(symbol, t))) + await JoinImpl(GetChannel(symbol, t)); + } + + public async Task JoinLobby(bool? tradesOnly) + { + await Join(LobbyName, tradesOnly); + } + + public async Task Join(string[] symbols, bool? tradesOnly) + { + bool t = tradesOnly.HasValue ? tradesOnly.Value || _config.TradesOnly : false || _config.TradesOnly; + while (!IsReady()) + await Task.Delay(1000); + HashSet symbolsToAdd = symbols.Select(s => GetChannel(s, t)).ToHashSet(); + symbolsToAdd.ExceptWith(Channels); + foreach (string channel in symbolsToAdd) + await JoinImpl(channel); + } + + public async Task Leave() + { + await LeaveImpl(); + } + + public async Task Leave(string symbol) + { + foreach (string channel in Channels.Where(c => symbol == GetSymbolFromChannel(c))) + await LeaveImpl(channel); + } + + public async Task LeaveLobby() + { + await Leave(LobbyName); + } + + public async Task Leave(string[] symbols) + { + HashSet hashSymbols = new HashSet(symbols); + foreach (string channel in Channels.Where(c => hashSymbols.Contains(GetSymbolFromChannel(c)))) + await LeaveImpl(channel); + } + #endregion //Public Methods + + #region Private Methods + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private string GetChannel(string symbol, bool tradesOnly) + { + return String.Format(ChannelFormat, symbol, _config.TradesOnly.ToString()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private string GetSymbolFromChannel(string channel) + { + return channel.Split('|')[0]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool GetTradesOnlyFromChannel(string channel) + { + return Boolean.Parse(channel.Split('|')[2]); + } + + protected override string GetLogPrefix() + { + return _logPrefix; + } + + protected override string GetAuthUrl() + { + switch (_config.Provider) + { + case Provider.OPRA: + return $"https://realtime-options.intrinio.com/auth?api_key={_config.ApiKey}"; + break; + case Provider.MANUAL: + return $"http://{_config.IPAddress}/auth?api_key={_config.ApiKey}"; + break; + default: + throw new ArgumentException("Provider not specified!"); + break; + } + } + + protected override string GetWebSocketUrl(string token) + { + string delayedPart = _config.Delayed ? "&delayed=true" : String.Empty; + switch (_config.Provider) + { + case Provider.OPRA: + return $"wss://realtime-options.intrinio.com/socket/websocket?vsn=1.0.0&token={token}{delayedPart}"; + break; + case Provider.MANUAL: + return $"ws://{_config.IPAddress}/socket/websocket?vsn=1.0.0&token={token}{delayedPart}"; + break; + default: + throw new ArgumentException("Provider not specified!"); + break; + } + } + + protected override List> GetCustomSocketHeaders() + { + List> headers = new List>(); + headers.Add(new KeyValuePair(MessageVersionHeaderKey, MessageVersionHeaderValue)); + if (_config.Delayed) + headers.Add(new KeyValuePair(DelayHeaderKey, DelayHeaderValue)); + return headers; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected override int GetNextChunkLength(ReadOnlySpan bytes) + { + byte msgType = bytes[MessageTypeIndex]; //in the bytes, symbol length is first, then symbol, then msg type. + + //using if-else vs switch for hotpathing + if (msgType == 1u) + return QuoteMessageSize; + if (msgType == 0u) + return TradeMessageSize; + if (msgType == 2u) + return RefreshMessageSize; + return UnusualActivityMessageSize; + } + + private string FormatContract(ReadOnlySpan alternateFormattedChars) + { + //Transform from server format to normal format + //From this: AAPL_201016C100.00 or ABC_201016C100.003 + //Patch: some upstream contracts now have 4 decimals. We are truncating the last decimal for now to fit in this format. + //To this: AAPL__201016C00100000 or ABC___201016C00100003 + // strike: 5 whole digits, 3 decimal digits + + Span contractChars = stackalloc byte[21]; + contractChars[0] = (byte)'_'; + contractChars[1] = (byte)'_'; + contractChars[2] = (byte)'_'; + contractChars[3] = (byte)'_'; + contractChars[4] = (byte)'_'; + contractChars[5] = (byte)'_'; + contractChars[6] = (byte)'2'; + contractChars[7] = (byte)'2'; + contractChars[8] = (byte)'0'; + contractChars[9] = (byte)'1'; + contractChars[10] = (byte)'0'; + contractChars[11] = (byte)'1'; + contractChars[12] = (byte)'C'; + contractChars[13] = (byte)'0'; + contractChars[14] = (byte)'0'; + contractChars[15] = (byte)'0'; + contractChars[16] = (byte)'0'; + contractChars[17] = (byte)'0'; + contractChars[18] = (byte)'0'; + contractChars[19] = (byte)'0'; + contractChars[20] = (byte)'0'; + + int underscoreIndex = alternateFormattedChars.IndexOf((byte)'_'); + int decimalIndex = alternateFormattedChars.Slice(9).IndexOf((byte)'.') + 9; //ignore decimals in tickersymbol + + alternateFormattedChars.Slice(0, underscoreIndex).CopyTo(contractChars); //copy symbol + alternateFormattedChars.Slice(underscoreIndex + 1, 6).CopyTo(contractChars.Slice(6)); //copy date + alternateFormattedChars.Slice(underscoreIndex + 7, 1).CopyTo(contractChars.Slice(12)); //copy put/call + alternateFormattedChars.Slice(underscoreIndex + 8, decimalIndex - underscoreIndex - 8).CopyTo(contractChars.Slice(18 - (decimalIndex - underscoreIndex - 8))); //whole number copy + alternateFormattedChars.Slice(decimalIndex + 1, Math.Min(3, alternateFormattedChars.Length - decimalIndex - 1)).CopyTo(contractChars.Slice(18)); //decimal number copy. Truncate decimals over 3 digits for now. + + return Encoding.ASCII.GetString(contractChars); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Exchange ParseExchange(char c) + { + switch (c) + { + case 'A': + case 'a': + return Exchange.NYSE_AMERICAN; + case 'B': + case 'b': + return Exchange.BOSTON; + case 'C': + case 'c': + return Exchange.CBOE; + case 'D': + case 'd': + return Exchange.MIAMI_EMERALD; + case 'E': + case 'e': + return Exchange.BATS_EDGX; + case 'H': + case 'h': + return Exchange.ISE_GEMINI; + case 'I': + case 'i': + return Exchange.ISE; + case 'J': + case 'j': + return Exchange.MERCURY; + case 'M': + case 'm': + return Exchange.MIAMI; + case 'N': + case 'n': + case 'P': + case 'p': + return Exchange.NYSE_ARCA; + case 'O': + case 'o': + return Exchange.MIAMI_PEARL; + case 'Q': + case 'q': + return Exchange.NASDAQ; + case 'S': + case 's': + return Exchange.MIAX_SAPPHIRE; + case 'T': + case 't': + return Exchange.NASDAQ_BX; + case 'U': + case 'u': + return Exchange.MEMX; + case 'W': + case 'w': + return Exchange.CBOE_C2; + case 'X': + case 'x': + return Exchange.PHLX; + case 'Z': + case 'z': + return Exchange.BATS_BZX; + default: + return Exchange.UNKNOWN; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Trade ParseTrade(ReadOnlySpan bytes) + { + Conditions conditions = new Conditions(); + bytes.Slice(61, 4).CopyTo(conditions); + + return new Trade(FormatContract(bytes.Slice(1, (int)bytes[0])), + ParseExchange((char)bytes[65]), + bytes[23], + bytes[24], + BitConverter.ToInt32(bytes.Slice(25, 4)), + BitConverter.ToUInt32(bytes.Slice(29, 4)), + BitConverter.ToUInt64(bytes.Slice(33, 8)), + BitConverter.ToUInt64(bytes.Slice(41, 8)), + conditions, + BitConverter.ToInt32(bytes.Slice(49, 4)), + BitConverter.ToInt32(bytes.Slice(53, 4)), + BitConverter.ToInt32(bytes.Slice(57, 4)) + ); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Quote ParseQuote(ReadOnlySpan bytes) + { + return new Quote(FormatContract(bytes.Slice(1, (int)bytes[0])), + bytes[23], + BitConverter.ToInt32(bytes.Slice(24, 4)), + BitConverter.ToUInt32(bytes.Slice(28, 4)), + BitConverter.ToInt32(bytes.Slice(32, 4)), + BitConverter.ToUInt32(bytes.Slice(36, 4)), + BitConverter.ToUInt64(bytes.Slice(40, 8)) + ); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Refresh ParseRefresh(ReadOnlySpan bytes) + { + return new Refresh(FormatContract(bytes.Slice(1, (int)bytes[0])), + bytes[23], + BitConverter.ToUInt32(bytes.Slice(24, 4)), + BitConverter.ToInt32(bytes.Slice(28, 4)), + BitConverter.ToInt32(bytes.Slice(32, 4)), + BitConverter.ToInt32(bytes.Slice(36, 4)), + BitConverter.ToInt32(bytes.Slice(40, 4)) + ); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private UnusualActivity ParseUnusualActivity(ReadOnlySpan bytes) + { + return new UnusualActivity(FormatContract(bytes.Slice(1, (int)bytes[0])), + (UAType)((int)bytes[22]), + (UASentiment)((int)bytes[23]), + bytes[24], + bytes[25], + BitConverter.ToUInt64(bytes.Slice(26, 8)), + BitConverter.ToUInt32(bytes.Slice(34, 4)), + BitConverter.ToInt32(bytes.Slice(38, 4)), + BitConverter.ToInt32(bytes.Slice(42, 4)), + BitConverter.ToInt32(bytes.Slice(46, 4)), + BitConverter.ToInt32(bytes.Slice(50, 4)), + BitConverter.ToUInt64(bytes.Slice(54, 8)) + ); + } + + protected override void HandleMessage(ReadOnlySpan bytes) + { + byte msgType = bytes[MessageTypeIndex]; + + if (msgType == 1u && _useOnQuote) + { + Quote quote = ParseQuote(bytes); + Interlocked.Increment(ref _dataQuoteCount); + try { _onQuote.Invoke(quote); } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while invoking user supplied OnQuote: {0}; {1}", new object[]{e.Message, e.StackTrace}); + } + } + if (msgType == 0u && _useOnTrade) + { + Trade trade = ParseTrade(bytes); + Interlocked.Increment(ref _dataTradeCount); + try { _onTrade.Invoke(trade); } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while invoking user supplied OnTrade: {0}; {1}", new object[]{e.Message, e.StackTrace}); + } + } + if (msgType == 2u && _useOnRefresh) + { + Refresh refresh = ParseRefresh(bytes); + Interlocked.Increment(ref _dataRefreshCount); + try { _onRefresh.Invoke(refresh); } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while invoking user supplied OnRefresh: {0}; {1}", new object[]{e.Message, e.StackTrace}); + } + } + if (msgType > 2u && _useOnUnusualActivity) + { + UnusualActivity unusualActivity = ParseUnusualActivity(bytes); + Interlocked.Increment(ref _dataUnusualActivityCount); + try { _onUnusualActivity.Invoke(unusualActivity); } + catch (Exception e) + { + LogMessage(LogLevel.ERROR, "Error while invoking user supplied OnUnusualActivity: {0}; {1}", new object[]{e.Message, e.StackTrace}); + } + } + } + + protected override byte[] MakeJoinMessage(string channel) + { + string symbol = GetSymbolFromChannel(channel); + bool tradesOnly = GetTradesOnlyFromChannel(channel); + byte mask = 0; + if (_useOnTrade) SetUsesTrade(ref mask); + if (_useOnQuote && !tradesOnly) SetUsesQuote(ref mask); + if (_useOnRefresh) SetUsesRefresh(ref mask); + if (_useOnUnusualActivity) SetUsesUA(ref mask); + switch (symbol) + { + case LobbyName: + { + byte[] message = new byte[11]; //1 + 1 + 9 + message[0] = Convert.ToByte(74); //type: join (74uy) or leave (76uy) + message[1] = mask; + Encoding.ASCII.GetBytes("$FIREHOSE").CopyTo(message, 2); + return message; + } + default: + { + string translatedSymbol = Config.TranslateContract(symbol); + byte[] message = new byte[2 + translatedSymbol.Length]; //1 + 1 + symbol.Length + message[0] = Convert.ToByte(74); //type: join (74uy) or leave (76uy) + message[1] = mask; + Encoding.ASCII.GetBytes(translatedSymbol).CopyTo(message, 2); + return message; + } + } + } + + protected override byte[] MakeLeaveMessage(string channel) + { + string symbol = GetSymbolFromChannel(channel); + bool tradesOnly = GetTradesOnlyFromChannel(channel); + switch (symbol) + { + case LobbyName: + { + byte[] message = new byte[10]; // 1 (type = join) + 9 (symbol = $FIREHOSE) + message[0] = Convert.ToByte(76); //type: join (74uy) or leave (76uy) + Encoding.ASCII.GetBytes("$FIREHOSE").CopyTo(message, 1); + return message; + } + default: + { + string translatedSymbol = Config.TranslateContract(symbol); + byte[] message = new byte[2 + translatedSymbol.Length]; //1 + symbol.Length + message[0] = Convert.ToByte(76); //type: join (74uy) or leave (76uy) + Encoding.ASCII.GetBytes(translatedSymbol).CopyTo(message, 2); + return message; + } + } + } + + private static void SetUsesTrade(ref byte bitMask) { bitMask = (byte)(bitMask | 0b1); } + private static void SetUsesQuote(ref byte bitMask) { bitMask = (byte)(bitMask | 0b10); } + private static void SetUsesRefresh(ref byte bitMask) { bitMask = (byte)(bitMask | 0b100); } + private static void SetUsesUA(ref byte bitMask) { bitMask = (byte)(bitMask | 0b1000); } + #endregion //Private Methods +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Provider.cs b/Intrinio.Realtime/Options/Provider.cs new file mode 100644 index 0000000..a813b77 --- /dev/null +++ b/Intrinio.Realtime/Options/Provider.cs @@ -0,0 +1,8 @@ +namespace Intrinio.Realtime.Options; + +public enum Provider +{ + NONE = 0, + OPRA = 1, + MANUAL = 2 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Quote.cs b/Intrinio.Realtime/Options/Quote.cs new file mode 100644 index 0000000..5f9f7cd --- /dev/null +++ b/Intrinio.Realtime/Options/Quote.cs @@ -0,0 +1,95 @@ +using System.Globalization; + +namespace Intrinio.Realtime.Options; + +using System; + +public struct Quote +{ + private readonly string _contract; + private readonly byte _priceType; + private readonly Int32 _askPrice; + private readonly UInt32 _askSize; + private readonly Int32 _bidPrice; + private readonly UInt32 _bidSize; + private readonly UInt64 _timeStamp; + + public string Contract { get { return _contract;} } + public double AskPrice { get { return (_askPrice == Int32.MaxValue) || (_askPrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_askPrice, _priceType);} } + public UInt32 AskSize { get { return _askSize;} } + public double BidPrice { get { return (_bidPrice == Int32.MaxValue) || (_bidPrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_bidPrice, _priceType);} } + public UInt32 BidSize { get { return _bidSize;} } + public double Timestamp { get { return Helpers.ScaleTimestampToSeconds(_timeStamp);} } + + /// + /// A 'Quote' is a unit of data representing a conflated market bid and/or ask event. + /// + /// The id of the option contract (e.g. AAPL__201016C00100000). + /// The scalar for the prices. + /// The dollar price of the last ask. + /// The number of contacts for the ask. + /// The dollars price of the last bid. + /// The number of contacts for the bid. + /// The time that the Quote was made (a unix timestamp representing the number of seconds (or better) since the unix epoch). + public Quote(string contract, byte priceType, Int32 askPrice, UInt32 askSize, Int32 bidPrice, UInt32 bidSize, UInt64 timeStamp) + { + _contract = contract; + _priceType = priceType; + _askPrice = askPrice; + _askSize = askSize; + _bidPrice = bidPrice; + _bidSize = bidSize; + _timeStamp = timeStamp; + } + + public override string ToString() + { + return $"Quote (Contract: {Contract}, AskPrice: {AskPrice.ToString("f3")}, AskSize: {AskSize.ToString()}, BidPrice: {BidPrice.ToString("f3")}, BidSize: {BidSize.ToString()}, Timestamp: {Timestamp.ToString("f6")})"; + } + + public string GetUnderlyingSymbol() + { + return Contract.Substring(0, 6).TrimEnd('_'); + } + + public DateTime GetExpirationDate() + { + return DateTime.ParseExact(Contract.Substring(6, 6), "yyMMdd", CultureInfo.InvariantCulture); + } + + public bool IsCall() + { + return Contract[12] == 'C'; + } + + public bool IsPut() + { + return Contract[12] == 'P'; + } + + public double GetStrikePrice() + { + const UInt32 zeroChar = (UInt32)'0'; + + UInt32 whole = ((UInt32)Contract[13] - zeroChar) * 10_000u + + ((UInt32)Contract[14] - zeroChar) * 1_000u + + ((UInt32)Contract[15] - zeroChar) * 100u + + ((UInt32)Contract[16] - zeroChar) * 10u + + ((UInt32)Contract[17] - zeroChar) * 1u; + + double part = Convert.ToDouble((UInt32) Contract[18] - zeroChar) * 0.1D + + Convert.ToDouble((UInt32) Contract[19] - zeroChar) * 0.01D + + Convert.ToDouble((UInt32) Contract[20] - zeroChar) * 0.001D; + + return Convert.ToDouble(whole) + part; + } + + public static Quote CreateUnitTestObject(string contract, double askPrice, UInt32 askSize, double bidPrice, UInt32 bidSize, UInt64 nanoSecondsSinceUnixEpoch) + { + byte priceType = (byte)4; + int unscaledAskPrice = Convert.ToInt32(askPrice * 10000.0); + int unscaledBidPrice = Convert.ToInt32(bidPrice * 10000.0); + Quote quote = new Quote(contract, priceType, unscaledAskPrice, askSize, unscaledBidPrice, bidSize, nanoSecondsSinceUnixEpoch); + return quote; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/QuoteCandleStick.cs b/Intrinio.Realtime/Options/QuoteCandleStick.cs new file mode 100644 index 0000000..5b5362b --- /dev/null +++ b/Intrinio.Realtime/Options/QuoteCandleStick.cs @@ -0,0 +1,167 @@ +using System.Globalization; + +namespace Intrinio.Realtime.Options; + +using System; + +public class QuoteCandleStick : CandleStick, IEquatable, IComparable, IComparable +{ + private readonly string _contract; + private readonly QuoteType _quoteType; + public string Contract + { + get { return _contract; } + } + public QuoteType QuoteType + { + get { return _quoteType; } + } + + public QuoteCandleStick(string contract, UInt32 volume, double price, QuoteType quoteType, double openTimestamp, double closeTimestamp, IntervalType interval, double quoteTime) + :base(volume, price, openTimestamp, closeTimestamp, interval, quoteTime) + { + _contract = contract; + _quoteType = quoteType; + } + + public QuoteCandleStick(string contract, UInt32 volume, double high, double low, double closePrice, double openPrice, QuoteType quoteType, double openTimestamp, double closeTimestamp, double firstTimestamp, double lastTimestamp, bool complete, double average, double change, IntervalType interval) + : base(volume, high, low, closePrice, openPrice, openTimestamp, closeTimestamp, firstTimestamp, lastTimestamp, complete, average, change, interval) + { + _contract = contract; + _quoteType = quoteType; + } + + public override bool Equals(object other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (other is QuoteCandleStick) + && (Contract.Equals(((QuoteCandleStick)other).Contract)) + && (Interval.Equals(((QuoteCandleStick)other).Interval)) + && (QuoteType.Equals(((QuoteCandleStick)other).QuoteType)) + && (OpenTimestamp.Equals(((QuoteCandleStick)other).OpenTimestamp)) + ); + } + + public override int GetHashCode() + { + return Contract.GetHashCode() ^ Interval.GetHashCode() ^ OpenTimestamp.GetHashCode() ^ QuoteType.GetHashCode(); + } + + public bool Equals(QuoteCandleStick other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (Contract.Equals(other.Contract)) + && (Interval.Equals(other.Interval)) + && (QuoteType.Equals(other.QuoteType)) + && (OpenTimestamp.Equals(other.OpenTimestamp)) + ); + } + + public int CompareTo(object other) + { + return Equals(other) switch + { + true => 0, + false => ReferenceEquals(other, null) switch + { + true => 1, + false => (other is QuoteCandleStick) switch + { + true => Contract.CompareTo(((QuoteCandleStick)other).Contract) switch + { + < 0 => -1, + > 0 => 1, + 0 => Interval.CompareTo(((QuoteCandleStick)other).Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => QuoteType.CompareTo(((QuoteCandleStick)other).QuoteType) switch + { + < 0 => -1, + > 0 => 1, + 0 => OpenTimestamp.CompareTo(((QuoteCandleStick)other).OpenTimestamp) + } + } + }, + false => 1 + } + } + }; + } + + public int CompareTo(QuoteCandleStick other) + { + return Equals(other) switch + { + true => 0, + false => Object.ReferenceEquals(other, null) switch + { + true => 1, + false => Contract.CompareTo(other.Contract) switch + { + < 0 => -1, + > 0 => 1, + 0 => Interval.CompareTo(other.Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => QuoteType.CompareTo(other.QuoteType) switch + { + < 0 => -1, + > 0 => 1, + 0 => OpenTimestamp.CompareTo(other.OpenTimestamp) + } + } + } + } + }; + } + + public override string ToString() + { + return $"QuoteCandleStick (Contract: {Contract}, QuoteType: {QuoteType.ToString()}, High: {High.ToString("f3")}, Low: {Low.ToString("f3")}, Close: {Close.ToString("f3")}, Open: {Open.ToString("f3")}, OpenTimestamp: {OpenTimestamp.ToString("f6")}, CloseTimestamp: {CloseTimestamp.ToString("f6")}, Change: {Change.ToString("f6")}, Complete: {Complete.ToString()})"; + } + + public string GetUnderlyingSymbol() + { + return Contract.Substring(0, 6).TrimEnd('_'); + } + + public DateTime GetExpirationDate() + { + return DateTime.ParseExact(Contract.Substring(6, 6), "yyMMdd", CultureInfo.InvariantCulture); + } + + public bool IsCall() + { + return Contract[12] == 'C'; + } + + public bool IsPut() + { + return Contract[12] == 'P'; + } + + public double GetStrikePrice() + { + const UInt32 zeroChar = (UInt32)'0'; + + UInt32 whole = ((UInt32)Contract[13] - zeroChar) * 10_000u + + ((UInt32)Contract[14] - zeroChar) * 1_000u + + ((UInt32)Contract[15] - zeroChar) * 100u + + ((UInt32)Contract[16] - zeroChar) * 10u + + ((UInt32)Contract[17] - zeroChar) * 1u; + + double part = Convert.ToDouble((UInt32) Contract[18] - zeroChar) * 0.1D + + Convert.ToDouble((UInt32) Contract[19] - zeroChar) * 0.01D + + Convert.ToDouble((UInt32) Contract[20] - zeroChar) * 0.001D; + + return Convert.ToDouble(whole) + part; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/QuoteType.cs b/Intrinio.Realtime/Options/QuoteType.cs new file mode 100644 index 0000000..f626773 --- /dev/null +++ b/Intrinio.Realtime/Options/QuoteType.cs @@ -0,0 +1,7 @@ +namespace Intrinio.Realtime.Options; + +public enum QuoteType +{ + Ask = 0, + Bid = 1 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Refresh.cs b/Intrinio.Realtime/Options/Refresh.cs new file mode 100644 index 0000000..8eb0d98 --- /dev/null +++ b/Intrinio.Realtime/Options/Refresh.cs @@ -0,0 +1,98 @@ +using System.Globalization; + +namespace Intrinio.Realtime.Options; + +using System; + +public struct Refresh +{ + private readonly string _contract; + private readonly byte _priceType; + private readonly UInt32 _openInterest; + private readonly int _openPrice; + private readonly int _closePrice; + private readonly int _highPrice; + private readonly int _lowPrice; + + public string Contract { get { return _contract; } } + public UInt32 OpenInterest { get { return _openInterest; } } + public double OpenPrice { get { return (_openPrice == Int32.MaxValue) || (_openPrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_openPrice, _priceType); } } + public double ClosePrice { get { return (_closePrice == Int32.MaxValue) || (_closePrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_closePrice, _priceType); } } + public double HighPrice { get { return (_highPrice == Int32.MaxValue) || (_highPrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_highPrice, _priceType); } } + public double LowPrice { get { return (_lowPrice == Int32.MaxValue) || (_lowPrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_lowPrice, _priceType); } } + + /// + /// A 'Refresh' is an event that periodically sends updated values for open interest and high/low/open/close. + /// + /// The id of the option contract (e.g. AAPL__201016C00100000). + /// The scalar for the price. + /// Number of total active contracts for this contract. + /// The opening price for this contract for the day. + /// The closing price for this contract for the day. + /// The running high price for this contract today. + /// The running low price for this contract today. + public Refresh(string contract, byte priceType, UInt32 openInterest, int openPrice, int closePrice, int highPrice, int lowPrice) + { + _contract = contract; + _priceType = priceType; + _openInterest = openInterest; + _openPrice = openPrice; + _closePrice = closePrice; + _highPrice = highPrice; + _lowPrice = lowPrice; + } + + public override string ToString() + { + return $"Refresh (Contract: {Contract}, OpenInterest: {OpenInterest.ToString()}, OpenPrice: {OpenPrice.ToString("f3")}, ClosePrice: {ClosePrice.ToString("f3")}, HighPrice: {HighPrice.ToString("f3")}, LowPrice: {LowPrice.ToString("f3")})"; + } + + public string GetUnderlyingSymbol() + { + return Contract.Substring(0, 6).TrimEnd('_'); + } + + public DateTime GetExpirationDate() + { + return DateTime.ParseExact(Contract.Substring(6, 6), "yyMMdd", CultureInfo.InvariantCulture); + } + + public bool IsCall() + { + return Contract[12] == 'C'; + } + + public bool IsPut() + { + return Contract[12] == 'P'; + } + + public double GetStrikePrice() + { + const UInt32 zeroChar = (UInt32)'0'; + + UInt32 whole = ((UInt32)Contract[13] - zeroChar) * 10_000u + + ((UInt32)Contract[14] - zeroChar) * 1_000u + + ((UInt32)Contract[15] - zeroChar) * 100u + + ((UInt32)Contract[16] - zeroChar) * 10u + + ((UInt32)Contract[17] - zeroChar) * 1u; + + double part = Convert.ToDouble((UInt32) Contract[18] - zeroChar) * 0.1D + + Convert.ToDouble((UInt32) Contract[19] - zeroChar) * 0.01D + + Convert.ToDouble((UInt32) Contract[20] - zeroChar) * 0.001D; + + return Convert.ToDouble(whole) + part; + } + + public static Refresh CreateUnitTestObject(string contract, UInt32 openInterest, double openPrice, double closePrice, double highPrice, double lowPrice) + { + byte priceType = (byte)4; + int unscaledOpenPrice = Convert.ToInt32(openPrice * 10000.0); + int unscaledClosePrice = Convert.ToInt32(closePrice * 10000.0); + int unscaledHighPrice = Convert.ToInt32(highPrice * 10000.0); + int unscaledLowPrice = Convert.ToInt32(lowPrice * 10000.0); + + Refresh refresh = new Refresh(contract, priceType, openInterest, unscaledOpenPrice, unscaledClosePrice, unscaledHighPrice, unscaledLowPrice); + return refresh; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/ReplayClient.cs b/Intrinio.Realtime/Options/ReplayClient.cs new file mode 100644 index 0000000..dc1a5b7 --- /dev/null +++ b/Intrinio.Realtime/Options/ReplayClient.cs @@ -0,0 +1,705 @@ +// using System.Linq; +// +// namespace Intrinio.Realtime.Options; +// +// using Intrinio.SDK.Model; +// using System; +// using System.IO; +// using System.Net.Http; +// using System.Text; +// using System.Collections.Concurrent; +// using System.Collections.Generic; +// using System.Threading; +// using System.Threading.Tasks; +// +// public class ReplayClient : IOptionsWebSocketClient +// { +// #region Data Members +// private const string LobbyName = "lobby"; +// public Action OnTrade { get; set; } +// public Action OnQuote { get; set; } +// public Action OnRefresh { get; set; } +// public Action OnUnusualActivity { get; set; } +// private readonly Config _config; +// private readonly DateTime _date; +// private readonly bool _withSimulatedDelay; +// private readonly bool _deleteFileWhenDone; +// private readonly bool _writeToCsv; +// private readonly string _csvFilePath; +// private ulong _dataMsgCount; +// private ulong _dataEventCount; +// private ulong _dataTradeCount; +// private ulong _dataQuoteCount; +// private ulong _dataRefreshCount; +// private ulong _dataUnusualActivityCount; +// private ulong _textMsgCount; +// private readonly HashSet _channels; +// private readonly CancellationTokenSource _ctSource; +// private readonly ConcurrentQueue _data; +// private bool _useOnTrade { get {return !(ReferenceEquals(OnTrade, null));} } +// private bool _useOnQuote { get {return !(ReferenceEquals(OnQuote, null));} } +// private bool _useOnRefresh { get {return !(ReferenceEquals(OnRefresh, null));} } +// private bool _useOnUnusualActivity { get {return !(ReferenceEquals(OnUnusualActivity, null));} } +// +// private readonly string _logPrefix; +// private readonly object _csvLock; +// private readonly Thread[] _threads; +// private readonly Thread _replayThread; +// public UInt64 TradeCount { get { return Interlocked.Read(ref _dataTradeCount); } } +// public UInt64 QuoteCount { get { return Interlocked.Read(ref _dataQuoteCount); } } +// public UInt64 RefreshCount { get { return Interlocked.Read(ref _dataRefreshCount); } } +// public UInt64 UnusualActivityCount { get { return Interlocked.Read(ref _dataUnusualActivityCount); } } +// #endregion //Data Members +// +// #region Constructors +// public ReplayClient(Action onTrade, Action onQuote, Config config, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) +// { +// this.OnTrade = onTrade; +// this.OnQuote = onQuote; +// this._config = config; +// this._date = date; +// this._withSimulatedDelay = withSimulatedDelay; +// this._deleteFileWhenDone = deleteFileWhenDone; +// this._writeToCsv = writeToCsv; +// this._csvFilePath = csvFilePath; +// +// _dataMsgCount = 0UL; +// _dataEventCount = 0UL; +// _dataTradeCount = 0UL; +// _dataQuoteCount = 0UL; +// _textMsgCount = 0UL; +// _channels = new HashSet(); +// _ctSource = new CancellationTokenSource(); +// _data = new ConcurrentQueue(); +// +// _logPrefix = _logPrefix = String.Format("{0}: ", config.Provider.ToString()); +// _csvLock = new Object(); +// _threads = new Thread[config.NumThreads]; +// for (int i = 0; i < _threads.Length; i++) +// _threads[i] = new Thread(ThreadFn); +// // _replayThread = new Thread(ReplayThreadFn); +// +// config.Validate(); +// } +// +// public ReplayClient(Action onTrade, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) : this(onTrade, null, Config.LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) +// { +// +// } +// +// public ReplayClient(Action onQuote, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) : this(null, onQuote, Config.LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) +// { +// +// } +// +// public ReplayClient(Action onTrade, Action onQuote, DateTime date, bool withSimulatedDelay, bool deleteFileWhenDone, bool writeToCsv, string csvFilePath) : this(onTrade, onQuote, Config.LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) +// { +// +// } +// #endregion //Constructors +// +// #region Public Methods +// +// public Task Join() +// { +// HashSet symbolsToAdd = _config.Symbols.Select(s => new Channel(s, _config.TradesOnly)).ToHashSet(); +// symbolsToAdd.ExceptWith(_channels); +// +// foreach (Channel channel in symbolsToAdd) +// Join(channel.ticker, channel.tradesOnly); +// +// return Task.CompletedTask; +// } +// +// public Task Join(string symbol, bool? tradesOnly) +// { +// bool t = tradesOnly.HasValue +// ? tradesOnly.Value || _config.TradesOnly +// : _config.TradesOnly; +// if (!_channels.Contains(new Channel(symbol, t))) +// Join(symbol, t); +// +// return Task.CompletedTask; +// } +// +// public async Task JoinLobby(bool? tradesOnly) +// { +// await Join(LobbyName, tradesOnly); +// } +// +// public Task Join(string[] symbols, bool? tradesOnly) +// { +// bool t = tradesOnly.HasValue +// ? tradesOnly.Value || _config.TradesOnly +// : _config.TradesOnly; +// HashSet symbolsToAdd = symbols.Select(s => new Channel(s, t)).ToHashSet(); +// symbolsToAdd.ExceptWith(_channels); +// foreach (Channel channel in symbolsToAdd) +// Join(channel.ticker, channel.tradesOnly); +// return Task.CompletedTask; +// } +// +// public Task Leave() +// { +// foreach (Channel channel in _channels) +// Leave(channel.ticker, channel.tradesOnly); +// return Task.CompletedTask; +// } +// +// public Task Leave(string symbol) +// { +// IEnumerable matchingChannels = _channels.Where(c => c.ticker == symbol); +// foreach (Channel channel in matchingChannels) +// Leave(channel.ticker, channel.tradesOnly); +// return Task.CompletedTask; +// } +// +// public async Task LeaveLobby() +// { +// await Leave(LobbyName); +// } +// +// public Task Leave(string[] symbols) +// { +// HashSet _symbols = new HashSet(symbols); +// IEnumerable matchingChannels = _channels.Where(c => _symbols.Contains(c.ticker)); +// foreach (Channel channel in matchingChannels) +// Leave(channel.ticker, channel.tradesOnly); +// return Task.CompletedTask; +// } +// +// public Task Start() +// { +// foreach (Thread thread in _threads) +// thread.Start(); +// if (_writeToCsv) +// WriteHeaderRow(); +// _replayThread.Start(); +// +// return Task.CompletedTask; +// } +// +// public Task Stop() +// { +// foreach (Channel channel in _channels) +// Leave(channel.ticker, channel.tradesOnly); +// +// _ctSource.Cancel(); +// LogMessage(LogLevel.INFORMATION, "Websocket - Closing..."); +// +// foreach (Thread thread in _threads) +// thread.Join(); +// +// _replayThread.Join(); +// +// LogMessage(LogLevel.INFORMATION, "Stopped"); +// return Task.CompletedTask; +// } +// +// public ClientStats GetStats() +// { +// return new ClientStats( +// Interlocked.Read(ref _dataMsgCount), +// Interlocked.Read(ref _textMsgCount), +// _data.Count, +// Interlocked.Read(ref _dataEventCount), +// Int32.MaxValue, +// 0, +// Int32.MaxValue, +// 0, +// 0 +// ); +// } +// +// [Serilog.Core.MessageTemplateFormatMethod("messageTemplate")] +// public void LogMessage(LogLevel logLevel, string messageTemplate, params object[] propertyValues) +// { +// switch (logLevel) +// { +// case LogLevel.DEBUG: +// Serilog.Log.Debug(_logPrefix + messageTemplate, propertyValues); +// break; +// case LogLevel.INFORMATION: +// Serilog.Log.Information(_logPrefix + messageTemplate, propertyValues); +// break; +// case LogLevel.WARNING: +// Serilog.Log.Warning(_logPrefix + messageTemplate, propertyValues); +// break; +// case LogLevel.ERROR: +// Serilog.Log.Error(_logPrefix + messageTemplate, propertyValues); +// break; +// default: +// throw new ArgumentException("LogLevel not specified!"); +// break; +// } +// } +// #endregion //Public Methods +// +// #region Private Methods +// private DateTime ParseTimeReceived(ReadOnlySpan bytes) +// { +// return DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes) / 100UL)); +// } +// +// private Trade ParseTrade(ReadOnlySpan bytes) +// { +// throw new NotImplementedException(); +// // int symbolLength = Convert.ToInt32(bytes[2]); +// // int conditionLength = Convert.ToInt32(bytes[26 + symbolLength]); +// // Trade trade = new Trade( +// // Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)), +// // Convert.ToDouble(BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4))), +// // BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)), +// // BitConverter.ToUInt32(bytes.Slice(22 + symbolLength, 4)), +// // DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)), +// // (SubProvider)Convert.ToInt32(bytes[3 + symbolLength]), +// // BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)), +// // conditionLength > 0 ? Encoding.ASCII.GetString(bytes.Slice(27 + symbolLength, conditionLength)) : String.Empty +// // ); +// // +// // return trade; +// } +// +// private Quote ParseQuote(ReadOnlySpan bytes) +// { +// throw new NotImplementedException(); +// // int symbolLength = Convert.ToInt32(bytes[2]); +// // int conditionLength = Convert.ToInt32(bytes[22 + symbolLength]); +// // +// // Quote quote = new Quote( +// // (QuoteType)(Convert.ToInt32(bytes[0])), +// // Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)), +// // (Convert.ToDouble(BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4)))), +// // BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)), +// // DateTime.UnixEpoch + TimeSpan.FromTicks(Convert.ToInt64(BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)), +// // (SubProvider)(Convert.ToInt32(bytes[3 + symbolLength])), +// // BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)), +// // conditionLength > 0 ? Encoding.ASCII.GetString(bytes.Slice(23 + symbolLength, conditionLength)) : String.Empty +// // ); +// // +// // return quote; +// } +// +// private void WriteRowToOpenCsvWithoutLock(IEnumerable row) +// { +// bool first = true; +// using (FileStream fs = new FileStream(_csvFilePath, FileMode.Append)) +// using (TextWriter tw = new StreamWriter(fs)) +// { +// foreach (string s in row) +// { +// if (!first) +// tw.Write(","); +// else +// first = false; +// tw.Write($"\"{s}\""); +// } +// +// tw.WriteLine(); +// } +// } +// +// private void WriteRowToOpenCsvWithLock(IEnumerable row) +// { +// lock (_csvLock) +// { +// WriteRowToOpenCsvWithoutLock(row); +// } +// } +// +// private string DoubleRoundSecRule612(double value) +// { +// if (value >= 1.0D) +// return value.ToString("0.00"); +// +// return value.ToString("0.0000"); +// } +// +// private IEnumerable MapTradeToRow(Trade trade) +// { +// throw new NotImplementedException(); +// // yield return MessageType.Trade.ToString(); +// // yield return trade.Symbol; +// // yield return DoubleRoundSecRule612(trade.Price); +// // yield return trade.Size.ToString(); +// // yield return trade.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffK"); +// // yield return trade.SubProvider.ToString(); +// // yield return trade.MarketCenter.ToString(); +// // yield return trade.Condition; +// // yield return trade.TotalVolume.ToString(); +// } +// +// private void WriteTradeToCsv(Trade trade) +// { +// WriteRowToOpenCsvWithLock(MapTradeToRow(trade)); +// } +// +// private IEnumerable MapQuoteToRow(Quote quote) +// { +// throw new NotImplementedException(); +// // yield return quote.Type.ToString(); +// // yield return quote.Symbol; +// // yield return DoubleRoundSecRule612(quote.Price); +// // yield return quote.Size.ToString(); +// // yield return quote.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffK"); +// // yield return quote.SubProvider.ToString(); +// // yield return quote.MarketCenter.ToString(); +// // yield return quote.Condition; +// } +// +// private void WriteQuoteToCsv(Quote quote) +// { +// WriteRowToOpenCsvWithLock(MapQuoteToRow(quote)); +// } +// +// private void WriteHeaderRow() +// { +// WriteRowToOpenCsvWithLock(new string[]{"Type", "Symbol", "Price", "Size", "Timestamp", "SubProvider", "MarketCenter", "Condition", "TotalVolume"}); +// } +// +// private void ThreadFn() +// { +// CancellationToken ct = _ctSource.Token; +// while (!ct.IsCancellationRequested) +// { +// try +// { +// if (_data.TryDequeue(out Tick datum)) +// { +// if (datum.IsTrade()) +// { +// if (_useOnTrade) +// { +// Interlocked.Increment(ref _dataTradeCount); +// OnTrade.Invoke(datum.Trade); +// } +// } +// else +// { +// if (_useOnQuote) +// { +// Interlocked.Increment(ref _dataQuoteCount); +// OnQuote.Invoke(datum.Quote); +// } +// } +// } +// else +// Thread.Sleep(1); +// } +// catch (OperationCanceledException) +// { +// } +// catch (Exception exn) +// { +// LogMessage(LogLevel.ERROR, "Error parsing message: {0}; {1}", exn.Message, exn.StackTrace); +// } +// } +// } +// +// // /// +// // /// The results of this should be streamed and not ToList-ed. +// // /// +// // /// +// // /// +// // /// +// // private IEnumerable ReplayTickFileWithoutDelay(string fullFilePath, int byteBufferSize, CancellationToken ct) +// // { +// // if (File.Exists(fullFilePath)) +// // { +// // using (FileStream fRead = new FileStream(fullFilePath, FileMode.Open, FileAccess.Read, FileShare.None)) +// // { +// // if (fRead.CanRead) +// // { +// // int readResult = fRead.ReadByte(); //This is message type +// // +// // while (readResult != -1) +// // { +// // if (!ct.IsCancellationRequested) +// // { +// // byte[] eventBuffer = new byte[byteBufferSize]; +// // byte[] timeReceivedBuffer = new byte[8]; +// // ReadOnlySpan eventSpanBuffer = new ReadOnlySpan(eventBuffer); +// // ReadOnlySpan timeReceivedSpanBuffer = new ReadOnlySpan(timeReceivedBuffer); +// // eventBuffer[0] = (byte)readResult; //This is message type +// // eventBuffer[1] = (byte)(fRead.ReadByte()); //This is message length, including this and the previous byte. +// // int bytesRead = fRead.Read(eventBuffer, 2, (System.Convert.ToInt32(eventBuffer[1]) - 2)); //read the rest of the message +// // int timeBytesRead = fRead.Read(timeReceivedBuffer, 0, 8); //get the time received +// // DateTime timeReceived = ParseTimeReceived(timeReceivedSpanBuffer); +// // +// // switch ((MessageType)(Convert.ToInt32(eventBuffer[0]))) +// // { +// // case MessageType.Trade: +// // Trade trade = ParseTrade(eventSpanBuffer); +// // if (_channels.Contains(new Channel(LobbyName, true)) +// // || _channels.Contains(new Channel(LobbyName, false)) +// // || _channels.Contains(new Channel(trade.Contract, true)) +// // || _channels.Contains(new Channel(trade.Contract, false))) +// // { +// // if (_writeToCsv) +// // WriteTradeToCsv(trade); +// // yield return new Tick(timeReceived, trade, null); +// // } +// // break; +// // case MessageType.Ask: +// // case MessageType.Bid: +// // Quote quote = ParseQuote(eventSpanBuffer); +// // if (_channels.Contains (new Channel(LobbyName, false)) || _channels.Contains (new Channel(quote.Contract, false))) +// // { +// // if (_writeToCsv) +// // WriteQuoteToCsv(quote); +// // yield return new Tick(timeReceived, null, quote); +// // } +// // break; +// // default: +// // LogMessage(LogLevel.ERROR, "Invalid MessageType: {0}", eventBuffer[0]); +// // break; +// // } +// // +// // //Set up the next iteration +// // readResult = fRead.ReadByte(); +// // } +// // else +// // readResult = -1; +// // } +// // } +// // else +// // throw new FileLoadException("Unable to read replay file."); +// // } +// // } +// // else +// // { +// // yield break; +// // } +// // } +// +// // /// +// // /// The results of this should be streamed and not ToList-ed. +// // /// +// // /// +// // /// +// // /// returns +// // private IEnumerable ReplayTickFileWithDelay(string fullFilePath, int byteBufferSize, CancellationToken ct) +// // { +// // long start = DateTime.UtcNow.Ticks; +// // long offset = 0L; +// // foreach (Tick tick in ReplayTickFileWithoutDelay(fullFilePath, byteBufferSize, ct)) +// // { +// // if (offset == 0L) +// // offset = start - tick.TimeReceived().Ticks; +// // +// // if (!ct.IsCancellationRequested) +// // { +// // SpinWait.SpinUntil(() => (tick.TimeReceived().Ticks + offset) <= DateTime.UtcNow.Ticks); +// // yield return tick; +// // } +// // } +// // } +// +// // private string MapSubProviderToApiValue(SubProvider subProvider) +// // { +// // switch (subProvider) +// // { +// // case SubProvider.IEX: return "iex"; +// // case SubProvider.UTP: return "utp_delayed"; +// // case SubProvider.CTA_A: return "cta_a_delayed"; +// // case SubProvider.CTA_B: return "cta_b_delayed"; +// // case SubProvider.OTC: return "otc_delayed"; +// // case SubProvider.NASDAQ_BASIC: return "nasdaq_basic"; +// // default: return "iex"; +// // } +// // } +// +// // private SubProvider[] MapProviderToSubProviders(Intrinio.Realtime.Equities.Provider provider) +// // { +// // switch (provider) +// // { +// // case Provider.NONE: return Array.Empty(); +// // case Provider.MANUAL: return Array.Empty(); +// // case Provider.REALTIME: return new SubProvider[]{SubProvider.IEX}; +// // case Provider.DELAYED_SIP: return new SubProvider[]{SubProvider.UTP, SubProvider.CTA_A, SubProvider.CTA_B, SubProvider.OTC}; +// // case Provider.NASDAQ_BASIC: return new SubProvider[]{SubProvider.NASDAQ_BASIC}; +// // default: return new SubProvider[0]; +// // } +// // } +// +// // private string FetchReplayFile(SubProvider subProvider) +// // { +// // Intrinio.SDK.Api.SecurityApi api = new Intrinio.SDK.Api.SecurityApi(); +// // +// // if (!api.Configuration.ApiKey.ContainsKey("api_key")) +// // api.Configuration.ApiKey.Add("api_key", _config.ApiKey); +// // +// // try +// // { +// // SecurityReplayFileResult result = api.GetSecurityReplayFile(MapSubProviderToApiValue(subProvider), _date); +// // string decodedUrl = result.Url.Replace(@"\u0026", "&"); +// // string tempDir = System.IO.Path.GetTempPath(); +// // string fileName = Path.Combine(tempDir, result.Name); +// // +// // using (FileStream outputFile = new FileStream(fileName,System.IO.FileMode.Create)) +// // using (HttpClient httpClient = new HttpClient()) +// // { +// // httpClient.Timeout = TimeSpan.FromHours(1); +// // httpClient.BaseAddress = new Uri(decodedUrl); +// // using (HttpResponseMessage response = httpClient.GetAsync(decodedUrl, HttpCompletionOption.ResponseHeadersRead).Result) +// // using (Stream streamToReadFrom = response.Content.ReadAsStreamAsync().Result) +// // { +// // streamToReadFrom.CopyTo(outputFile); +// // } +// // } +// // +// // return fileName; +// // } +// // catch (Exception e) +// // { +// // LogMessage(LogLevel.ERROR, "Error while fetching {0} file: {1}", subProvider.ToString(), e.Message); +// // return null; +// // } +// // } +// +// private void FillNextTicks(IEnumerator[] enumerators, Tick[] nextTicks) +// { +// for (int i = 0; i < nextTicks.Length; i++) +// if (nextTicks[i] == null && enumerators[i].MoveNext()) +// nextTicks[i] = enumerators[i].Current; +// } +// +// private Tick PullNextTick(Tick[] nextTicks) +// { +// int pullIndex = 0; +// DateTime t = DateTime.MaxValue; +// for (int i = 0; i < nextTicks.Length; i++) +// { +// if (nextTicks[i] != null && nextTicks[i].TimeReceived() < t) +// { +// pullIndex = i; +// t = nextTicks[i].TimeReceived(); +// } +// } +// +// Tick pulledTick = nextTicks[pullIndex]; +// nextTicks[pullIndex] = null; +// return pulledTick; +// } +// +// private bool HasAnyValue(Tick[] nextTicks) +// { +// bool hasValue = false; +// +// for (int i = 0; i < nextTicks.Length; i++) +// if (nextTicks[i] != null) +// hasValue = true; +// +// return hasValue; +// } +// +// private IEnumerable ReplayFileGroupWithoutDelay(IEnumerable[] tickGroup, CancellationToken ct) +// { +// Tick[] nextTicks = new Tick[tickGroup.Length]; +// IEnumerator[] enumerators = new IEnumerator[tickGroup.Length]; +// for (int i = 0; i < tickGroup.Length; i++) +// { +// enumerators[i] = tickGroup[i].GetEnumerator(); +// } +// +// FillNextTicks(enumerators, nextTicks); +// while (HasAnyValue(nextTicks)) +// { +// Tick nextTick = PullNextTick(nextTicks); +// if (nextTick != null) +// yield return nextTick; +// +// FillNextTicks(enumerators, nextTicks); +// } +// } +// +// private IEnumerable ReplayFileGroupWithDelay(IEnumerable[] tickGroup, CancellationToken ct) +// { +// Int64 start = DateTime.UtcNow.Ticks; +// Int64 offset = 0L; +// +// foreach (Tick tick in ReplayFileGroupWithoutDelay(tickGroup, ct)) +// { +// if (offset == 0L) +// { +// offset = start - tick.TimeReceived().Ticks; +// } +// +// if (!ct.IsCancellationRequested) +// { +// System.Threading.SpinWait.SpinUntil(() => (tick.TimeReceived().Ticks + offset) <= DateTime.UtcNow.Ticks); +// yield return tick; +// } +// } +// } +// +// // private void ReplayThreadFn() +// // { +// // CancellationToken ct = _ctSource.Token; +// // SubProvider[] subProviders = MapProviderToSubProviders(_config.Provider); +// // string[] replayFiles = new string[subProviders.Length]; +// // IEnumerable[] allTicks = new IEnumerable[subProviders.Length]; +// // +// // try +// // { +// // for (int i = 0; i < subProviders.Length; i++) +// // { +// // LogMessage(LogLevel.INFORMATION, "Downloading Replay file for {0} on {1}...", subProviders[i].ToString(), _date.Date.ToString()); +// // replayFiles[i] = FetchReplayFile(subProviders[i]); +// // LogMessage(LogLevel.INFORMATION, "Downloaded Replay file to: {0}", replayFiles[i]); +// // allTicks[i] = ReplayTickFileWithoutDelay(replayFiles[i], 100, ct); +// // } +// // +// // IEnumerable aggregatedTicks = _withSimulatedDelay +// // ? ReplayFileGroupWithDelay(allTicks, ct) +// // : ReplayFileGroupWithoutDelay(allTicks, ct); +// // +// // foreach (Tick tick in aggregatedTicks) +// // { +// // if (!ct.IsCancellationRequested) +// // { +// // Interlocked.Increment(ref _dataEventCount); +// // Interlocked.Increment(ref _dataMsgCount); +// // _data.Enqueue(tick); +// // } +// // } +// // } +// // catch (Exception e) +// // { +// // LogMessage(LogLevel.ERROR, "Error while replaying file: {0}", e.Message); +// // } +// // +// // if (_deleteFileWhenDone) +// // { +// // foreach (string deleteFilePath in replayFiles) +// // { +// // if (File.Exists(deleteFilePath)) +// // { +// // LogMessage(LogLevel.INFORMATION, "Deleting Replay file: {0}", deleteFilePath); +// // File.Delete(deleteFilePath); +// // } +// // } +// // } +// // } +// +// private void Join(string symbol, bool tradesOnly) +// { +// string lastOnly = tradesOnly ? "true" : "false"; +// if (_channels.Add(new (symbol, tradesOnly))) +// { +// LogMessage(LogLevel.INFORMATION, "Websocket - Joining channel: {0} (trades only = {1})", symbol, lastOnly); +// } +// } +// +// private void Leave(string symbol, bool tradesOnly) +// { +// string lastOnly = tradesOnly ? "true" : "false"; +// if (_channels.Remove(new (symbol, tradesOnly))) +// { +// LogMessage(LogLevel.INFORMATION, "Websocket - Leaving channel: {0} (trades only = {1})", symbol, lastOnly); +// } +// } +// #endregion //Private Methods +// +// private record Channel(string ticker, bool tradesOnly); +// } \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Tick.cs b/Intrinio.Realtime/Options/Tick.cs new file mode 100644 index 0000000..756e40c --- /dev/null +++ b/Intrinio.Realtime/Options/Tick.cs @@ -0,0 +1,241 @@ +using System.Runtime.CompilerServices; + +namespace Intrinio.Realtime.Options; + +using System; +using System.Text; + +public class Tick +{ + private readonly DateTime _timeReceived; + private readonly Trade? _trade; + private readonly Quote? _quote; + private readonly Refresh? _refresh; + private readonly UnusualActivity? _unusualActivity; + + public Tick(DateTime timeReceived, Trade trade, Quote quote, Refresh refresh, UnusualActivity unusualActivity) + { + _timeReceived = timeReceived; + _trade = trade; + _quote = quote; + _refresh = refresh; + _unusualActivity = unusualActivity; + } + + public static byte[] GetTradeBytes(Trade trade) + { + byte[] contractBytes = Encoding.ASCII.GetBytes(trade.Contract); + byte contractLength = System.Convert.ToByte(contractBytes.Length); + int contractLengthInt32 = System.Convert.ToInt32(contractLength); + char exchangeChar = (char)trade.Exchange; + byte exchangeByte = (byte) exchangeChar; + byte[] priceBytes = BitConverter.GetBytes(trade.Price); // 8 byte float + byte[] sizeBytes = BitConverter.GetBytes(trade.Size); // 4 byte uint32 + byte[] timestampBytes = BitConverter.GetBytes(trade.Timestamp); // 8 byte float + byte[] totalVolumeBytes = BitConverter.GetBytes(trade.TotalVolume); // 8 byte uint64 + Conditions qualifiers = trade.Qualifiers; + byte[] askPriceAtExecutionBytes = BitConverter.GetBytes(trade.AskPriceAtExecution); // 8 byte float + byte[] bidPriceAtExecutionBytes = BitConverter.GetBytes(trade.BidPriceAtExecution); // 8 byte float + byte[] underlyingPriceAtExecutionBytes = BitConverter.GetBytes(trade.UnderlyingPriceAtExecution); // 8 byte float + + // byte 0 | type | byte + // byte 1 | messageLength (includes bytes 0 and 1) | byte + // byte 2 | contractLength | byte + // bytes [3...] | contract | string (ascii) + // next byte | exchange | char + // next 8 bytes | price | float64 + // next 4 bytes | size | uint32 + // next 8 bytes | timestamp | float64 + // next 8 bytes | totalvolume | uint64 + // next 4 bytes | qualifiers | 4 byte struct tuple + // next 8 bytes | askpriceatexecution | float64 + // next 8 bytes | bidpriceatexecution | float64 + // next 8 bytes | underlyingpriceatexecution | float64 + + byte messageLength = (byte)(60u + contractLength); + + byte[] bytes = new byte[System.Convert.ToInt32(messageLength)]; + bytes[0] = System.Convert.ToByte((int)(Options.MessageType.Trade)); + bytes[1] = messageLength; + bytes[2] = contractLength; + Array.Copy(contractBytes, 0, bytes, 3, contractLengthInt32); + bytes[3 + contractLengthInt32] = exchangeByte; + Array.Copy(priceBytes, 0, bytes, 4 + contractLengthInt32, priceBytes.Length); + Array.Copy(sizeBytes, 0, bytes, 12 + contractLengthInt32, sizeBytes.Length); + Array.Copy(timestampBytes, 0, bytes, 16 + contractLengthInt32, timestampBytes.Length); + Array.Copy(totalVolumeBytes, 0, bytes, 24 + contractLengthInt32, totalVolumeBytes.Length); + bytes[32 + contractLengthInt32] = qualifiers[0]; + bytes[33 + contractLengthInt32] = qualifiers[1]; + bytes[34 + contractLengthInt32] = qualifiers[2]; + bytes[35 + contractLengthInt32] = qualifiers[3]; + Array.Copy(askPriceAtExecutionBytes, 0, bytes, 36 + contractLengthInt32, askPriceAtExecutionBytes.Length); + Array.Copy(bidPriceAtExecutionBytes, 0, bytes, 44 + contractLengthInt32, bidPriceAtExecutionBytes.Length); + Array.Copy(underlyingPriceAtExecutionBytes, 0, bytes, 52 + contractLengthInt32, underlyingPriceAtExecutionBytes.Length); + + return bytes; + } + + public static byte[] GetQuoteBytes(Quote quote) + { + byte[] contractBytes = Encoding.ASCII.GetBytes(quote.Contract); + byte contractLength = System.Convert.ToByte(contractBytes.Length); + int contractLengthInt32 = System.Convert.ToInt32(contractLength); + byte[] askPriceBytes = BitConverter.GetBytes(quote.AskPrice); // 8 byte float + byte[] askSizeBytes = BitConverter.GetBytes(quote.AskSize); // 4 byte uint32 + byte[] bidPriceBytes = BitConverter.GetBytes(quote.BidPrice); // 8 byte float + byte[] bidSizeBytes = BitConverter.GetBytes(quote.BidSize); // 4 byte uint32 + byte[] timestampBytes= BitConverter.GetBytes(quote.Timestamp); // 8 byte float + + // byte 0 | type | byte + // byte 1 | messageLength (includes bytes 0 and 1) | byte + // byte 2 | contractLength | byte + // bytes [3...] | contract | string (ascii) + // next 8 bytes | askPrice | float64 + // next 4 bytes | askSize | uint32 + // next 8 bytes | bidPrice | float64 + // next 4 bytes | bidSize | uint32 + // next 8 bytes | timestamp | float64 + + byte messageLength = (byte)(35u + contractLength); + + byte[] bytes = new byte[System.Convert.ToInt32(messageLength)]; + bytes[0] = System.Convert.ToByte((int)(Options.MessageType.Quote)); + bytes[1] = messageLength; + bytes[2] = contractLength; + Array.Copy(contractBytes, 0, bytes, 3, contractLengthInt32); + Array.Copy(askPriceBytes, 0, bytes, 3 + contractLengthInt32, askPriceBytes.Length); + Array.Copy(askSizeBytes, 0, bytes, 11 + contractLengthInt32, askSizeBytes.Length); + Array.Copy(bidPriceBytes, 0, bytes, 15 + contractLengthInt32, bidPriceBytes.Length); + Array.Copy(bidSizeBytes, 0, bytes, 23 + contractLengthInt32, bidSizeBytes.Length); + Array.Copy(timestampBytes, 0, bytes, 27 + contractLengthInt32, timestampBytes.Length); + + return bytes; + } + + public static byte[] GetRefreshBytes(Refresh refresh) + { + byte[] contractBytes = Encoding.ASCII.GetBytes(refresh.Contract); + byte contractLength = System.Convert.ToByte(contractBytes.Length); + int contractLengthInt32 = System.Convert.ToInt32(contractLength); + byte[] openInterestBytes = BitConverter.GetBytes(refresh.OpenInterest); // 4 byte uint32 + byte[] openPriceBytes = BitConverter.GetBytes(refresh.OpenPrice); // 8 byte float + byte[] closePriceBytes = BitConverter.GetBytes(refresh.ClosePrice); // 8 byte float + byte[] highPriceBytes = BitConverter.GetBytes(refresh.HighPrice); // 8 byte float + byte[] lowPriceBytes = BitConverter.GetBytes(refresh.LowPrice); // 8 byte float + + // byte 0 | type | byte + // byte 1 | messageLength (includes bytes 0 and 1) | byte + // byte 2 | contractLength | byte + // bytes [3...] | contract | string (ascii) + // next 4 bytes | openInterest | uint32 + // next 8 bytes | openPrice | float64 + // next 8 bytes | closePrice | float64 + // next 8 bytes | highPrice | float64 + // next 8 bytes | lowPrice | float64 + + byte messageLength = (byte)(39u + contractLength); + + byte[] bytes = new byte[System.Convert.ToInt32(messageLength)]; + bytes[0] = System.Convert.ToByte((int)(Options.MessageType.Refresh)); + bytes[1] = messageLength; + bytes[2] = contractLength; + Array.Copy(contractBytes, 0, bytes, 3, contractLengthInt32); + Array.Copy(openInterestBytes, 0, bytes, 3 + contractLengthInt32, openInterestBytes.Length); + Array.Copy(openPriceBytes, 0, bytes, 7 + contractLengthInt32, openPriceBytes.Length); + Array.Copy(closePriceBytes, 0, bytes, 15 + contractLengthInt32, closePriceBytes.Length); + Array.Copy(highPriceBytes, 0, bytes, 23 + contractLengthInt32, highPriceBytes.Length); + Array.Copy(lowPriceBytes, 0, bytes, 31 + contractLengthInt32, lowPriceBytes.Length); + + return bytes; + } + + public static byte[] GetUnusualActivityBytes(UnusualActivity unusualActivity) + { + byte[] contractBytes = Encoding.ASCII.GetBytes(unusualActivity.Contract); + byte contractLength = System.Convert.ToByte(contractBytes.Length); + int contractLengthInt32 = System.Convert.ToInt32(contractLength); + int unusualActivityTypeInt32 = (int)unusualActivity.UnusualActivityType; + byte unusualActivityTypeByte = (byte) unusualActivityTypeInt32; + int sentimentInt32 = (int)unusualActivity.Sentiment; + byte sentimentByte = (byte) sentimentInt32; + byte[] totalValueBytes = BitConverter.GetBytes(unusualActivity.TotalValue); // 8 byte float + byte[] totalSizeBytes = BitConverter.GetBytes(unusualActivity.TotalSize); // 4 byte uint32 + byte[] averagePriceBytes = BitConverter.GetBytes(unusualActivity.AveragePrice); // 8 byte float + byte[] askPriceAtExecutionBytes = BitConverter.GetBytes(unusualActivity.AskPriceAtExecution); // 8 byte float + byte[] bidPriceAtExecutionBytes = BitConverter.GetBytes(unusualActivity.BidPriceAtExecution); // 8 byte float + byte[] underlyingPriceAtExecutionBytes = BitConverter.GetBytes(unusualActivity.UnderlyingPriceAtExecution); // 8 byte float + byte[] timestampBytes = BitConverter.GetBytes(unusualActivity.Timestamp); // 8 byte float + + //// byte 0 | type | byte + //// byte 1 | messageLength (includes bytes 0 and 1) | byte + //// byte 2 | contractLength | byte + //// bytes [3...] | contract | string (ascii) + //// next byte | unusualActivityType | char + //// next byte | sentiment | char + //// next 8 bytes | totalValue | float64 + //// next 4 bytes | totalSize | uint32 + //// next 8 bytes | averagePrice | float64 + //// next 8 bytes | askPriceAtExecution | float64 + //// next 8 bytes | bidPriceAtExecution | float64 + //// next 8 bytes | underlyingPriceAtExecution | float64 + //// next 8 bytes | timestamp | float64 + + byte messageLength = (byte)(57u + contractLength); + + byte[] bytes = new byte[System.Convert.ToInt32(messageLength)]; + bytes[0] = System.Convert.ToByte((int)(Options.MessageType.UnusualActivity)); + bytes[1] = messageLength; + bytes[2] = contractLength; + Array.Copy(contractBytes, 0, bytes, 3, contractLengthInt32); + bytes[3 + contractLengthInt32] = unusualActivityTypeByte; + bytes[4 + contractLengthInt32] = sentimentByte; + Array.Copy(totalValueBytes, 0, bytes, 5 + contractLengthInt32, totalValueBytes.Length); + Array.Copy(totalSizeBytes, 0, bytes, 13 + contractLengthInt32, totalSizeBytes.Length); + Array.Copy(averagePriceBytes, 0, bytes, 17 + contractLengthInt32, averagePriceBytes.Length); + Array.Copy(askPriceAtExecutionBytes, 0, bytes, 25 + contractLengthInt32, askPriceAtExecutionBytes.Length); + Array.Copy(bidPriceAtExecutionBytes, 0, bytes, 33 + contractLengthInt32, bidPriceAtExecutionBytes.Length); + Array.Copy(underlyingPriceAtExecutionBytes, 0, bytes, 41 + contractLengthInt32, underlyingPriceAtExecutionBytes.Length); + Array.Copy(timestampBytes, 0, bytes, 49 + contractLengthInt32, timestampBytes.Length); + + return bytes; + } + + public DateTime TimeReceived { get { return _timeReceived; } } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public MessageType MessageType() + { + return _trade.HasValue + ? Options.MessageType.Trade + : _quote.HasValue + ? Options.MessageType.Quote + : _refresh.HasValue + ? Options.MessageType.Refresh + : Options.MessageType.UnusualActivity; + } + + public Trade Trade { get { return _trade.Value; } } + public Quote Quote { get { return _quote.Value; } } + public Refresh Refresh { get { return _refresh.Value; } } + public UnusualActivity UnusualActivity { get { return _unusualActivity.Value; } } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public byte[] GetTimeReceivedBytes() + { + return BitConverter.GetBytes(System.Convert.ToUInt64((_timeReceived - DateTime.UnixEpoch).Ticks) * 100UL); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public byte[] GetEventBytes() + { + return _trade.HasValue + ? GetTradeBytes(_trade.Value) + : _quote.HasValue + ? GetQuoteBytes(_quote.Value) + : _refresh.HasValue + ? GetRefreshBytes(_refresh.Value) + : _unusualActivity.HasValue + ? GetUnusualActivityBytes(_unusualActivity.Value) + : Array.Empty(); + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/Trade.cs b/Intrinio.Realtime/Options/Trade.cs new file mode 100644 index 0000000..3d855a6 --- /dev/null +++ b/Intrinio.Realtime/Options/Trade.cs @@ -0,0 +1,117 @@ +using System.Globalization; + +namespace Intrinio.Realtime.Options; + +using System; + +public struct Trade +{ + private readonly string _contract; + private readonly Exchange _exchange; + private readonly byte _priceType; + private readonly byte _underlyingPriceType; + private readonly Int32 _price; + private readonly UInt32 _size; + private readonly UInt64 _timestamp; + private readonly UInt64 _totalVolume; + private readonly Conditions _qualifiers; + private readonly Int32 _askPriceAtExecution; + private readonly Int32 _bidPriceAtExecution; + private readonly Int32 _underlyingPriceAtExecution; + + public string Contract { get { return _contract; } } + public double Price { get { return (_price == Int32.MaxValue) || (_price == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_price, _priceType); } } + public UInt32 Size { get { return _size; } } + public UInt64 TotalVolume { get { return _totalVolume; } } + public double AskPriceAtExecution { get { return (_askPriceAtExecution == Int32.MaxValue) || (_askPriceAtExecution == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_askPriceAtExecution, _priceType); } } + public double BidPriceAtExecution { get { return (_bidPriceAtExecution == Int32.MaxValue) || (_bidPriceAtExecution == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_bidPriceAtExecution, _priceType); } } + public double UnderlyingPriceAtExecution { get { return (_underlyingPriceAtExecution == Int32.MaxValue) || (_underlyingPriceAtExecution == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_underlyingPriceAtExecution, _underlyingPriceType); } } + public double Timestamp { get { return Helpers.ScaleTimestampToSeconds(_timestamp); } } + public Exchange Exchange { get { return _exchange; } } + public Conditions Qualifiers { get { return _qualifiers; } } + + /// + /// A 'Trade' is a unit of data representing an individual market trade event. + /// + /// The id of the option contract (e.g. AAPL__201016C00100000). + /// The specific exchange through which the trade occurred. + /// The scalar for price. + /// The scalar for underlying price. + /// The dollar price of the last trade. + /// The number of contacts for the trade. + /// The time that the trade was executed (a unix timestamp representing the number of seconds (or better) since the unix epoch). + /// The running total trade volume for this contract today. + /// The exchange provided trade qualifiers. These can be used to classify whether a trade should be used, for example, for open, close, volume, high, or low. + /// The dollar price of the best ask at execution. + /// The dollar price of the best bid at execution. + /// The dollar price of the underlying security at the time of execution. + public Trade(string contract, Exchange exchange, byte priceType, byte underlyingPriceType, int price, UInt32 size, UInt64 timestamp, UInt64 totalVolume, Conditions qualifiers, int askPriceAtExecution, int bidPriceAtExecution, int underlyingPriceAtExecution) + { + + _contract = contract; + _exchange = exchange; + _priceType = priceType; + _underlyingPriceType = underlyingPriceType; + _price = price; //if + _size = size; + _totalVolume = totalVolume; + _qualifiers = qualifiers; + _askPriceAtExecution = askPriceAtExecution; + _bidPriceAtExecution = bidPriceAtExecution; + _underlyingPriceAtExecution = underlyingPriceAtExecution; + _timestamp = timestamp; + } + + public override string ToString() + { + return $"Trade (Contract: {Contract}, Exchange: {Exchange.ToString()}, Price: {Price.ToString("f3")}, Size: {Size.ToString()}, Timestamp: {Timestamp.ToString("f6")}, TotalVolume: {TotalVolume.ToString()}, Qualifiers: {Qualifiers}, AskPriceAtExecution: {AskPriceAtExecution.ToString("f3")}, BidPriceAtExecution: {BidPriceAtExecution.ToString("f3")}, UnderlyingPrice: {UnderlyingPriceAtExecution.ToString("f3")})"; + } + + public string GetUnderlyingSymbol() + { + return Contract.Substring(0, 6).TrimEnd('_'); + } + + public DateTime GetExpirationDate() + { + return DateTime.ParseExact(Contract.Substring(6, 6), "yyMMdd", CultureInfo.InvariantCulture); + } + + public bool IsCall() + { + return Contract[12] == 'C'; + } + + public bool IsPut() + { + return Contract[12] == 'P'; + } + + public double GetStrikePrice() + { + const UInt32 zeroChar = (UInt32)'0'; + + UInt32 whole = ((UInt32)Contract[13] - zeroChar) * 10_000u + + ((UInt32)Contract[14] - zeroChar) * 1_000u + + ((UInt32)Contract[15] - zeroChar) * 100u + + ((UInt32)Contract[16] - zeroChar) * 10u + + ((UInt32)Contract[17] - zeroChar) * 1u; + + double part = Convert.ToDouble((UInt32) Contract[18] - zeroChar) * 0.1D + + Convert.ToDouble((UInt32) Contract[19] - zeroChar) * 0.01D + + Convert.ToDouble((UInt32) Contract[20] - zeroChar) * 0.001D; + + return Convert.ToDouble(whole) + part; + } + + public static Trade CreateUnitTestObject(string contract, Exchange exchange, double price, UInt32 size, UInt64 nanoSecondsSinceUnixEpoch, UInt64 totalVolume, Conditions qualifiers, double askPriceAtExecution, double bidPriceAtExecution, double underlyingPriceAtExecution) + { + byte priceType = (byte)4; + int unscaledPrice = Convert.ToInt32(price * 10000.0); + int unscaledAskPriceAtExecution = Convert.ToInt32(askPriceAtExecution * 10000.0); + int unscaledBidPriceAtExecution = Convert.ToInt32(bidPriceAtExecution * 10000.0); + int unscaledUnderlyingPriceAtExecution = Convert.ToInt32(underlyingPriceAtExecution * 10000.0); + Trade trade = new Trade(contract, exchange, priceType, priceType, unscaledPrice, size, nanoSecondsSinceUnixEpoch, totalVolume, qualifiers, unscaledAskPriceAtExecution, unscaledBidPriceAtExecution, unscaledUnderlyingPriceAtExecution); + return trade; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/TradeCandleStick.cs b/Intrinio.Realtime/Options/TradeCandleStick.cs new file mode 100644 index 0000000..9f8fc5d --- /dev/null +++ b/Intrinio.Realtime/Options/TradeCandleStick.cs @@ -0,0 +1,149 @@ +using System.Globalization; + +namespace Intrinio.Realtime.Options; + +using System; + +public class TradeCandleStick : CandleStick, IEquatable, IComparable, IComparable +{ + private readonly string _contract; + + public string Contract + { + get { return _contract; } + } + + public TradeCandleStick(string contract, UInt32 volume, double price, double openTimestamp, double closeTimestamp, IntervalType interval, double tradeTime) + :base (volume, price, openTimestamp, closeTimestamp, interval, tradeTime) + { + _contract = contract; + } + + public TradeCandleStick(string contract, UInt32 volume, double high, double low, double closePrice, double openPrice, double openTimestamp, double closeTimestamp, double firstTimestamp, double lastTimestamp, bool complete, double average, double change, IntervalType interval) + :base(volume, high, low, closePrice, openPrice, openTimestamp, closeTimestamp, firstTimestamp, lastTimestamp, complete, average, change, interval) + { + _contract = contract; + } + + public override bool Equals(object other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (other is TradeCandleStick) + && (Contract.Equals(((TradeCandleStick)other).Contract)) + && (Interval.Equals(((TradeCandleStick)other).Interval)) + && (OpenTimestamp.Equals(((TradeCandleStick)other).OpenTimestamp)) + ); + } + + public override int GetHashCode() + { + return Contract.GetHashCode() ^ Interval.GetHashCode() ^ OpenTimestamp.GetHashCode(); + } + + public bool Equals(TradeCandleStick other) + { + return ((!(ReferenceEquals(other, null))) && ReferenceEquals(this, other)) + || ( + (!(ReferenceEquals(other, null))) + && (!(ReferenceEquals(this, other))) + && (Contract.Equals(other.Contract)) + && (Interval.Equals(other.Interval)) + && (OpenTimestamp.Equals(other.OpenTimestamp)) + ); + } + + public int CompareTo(object other) + { + return Equals(other) switch + { + true => 0, + false => ReferenceEquals(other, null) switch + { + true => 1, + false => (other is TradeCandleStick) switch + { + true => Contract.CompareTo(((TradeCandleStick)other).Contract) switch + { + < 0 => -1, + > 0 => 1, + 0 => Interval.CompareTo(((TradeCandleStick)other).Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => this.OpenTimestamp.CompareTo(((TradeCandleStick)other).OpenTimestamp) + } + }, + false => 1 + } + } + }; + } + + public int CompareTo(TradeCandleStick other) + { + return Equals(other) switch + { + true => 0, + false => Object.ReferenceEquals(other, null) switch + { + true => 1, + false => this.Contract.CompareTo(other.Contract) switch + { + < 0 => -1, + > 0 => 1, + 0 => this.Interval.CompareTo(other.Interval) switch + { + < 0 => -1, + > 0 => 1, + 0 => this.OpenTimestamp.CompareTo(other.OpenTimestamp) + } + } + } + }; + } + + public override string ToString() + { + return $"TradeCandleStick (Contract: {Contract}, Volume: {Volume.ToString()}, High: {High.ToString("f3")}, Low: {Low.ToString("f3")}, Close: {Close.ToString("f3")}, Open: {Open.ToString("f3")}, OpenTimestamp: {OpenTimestamp.ToString("f6")}, CloseTimestamp: {CloseTimestamp.ToString("f6")}, AveragePrice: {Average.ToString("f3")}, Change: {Change.ToString("f6")}, Complete: {Complete.ToString()})"; + } + + public string GetUnderlyingSymbol() + { + return Contract.Substring(0, 6).TrimEnd('_'); + } + + public DateTime GetExpirationDate() + { + return DateTime.ParseExact(Contract.Substring(6, 6), "yyMMdd", CultureInfo.InvariantCulture); + } + + public bool IsCall() + { + return Contract[12] == 'C'; + } + + public bool IsPut() + { + return Contract[12] == 'P'; + } + + public double GetStrikePrice() + { + const UInt32 zeroChar = (UInt32)'0'; + + UInt32 whole = ((UInt32)Contract[13] - zeroChar) * 10_000u + + ((UInt32)Contract[14] - zeroChar) * 1_000u + + ((UInt32)Contract[15] - zeroChar) * 100u + + ((UInt32)Contract[16] - zeroChar) * 10u + + ((UInt32)Contract[17] - zeroChar) * 1u; + + double part = Convert.ToDouble((UInt32) Contract[18] - zeroChar) * 0.1D + + Convert.ToDouble((UInt32) Contract[19] - zeroChar) * 0.01D + + Convert.ToDouble((UInt32) Contract[20] - zeroChar) * 0.001D; + + return Convert.ToDouble(whole) + part; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/UASentiment.cs b/Intrinio.Realtime/Options/UASentiment.cs new file mode 100644 index 0000000..0e447a2 --- /dev/null +++ b/Intrinio.Realtime/Options/UASentiment.cs @@ -0,0 +1,8 @@ +namespace Intrinio.Realtime.Options; + +public enum UASentiment +{ + Neutral = 0, + Bullish = 1, + Bearish = 2 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/UAType.cs b/Intrinio.Realtime/Options/UAType.cs new file mode 100644 index 0000000..263a110 --- /dev/null +++ b/Intrinio.Realtime/Options/UAType.cs @@ -0,0 +1,9 @@ +namespace Intrinio.Realtime.Options; + +public enum UAType +{ + Block = 3, + Sweep = 4, + Large = 5, + UnusualSweep = 6 +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/UnusualActivity.cs b/Intrinio.Realtime/Options/UnusualActivity.cs new file mode 100644 index 0000000..61acaca --- /dev/null +++ b/Intrinio.Realtime/Options/UnusualActivity.cs @@ -0,0 +1,117 @@ +using System.Globalization; + +namespace Intrinio.Realtime.Options; + +using System; + +public struct UnusualActivity +{ + private readonly string _contract; + private readonly UAType _unusualActivityType; + private readonly UASentiment _sentiment; + private readonly byte _priceType; + private readonly byte _underlyingPriceType; + private readonly UInt64 _totalValue; + private readonly UInt32 _totalSize; + private readonly int _averagePrice; + private readonly int _askPriceAtExecution; + private readonly int _bidPriceAtExecution; + private readonly int _underlyingPriceAtExecution; + private readonly UInt64 _timestamp; + + public string Contract { get { return _contract; } } + public UAType UnusualActivityType { get { return _unusualActivityType; } } + public UASentiment Sentiment { get { return _sentiment; } } + public double TotalValue { get { return (_totalValue == UInt64.MaxValue) || (_totalValue == 0UL) ? Double.NaN : Helpers.ScaleUInt64Price(_totalValue, _priceType); } } + public UInt32 TotalSize { get { return _totalSize; } } + public double AveragePrice { get { return (_averagePrice == Int32.MaxValue) || (_averagePrice == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_averagePrice, _priceType); } } + public double AskPriceAtExecution { get { return (_askPriceAtExecution == Int32.MaxValue) || (_askPriceAtExecution == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_askPriceAtExecution, _priceType); } } + public double BidPriceAtExecution { get { return (_bidPriceAtExecution == Int32.MaxValue) || (_bidPriceAtExecution == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_bidPriceAtExecution, _priceType); } } + public double UnderlyingPriceAtExecution { get { return (_underlyingPriceAtExecution == Int32.MaxValue) || (_underlyingPriceAtExecution == Int32.MinValue) ? Double.NaN : Helpers.ScaleInt32Price(_underlyingPriceAtExecution, _underlyingPriceType); } } + public double Timestamp { get { return Helpers.ScaleTimestampToSeconds(_timestamp); } } + + /// + /// An 'UnusualActivity' is an event that indicates unusual trading activity. + /// + /// The id of the option contract (e.g. AAPL__201016C00100000). + /// The type of unusual activity. + /// Bullish or Bearish. + /// The scalar for the price. + /// The scalar for the underlying price. + /// The total value in dollars of the unusual trading activity. + /// The total number of contracts of the unusual trading activity. + /// The average executed trade price of the unusual activity. + /// The best ask of this contract at the time of execution. + /// The best bid of this contract at the time of execution. + /// The dollar price of the underlying security at the time of execution. + /// The time that the unusual activity began (a unix timestamp representing the number of seconds (or better) since the unix epoch). + public UnusualActivity(string contract, UAType unusualActivityType, UASentiment sentiment, byte priceType , byte underlyingPriceType, UInt64 totalValue, UInt32 totalSize, int averagePrice, int askPriceAtExecution, int bidPriceAtExecution, int underlyingPriceAtExecution, UInt64 timestamp) + { + _contract = contract; + _unusualActivityType = unusualActivityType; + _sentiment = sentiment; + _priceType = priceType; + _underlyingPriceType = underlyingPriceType; + _totalValue = totalValue; + _totalSize = totalSize; + _averagePrice = averagePrice; + _askPriceAtExecution = askPriceAtExecution; + _bidPriceAtExecution = bidPriceAtExecution; + _underlyingPriceAtExecution = underlyingPriceAtExecution; + _timestamp = timestamp; + } + + public override string ToString() + { + return $"UnusualActivity (Contract: {Contract}, Type: {UnusualActivityType.ToString()}, Sentiment: {Sentiment.ToString()}, TotalValue: {TotalValue.ToString("f3")}, TotalSize: {TotalSize.ToString()}, AveragePrice: {AveragePrice.ToString("f3")}, AskPriceAtExecution: {AskPriceAtExecution.ToString("f3")}, BidPriceAtExecution: {BidPriceAtExecution.ToString("f3")}, UnderlyingPriceAtExecution: {UnderlyingPriceAtExecution.ToString("f3")}, Timestamp: {Timestamp.ToString("f6")})"; + } + + public string GetUnderlyingSymbol() + { + return Contract.Substring(0, 6).TrimEnd('_'); + } + + public DateTime GetExpirationDate() + { + return DateTime.ParseExact(Contract.Substring(6, 6), "yyMMdd", CultureInfo.InvariantCulture); + } + + public bool IsCall() + { + return Contract[12] == 'C'; + } + + public bool IsPut() + { + return Contract[12] == 'P'; + } + + public double GetStrikePrice() + { + const UInt32 zeroChar = (UInt32)'0'; + + UInt32 whole = ((UInt32)Contract[13] - zeroChar) * 10_000u + + ((UInt32)Contract[14] - zeroChar) * 1_000u + + ((UInt32)Contract[15] - zeroChar) * 100u + + ((UInt32)Contract[16] - zeroChar) * 10u + + ((UInt32)Contract[17] - zeroChar) * 1u; + + double part = Convert.ToDouble((UInt32) Contract[18] - zeroChar) * 0.1D + + Convert.ToDouble((UInt32) Contract[19] - zeroChar) * 0.01D + + Convert.ToDouble((UInt32) Contract[20] - zeroChar) * 0.001D; + + return Convert.ToDouble(whole) + part; + } + + public static UnusualActivity CreateUnitTestObject(string contract, UAType unusualActivityType, UASentiment sentimentType, double totalValue, UInt32 totalSize, double averagePrice, double askPriceAtExecution, double bidPriceAtExecution, double underlyingPriceAtExecution, UInt64 nanoSecondsSinceUnixEpoch) + { + byte priceType = (byte)4; + UInt64 unscaledTotalValue = Convert.ToUInt64(totalValue * 10000.0); + int unscaledAveragePrice = Convert.ToInt32(averagePrice * 10000.0); + int unscaledAskPriceAtExecution = Convert.ToInt32(askPriceAtExecution * 10000.0); + int unscaledBidPriceAtExecution = Convert.ToInt32(bidPriceAtExecution * 10000.0); + int unscaledUnderlyingPriceAtExecution = Convert.ToInt32(underlyingPriceAtExecution * 10000.0); + UnusualActivity ua = new UnusualActivity(contract, unusualActivityType, sentimentType, priceType, priceType, unscaledTotalValue, totalSize, unscaledAveragePrice, unscaledAskPriceAtExecution, unscaledBidPriceAtExecution, unscaledUnderlyingPriceAtExecution, nanoSecondsSinceUnixEpoch); + return ua; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/Options/config.json b/Intrinio.Realtime/Options/config.json new file mode 100644 index 0000000..06644cb --- /dev/null +++ b/Intrinio.Realtime/Options/config.json @@ -0,0 +1,27 @@ +{ + "Config": { + "ApiKey": "API_KEY_HERE", + "NumThreads": 16, + "Provider": "OPRA", + //"Provider": "MANUAL", + //"IPAddress": "1.2.3.4", + "BufferSize": 4096, + "OverflowBufferSize": 8192, + "Delayed": false, + "Symbols": [ "GOOG__220408C02870000", "MSFT__220408C00315000", "AAPL__220414C00180000", "SPY", "TSLA" ] + //"Symbols": [ "lobby" ] + }, + "Serilog": { + "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" } + ] + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/RingBuffer.cs b/Intrinio.Realtime/RingBuffer.cs new file mode 100644 index 0000000..fb030be --- /dev/null +++ b/Intrinio.Realtime/RingBuffer.cs @@ -0,0 +1,120 @@ +using System; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Intrinio.Realtime; + +internal class RingBuffer +{ + #region Data Members + private readonly byte[] _data; + private uint _blockNextReadIndex; + private uint _blockNextWriteIndex; + private readonly object _readLock; + private readonly object _writeLock; + private ulong _count; + private readonly uint _blockSize; + private readonly uint _blockCapacity; + private ulong _dropCount; + + public ulong Count { get { return Interlocked.Read(ref _count); } } + public uint BlockSize { get { return _blockSize; } } + public uint BlockCapacity { get { return _blockCapacity; } } + public ulong DropCount { get { return Interlocked.Read(ref _dropCount); } } + + public bool IsEmpty + { + get + { + return IsEmptyNoLock(); + } + } + + public bool IsFull + { + get + { + return IsFullNoLock(); + } + } + #endregion //Data Members + + #region Constructors + + public RingBuffer(uint blockSize, uint blockCapacity) + { + _blockSize = blockSize; + _blockCapacity = blockCapacity; + _blockNextReadIndex = 0u; + _blockNextWriteIndex = 0u; + _count = 0u; + _dropCount = 0UL; + _readLock = new object(); + _writeLock = new object(); + _data = new byte[blockSize * blockCapacity]; + } + + #endregion //Constructors + + /// + /// blockToWrite MUST be of length BlockSize! + /// + /// + public bool TryEnqueue(ReadOnlySpan blockToWrite) + { + if (IsFullNoLock()) + { + Interlocked.Increment(ref _dropCount); + return false; + } + + lock (_writeLock) + { + if (IsFullNoLock()) + { + Interlocked.Increment(ref _dropCount); + return false; + } + + Span target = new Span(_data, Convert.ToInt32(_blockNextWriteIndex * BlockSize), Convert.ToInt32(BlockSize)); + blockToWrite.CopyTo(target); + + _blockNextWriteIndex = (++_blockNextWriteIndex) % BlockCapacity; + Interlocked.Increment(ref _count); + + return true; + } + } + + /// + /// blockBuffer MUST be of length BlockSize! + /// + /// + public bool TryDequeue(Span blockBuffer) + { + lock (_readLock) + { + if (IsEmptyNoLock()) + return false; + + Span target = new Span(_data, Convert.ToInt32(_blockNextReadIndex * BlockSize), Convert.ToInt32(BlockSize)); + target.CopyTo(blockBuffer); + + _blockNextReadIndex = (++_blockNextReadIndex) % BlockCapacity; + Interlocked.Decrement(ref _count); + return true; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsFullNoLock() + { + return Interlocked.Read(ref _count) == _blockCapacity; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsEmptyNoLock() + { + return Interlocked.Read(ref _count) == 0UL; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/SingleProducerRingBuffer.cs b/Intrinio.Realtime/SingleProducerRingBuffer.cs new file mode 100644 index 0000000..2318481 --- /dev/null +++ b/Intrinio.Realtime/SingleProducerRingBuffer.cs @@ -0,0 +1,111 @@ +using System; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Intrinio.Realtime; + +internal class SingleProducerRingBuffer +{ + #region Data Members + private readonly byte[] _data; + private uint _blockNextReadIndex; + private uint _blockNextWriteIndex; + private readonly object _readLock; + private readonly object _writeLock; + private ulong _count; + private readonly uint _blockSize; + private readonly uint _blockCapacity; + private ulong _dropCount; + + public ulong Count { get { return Interlocked.Read(ref _count); } } + public uint BlockSize { get { return _blockSize; } } + public uint BlockCapacity { get { return _blockCapacity; } } + public ulong DropCount { get { return Interlocked.Read(ref _dropCount); } } + + public bool IsEmpty + { + get + { + return IsEmptyNoLock(); + } + } + + public bool IsFull + { + get + { + return IsFullNoLock(); + } + } + #endregion //Data Members + + #region Constructors + + public SingleProducerRingBuffer(uint blockSize, uint blockCapacity) + { + _blockSize = blockSize; + _blockCapacity = blockCapacity; + _blockNextReadIndex = 0u; + _blockNextWriteIndex = 0u; + _count = 0u; + _dropCount = 0UL; + _readLock = new object(); + _writeLock = new object(); + _data = new byte[blockSize * blockCapacity]; + } + + #endregion //Constructors + + /// + /// blockToWrite MUST be of length BlockSize! + /// + /// + public bool TryEnqueue(ReadOnlySpan blockToWrite) + { + if (IsFullNoLock()) + { + Interlocked.Increment(ref _dropCount); + return false; + } + + Span target = new Span(_data, Convert.ToInt32(_blockNextWriteIndex * BlockSize), Convert.ToInt32(BlockSize)); + blockToWrite.CopyTo(target); + + _blockNextWriteIndex = (++_blockNextWriteIndex) % BlockCapacity; + Interlocked.Increment(ref _count); + + return true; + } + + /// + /// blockBuffer MUST be of length BlockSize! + /// + /// + public bool TryDequeue(Span blockBuffer) + { + lock (_readLock) + { + if (IsEmptyNoLock()) + return false; + + Span target = new Span(_data, Convert.ToInt32(_blockNextReadIndex * BlockSize), Convert.ToInt32(BlockSize)); + target.CopyTo(blockBuffer); + + _blockNextReadIndex = (++_blockNextReadIndex) % BlockCapacity; + Interlocked.Decrement(ref _count); + return true; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsFullNoLock() + { + return Interlocked.Read(ref _count) == _blockCapacity; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsEmptyNoLock() + { + return Interlocked.Read(ref _count) == 0UL; + } +} \ No newline at end of file diff --git a/Intrinio.Realtime/WebSocketClient.cs b/Intrinio.Realtime/WebSocketClient.cs new file mode 100644 index 0000000..595a35e --- /dev/null +++ b/Intrinio.Realtime/WebSocketClient.cs @@ -0,0 +1,558 @@ +using System.Linq; +using System.Net; +using System.Net.WebSockets; + +namespace Intrinio.Realtime; + +using System; +using System.Net.Http; +using System.Text; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using System.Net.Sockets; + +public abstract class WebSocketClient +{ + #region Data Members + private readonly uint _processingThreadsQuantity; + private readonly uint _bufferSize; + private readonly uint _overflowBufferSize; + private readonly int[] _selfHealBackoffs = new int[] { 10_000, 30_000, 60_000, 300_000, 600_000 }; + private readonly object _tLock = new (); + private readonly object _wsLock = new (); + private Tuple _token = new (null, DateTime.Now); + private WebSocketState _wsState = null; + private UInt64 _dataMsgCount = 0UL; + private UInt64 _dataEventCount = 0UL; + private UInt64 _textMsgCount = 0UL; + private readonly HashSet _channels = new (); + protected IEnumerable Channels { get { return _channels.ToArray(); } } + private readonly CancellationTokenSource _ctSource = new (); + protected CancellationToken CancellationToken { get { return _ctSource.Token; } } + private readonly uint _maxMessageSize; + private readonly uint _bufferBlockSize; + private readonly SingleProducerRingBuffer _data; + private readonly DropOldestRingBuffer _overflowData; + private readonly Action _tryReconnect; + private readonly HttpClient _httpClient = new (); + private const string ClientInfoHeaderKey = "Client-Information"; + private const string ClientInfoHeaderValue = "IntrinioDotNetSDKv11.0"; + private readonly ThreadPriority _mainThreadPriority; + private readonly Thread[] _threads; + private Thread _receiveThread; + private bool _started; + #endregion //Data Members + + #region Constuctors + /// + /// Create a new Equities websocket client. + /// + /// + /// + /// + public WebSocketClient(uint processingThreadsQuantity, uint bufferSize, uint overflowBufferSize, uint maxMessageSize) + { + _started = false; + _mainThreadPriority = Thread.CurrentThread.Priority; //this is set outside of our scope - let's not interfere. + _maxMessageSize = maxMessageSize; + _bufferBlockSize = 256 * _maxMessageSize; //256 possible messages in a group, and let's buffer 64bytes per message + _processingThreadsQuantity = processingThreadsQuantity > 0 ? processingThreadsQuantity : 2; + _bufferSize = bufferSize >= 2048 ? bufferSize : 2048; + _overflowBufferSize = overflowBufferSize >= 2048 ? overflowBufferSize : 2048; + _threads = GC.AllocateUninitializedArray(Convert.ToInt32(_processingThreadsQuantity)); + + _data = new SingleProducerRingBuffer(_bufferBlockSize, Convert.ToUInt32(_bufferSize)); + _overflowData = new DropOldestRingBuffer(_bufferBlockSize, Convert.ToUInt32(_overflowBufferSize)); + + _httpClient.Timeout = TimeSpan.FromSeconds(5.0); + + _tryReconnect = async () => + { + DoBackoff(() => + { + LogMessage(LogLevel.INFORMATION, "Websocket - Reconnecting...", Array.Empty()); + if (_wsState.IsReady) + return true; + + lock (_wsLock) + { + _wsState.IsReconnecting = true; + } + + string token = GetToken(); + ResetWebSocket(token).Wait(); + return false; + }); + }; + } + #endregion //Constructors + + #region Public Methods + + public async Task Start() + { + if (_started) + return; + _started = true; + + _receiveThread = new Thread(ReceiveFn); + for (int i = 0; i < _threads.Length; i++) + _threads[i] = new Thread(ProcessFn); + + _httpClient.DefaultRequestHeaders.Add(ClientInfoHeaderKey, ClientInfoHeaderValue); + foreach (KeyValuePair customSocketHeader in GetCustomSocketHeaders()) + { + _httpClient.DefaultRequestHeaders.Add(customSocketHeader.Key, customSocketHeader.Value); + } + string token = GetToken(); + await InitializeWebSockets(token); + } + + public async Task Stop() + { + if (_started) + { + try + { + LeaveImpl(); + Thread.Sleep(1000); + lock (_wsLock) + { + _wsState.IsReady = false; + } + LogMessage(LogLevel.INFORMATION, "Websocket - Closing...", Array.Empty()); + try + { + await _wsState.WebSocket.CloseAsync(System.Net.WebSockets.WebSocketCloseStatus.NormalClosure, null, _ctSource.Token); + } + catch (Exception e) + { + Console.WriteLine(e); + } + _ctSource.Cancel(); + if (_receiveThread != null) + _receiveThread.Join(); + foreach (Thread thread in _threads) + if (thread != null) + thread.Join(); + } + finally + { + _started = false; + } + } + + LogMessage(LogLevel.INFORMATION, "Stopped", Array.Empty()); + } + + public ClientStats GetStats() + { + return new ClientStats(Interlocked.Read(ref _dataMsgCount), + Interlocked.Read(ref _textMsgCount), + Convert.ToInt32(_data.Count), + Interlocked.Read(ref _dataEventCount), + Convert.ToInt32(_data.BlockCapacity), + Convert.ToInt32(_overflowData.Count), + Convert.ToInt32(_overflowData.BlockCapacity), + Convert.ToInt32(_overflowData.DropCount), + System.Convert.ToInt32(_data.DropCount)); + } + + [Serilog.Core.MessageTemplateFormatMethod("messageTemplate")] + public void LogMessage(LogLevel logLevel, string messageTemplate, params object[] propertyValues) + { + switch (logLevel) + { + case LogLevel.DEBUG: + Serilog.Log.Debug(GetLogPrefix + messageTemplate, propertyValues); + break; + case LogLevel.INFORMATION: + Serilog.Log.Information(GetLogPrefix + messageTemplate, propertyValues); + break; + case LogLevel.WARNING: + Serilog.Log.Warning(GetLogPrefix + messageTemplate, propertyValues); + break; + case LogLevel.ERROR: + Serilog.Log.Error(GetLogPrefix + messageTemplate, propertyValues); + break; + default: + throw new ArgumentException("LogLevel not specified!"); + break; + } + } + #endregion //Public Methods + + #region Protected Methods + + protected bool IsReady() + { + lock (_wsLock) + { + return !ReferenceEquals(null, _wsState) && _wsState.IsReady; + } + } + + protected async Task LeaveImpl() + { + foreach (string channel in _channels.ToArray()) + { + await LeaveImpl(channel); + } + } + + protected async Task LeaveImpl(string channel) + { + if (_channels.Remove(channel)) + { + byte[] message = MakeLeaveMessage(channel); + LogMessage(LogLevel.INFORMATION, "Websocket - Leaving channel: {0}", new object[]{channel}); + try + { + await _wsState.WebSocket.SendAsync(message, WebSocketMessageType.Binary, true, _ctSource.Token); + } + catch(Exception e) + { + LogMessage(LogLevel.INFORMATION, "Websocket - Error while leaving channel: {0}; Message: {1}; Stack Trace: {2}", new object[]{channel, e.Message, e.StackTrace}); + } + } + } + + protected async Task JoinImpl(IEnumerable channels, bool skipAddCheck = false) + { + foreach (string channel in channels) + { + JoinImpl(channel, skipAddCheck); + } + } + + protected async Task JoinImpl(string channel, bool skipAddCheck = false) + { + if (_channels.Add(channel) || skipAddCheck) + { + byte[] message = MakeJoinMessage(channel); + LogMessage(LogLevel.INFORMATION, "Websocket - Joining channel: {0}", new object[]{channel}); + try + { + await _wsState.WebSocket.SendAsync(message, WebSocketMessageType.Binary, true, _ctSource.Token); + } + catch(Exception e) + { + _channels.Remove(channel); + LogMessage(LogLevel.INFORMATION, "Websocket - Error while joining channel: {0}; Message: {1}; Stack Trace: {2}", new object[]{channel, e.Message, e.StackTrace}); + } + } + } + + #endregion //Protected Methods + + #region Abstract Methods + protected abstract string GetLogPrefix(); + protected abstract string GetAuthUrl(); + protected abstract string GetWebSocketUrl(string token); + protected abstract List> GetCustomSocketHeaders(); + protected abstract byte[] MakeJoinMessage(string channel); + protected abstract byte[] MakeLeaveMessage(string channel); + protected abstract void HandleMessage(ReadOnlySpan bytes); + protected abstract int GetNextChunkLength(ReadOnlySpan bytes); + + #endregion //Abstract Methods + + #region Private Methods + + private enum CloseType + { + Closed, + Refused, + Unavailable, + Other + } + + private CloseType GetCloseType(Exception exception) + { + if ((exception.GetType() == typeof(SocketException)) + || exception.Message.StartsWith("A connection attempt failed because the connected party did not properly respond after a period of time") + || exception.Message.StartsWith("The remote party closed the WebSocket connection without completing the close handshake") + ) + { + return CloseType.Closed; + } + if ((exception.GetType() == typeof(SocketException)) && (exception.Message == "No connection could be made because the target machine actively refused it.")) + { + return CloseType.Refused; + } + if (exception.Message.StartsWith("HTTP/1.1 503")) + { + return CloseType.Unavailable; + } + return CloseType.Other; + } + + private async void ReceiveFn() + { + CancellationToken token = _ctSource.Token; + Thread.CurrentThread.Priority = ThreadPriority.Highest; + byte[] buffer = new byte[_bufferBlockSize]; + while (!token.IsCancellationRequested) + { + try + { + if (_wsState.IsConnected) + { + var result = await _wsState.WebSocket.ReceiveAsync(buffer, token); + switch (result.MessageType) + { + case WebSocketMessageType.Binary: + if (result.Count > 0) + { + Interlocked.Increment(ref _dataMsgCount); + if (!_data.TryEnqueue(buffer)) + _overflowData.Enqueue(buffer); + } + break; + case WebSocketMessageType.Text: + OnTextMessageReceived(buffer); + break; + case WebSocketMessageType.Close: + OnClose(buffer); + break; + } + } + else + await Task.Delay(1000, token); + } + catch (NullReferenceException ex) + { + //Do nothing, websocket is resetting. + } + catch (OperationCanceledException) + { + } + catch (Exception exn) + { + CloseType exceptionType = GetCloseType(exn); + switch (exceptionType) + { + case CloseType.Closed: + LogMessage(LogLevel.WARNING, "Websocket - Error - Connection failed", Array.Empty()); + break; + case CloseType.Refused: + LogMessage(LogLevel.WARNING, "Websocket - Error - Connection refused", Array.Empty()); + break; + case CloseType.Unavailable: + LogMessage(LogLevel.WARNING, "Websocket - Error - Server unavailable", Array.Empty()); + break; + default: + LogMessage(LogLevel.ERROR, "Websocket - Error - {0}:{1}", new object[]{exn.GetType(), exn.Message}); + break; + } + + OnClose(buffer); + } + } + } + + private void ProcessFn() + { + CancellationToken ct = _ctSource.Token; + Thread.CurrentThread.Priority = (ThreadPriority)(Math.Max((((int)_mainThreadPriority) - 1), 0)); //Set below main thread priority so doesn't interfere with main thread accepting messages. + byte[] underlyingBuffer = new byte[_bufferBlockSize]; + Span datum = new Span(underlyingBuffer); + while (!ct.IsCancellationRequested) + { + try + { + if (_data.TryDequeue(datum) || _overflowData.TryDequeue(datum)) + { + // These are grouped (many) messages. + // The first byte tells us how many messages there are. + // From there, for each message, check the message length at index 1 of each chunk to know how many bytes each chunk has. + UInt64 cnt = Convert.ToUInt64(datum[0]); + Interlocked.Add(ref _dataEventCount, cnt); + int startIndex = 1; + for (ulong i = 0UL; i < cnt; ++i) + { + int msgLength = 1; //default value in case corrupt array so we don't reprocess same bytes over and over. + try + { + msgLength = GetNextChunkLength(datum.Slice(startIndex)); + ReadOnlySpan chunk = datum.Slice(startIndex, msgLength); + HandleMessage(chunk); + } + catch(Exception e) {LogMessage(LogLevel.ERROR, "Error parsing message: {0}; {1}", new object[]{e.Message, e.StackTrace});} + finally + { + startIndex += msgLength; + } + } + } + else + Thread.Sleep(10); + } + catch (OperationCanceledException) + { + } + catch (Exception exn) + { + LogMessage(LogLevel.ERROR, "Error parsing message: {0}; {1}", new object[]{exn.Message, exn.StackTrace}); + } + }; + } + + private void DoBackoff(Func fn) + { + int i = 0; + int backoff = _selfHealBackoffs[i]; + bool success = fn(); + while (!success) + { + Thread.Sleep(backoff); + i = Math.Min(i + 1, _selfHealBackoffs.Length - 1); + backoff = _selfHealBackoffs[i]; + success = fn(); + } + } + + private async Task TrySetToken() + { + LogMessage(LogLevel.INFORMATION, "Authorizing...", Array.Empty()); + string authUrl = GetAuthUrl(); + try + { + HttpResponseMessage response = await _httpClient.GetAsync(authUrl); + if (response.IsSuccessStatusCode) + { + string token = await response.Content.ReadAsStringAsync(); + Interlocked.Exchange(ref _token, new Tuple(token, DateTime.Now)); + return true; + } + else + { + LogMessage(LogLevel.WARNING, "Authorization Failure. Authorization server status code = {0}", new object[]{response.StatusCode}); + return false; + } + } + catch (System.InvalidOperationException exn) + { + LogMessage(LogLevel.ERROR, "Authorization Failure (bad URI): {0}", new object[]{exn.Message}); + return false; + } + catch (System.Net.Http.HttpRequestException exn) + { + LogMessage(LogLevel.ERROR, "Authoriztion Failure (bad network connection): {0}", new object[]{exn.Message}); + return false; + } + catch (TaskCanceledException exn) + { + LogMessage(LogLevel.ERROR, "Authorization Failure (timeout): {0}", new object[]{exn.Message}); + return false; + } + catch (AggregateException exn) + { + LogMessage(LogLevel.ERROR, "Authorization Failure: AggregateException: {0}", new object[]{exn.Message}); + return false; + } + catch (Exception exn) + { + LogMessage(LogLevel.ERROR, "Authorization Failure: {0}", new object[]{exn.Message}); + return false; + } + } + + private string GetToken() + { + lock (_tLock) + { + DoBackoff((() => TrySetToken().Result)); + } + + return _token.Item1; + } + + private async Task OnOpen() + { + LogMessage(LogLevel.INFORMATION, "Websocket - Connected", Array.Empty()); + lock (_wsLock) + { + _wsState.IsReady = true; + _wsState.IsReconnecting = false; + foreach (Thread thread in _threads) + { + if (!thread.IsAlive && thread.ThreadState.HasFlag(ThreadState.Unstarted)) + thread.Start(); + } + if (!_receiveThread.IsAlive && _receiveThread.ThreadState.HasFlag(ThreadState.Unstarted)) + _receiveThread.Start(); + } + + await JoinImpl(_channels, true); + } + + private void OnClose(ArraySegment message) + { + lock (_wsLock) + { + try + { + if (!_wsState.IsReconnecting) + { + // if (message != null && message.Count > 0) + // LogMessage(LogLevel.INFORMATION, "Websocket - Closed. {0}", Encoding.ASCII.GetString(message)); + // else + // LogMessage(LogLevel.INFORMATION, "Websocket - Closed."); + LogMessage(LogLevel.INFORMATION, "Websocket - Closed."); + + _wsState.IsReady = false; + + if (!_ctSource.IsCancellationRequested) + { + Task.Run(_tryReconnect); + } + } + } + catch(Exception e) + { + LogMessage(LogLevel.INFORMATION, "Websocket - Error on close: {0}. Stack Trace: {1}", e.Message, e.StackTrace); + } + } + } + + private void OnTextMessageReceived(ArraySegment message) + { + Interlocked.Increment(ref _textMsgCount); + LogMessage(LogLevel.ERROR, "Error received: {0}", Encoding.ASCII.GetString(message)); + } + + private async Task ResetWebSocket(string token) + { + LogMessage(LogLevel.INFORMATION, "Websocket - Resetting", Array.Empty()); + Uri wsUrl = new Uri(GetWebSocketUrl(token)); + List> headers = GetCustomSocketHeaders(); + ClientWebSocket ws = new ClientWebSocket(); + headers.ForEach(h => ws.Options.SetRequestHeader(h.Key, h.Value)); + lock (_wsLock) + { + _wsState.WebSocket = ws; + _wsState.Reset(); + } + await _wsState.WebSocket.ConnectAsync(wsUrl, _ctSource.Token); + await OnOpen(); + } + + private async Task InitializeWebSockets(string token) + { + Uri wsUrl = new Uri(GetWebSocketUrl(token)); + List> headers = GetCustomSocketHeaders(); + lock (_wsLock) + { + LogMessage(LogLevel.INFORMATION, "Websocket - Connecting...", Array.Empty()); + ClientWebSocket ws = new ClientWebSocket(); + headers.ForEach(h => ws.Options.SetRequestHeader(h.Key, h.Value)); + _wsState = new WebSocketState(ws); + } + await _wsState.WebSocket.ConnectAsync(wsUrl, _ctSource.Token); + await OnOpen(); + } + + #endregion //Private Methods +} \ No newline at end of file diff --git a/Intrinio.Realtime/WebSocketState.cs b/Intrinio.Realtime/WebSocketState.cs new file mode 100644 index 0000000..2cc2c3c --- /dev/null +++ b/Intrinio.Realtime/WebSocketState.cs @@ -0,0 +1,39 @@ +namespace Intrinio.Realtime; + +using System; +using System.Net.WebSockets; +//using WebSocket4Net; + +internal class WebSocketState +{ + public ClientWebSocket WebSocket { get; set; } + public bool IsReady { get; set; } + public bool IsReconnecting { get; set; } + + private DateTime _lastReset; + + public DateTime LastReset + { + get { return _lastReset; } + } + + public bool IsConnected { + get + { + return IsReady && !IsReconnecting && WebSocket != null && WebSocket.State == System.Net.WebSockets.WebSocketState.Open; + } + } + + public WebSocketState(ClientWebSocket ws) + { + WebSocket = ws; + IsReady = false; + IsReconnecting = false; + _lastReset = DateTime.Now; + } + + public void Reset() + { + _lastReset = DateTime.Now; + } +} \ No newline at end of file diff --git a/IntrinioRealTimeSDK/runtimeconfig.template.json b/Intrinio.Realtime/runtimeconfig.template.json similarity index 100% rename from IntrinioRealTimeSDK/runtimeconfig.template.json rename to Intrinio.Realtime/runtimeconfig.template.json diff --git a/IntrinioRealTimeClient.nuspec b/IntrinioRealTimeClient.nuspec index a4a21f0..ebad781 100644 --- a/IntrinioRealTimeClient.nuspec +++ b/IntrinioRealTimeClient.nuspec @@ -2,42 +2,40 @@ IntrinioRealTimeClient - 10.0.0 - Intrinio SDK for Real-Time Stock Prices + 11.0.0 + Intrinio SDK for Real-Time Stock and Option Prices Intrinio Intrinio false https://licenses.nuget.org/MIT https://github.com/intrinio/intrinio-realtime-csharp-sdk - Intrinio provides real-time stock prices via a two-way WebSocket connection. - Version 10.0.0 release. + Intrinio provides real-time stock and option prices via a two-way WebSocket connection. + Version 11.0.0 release. Copyright 2024 Intrinio - fintech stocks prices websocket real-time market finance + fintech stocks options prices websocket real-time market finance - - - - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + + diff --git a/IntrinioRealTimeSDK.sln b/IntrinioRealTimeSDK.sln index 3261c43..344a365 100644 --- a/IntrinioRealTimeSDK.sln +++ b/IntrinioRealTimeSDK.sln @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.30804.86 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "IntrinioRealtimeMultiExchange", "IntrinioRealtimeMultiExchange\IntrinioRealtimeMultiExchange.fsproj", "{3409E8EE-BB04-4E36-B2F8-029134727FFB}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SampleApp", "SampleApp\SampleApp.csproj", "{A013870E-4C79-4113-B7B2-12A2A864C2D7}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntrinioRealTimeSDK", "IntrinioRealTimeSDK\IntrinioRealTimeSDK.csproj", "{6E5BD6AB-9909-4980-BA1C-B0466445CD2B}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intrinio.Realtime", "Intrinio.Realtime\Intrinio.Realtime.csproj", "{B639A9AC-4A1C-4C80-8710-8CC5753918D9}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -13,14 +13,14 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {3409E8EE-BB04-4E36-B2F8-029134727FFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3409E8EE-BB04-4E36-B2F8-029134727FFB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3409E8EE-BB04-4E36-B2F8-029134727FFB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3409E8EE-BB04-4E36-B2F8-029134727FFB}.Release|Any CPU.Build.0 = Release|Any CPU - {6E5BD6AB-9909-4980-BA1C-B0466445CD2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6E5BD6AB-9909-4980-BA1C-B0466445CD2B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6E5BD6AB-9909-4980-BA1C-B0466445CD2B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6E5BD6AB-9909-4980-BA1C-B0466445CD2B}.Release|Any CPU.Build.0 = Release|Any CPU + {A013870E-4C79-4113-B7B2-12A2A864C2D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A013870E-4C79-4113-B7B2-12A2A864C2D7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A013870E-4C79-4113-B7B2-12A2A864C2D7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A013870E-4C79-4113-B7B2-12A2A864C2D7}.Release|Any CPU.Build.0 = Release|Any CPU + {B639A9AC-4A1C-4C80-8710-8CC5753918D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B639A9AC-4A1C-4C80-8710-8CC5753918D9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B639A9AC-4A1C-4C80-8710-8CC5753918D9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B639A9AC-4A1C-4C80-8710-8CC5753918D9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/IntrinioRealTimeSDK/IntrinioRealTimeSDK.csproj b/IntrinioRealTimeSDK/IntrinioRealTimeSDK.csproj deleted file mode 100644 index 0242817..0000000 --- a/IntrinioRealTimeSDK/IntrinioRealTimeSDK.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - Exe - net8.0 - en - - - - true - - - - - - - - - - - diff --git a/IntrinioRealTimeSDK/Program.cs b/IntrinioRealTimeSDK/Program.cs deleted file mode 100644 index fdbd0cf..0000000 --- a/IntrinioRealTimeSDK/Program.cs +++ /dev/null @@ -1,164 +0,0 @@ -using System; -using System.Threading; -using System.Collections.Concurrent; -using Intrinio.Realtime.Equities; -using Serilog; -using Serilog.Core; - -namespace SampleApp -{ - class Program - { - private static IEquitiesWebSocketClient client = null; - private static IEquitiesWebSocketClient replayClient = null; - private static CandleStickClient _candleStickClient = null; - private static Timer timer = null; - private static readonly ConcurrentDictionary trades = new(5, 15_000); - private static readonly ConcurrentDictionary quotes = new(5, 15_000); - private static int maxTradeCount = 0; - private static int maxQuoteCount = 0; - private static Trade maxCountTrade; - private static Quote maxCountQuote; - private static UInt64 _tradeCandleStickCount = 0UL; - private static UInt64 _tradeCandleStickCountIncomplete = 0UL; - private static UInt64 _AskCandleStickCount = 0UL; - private static UInt64 _AskCandleStickCountIncomplete = 0UL; - private static UInt64 _BidCandleStickCount = 0UL; - private static UInt64 _BidCandleStickCountIncomplete = 0UL; - private static bool _useTradeCandleSticks = false; - private static bool _useQuoteCandleSticks = false; - - static void OnQuote(Quote quote) - { - string key = quote.Symbol + ":" + quote.Type; - int updateFunc(string _, int prevValue) - { - if (prevValue + 1 > maxQuoteCount) - { - maxQuoteCount = prevValue + 1; - maxCountQuote = quote; - } - return (prevValue + 1); - } - quotes.AddOrUpdate(key, 1, updateFunc); - } - - static void OnTrade(Trade trade) - { - string key = trade.Symbol; - int updateFunc(string _, int prevValue) - { - if (prevValue + 1 > maxTradeCount) - { - maxTradeCount = prevValue + 1; - maxCountTrade = trade; - } - return (prevValue + 1); - } - trades.AddOrUpdate(key, 1, updateFunc); - } - - static void OnTradeCandleStick(TradeCandleStick tradeCandleStick) - { - if (tradeCandleStick.Complete) - { - Interlocked.Increment(ref _tradeCandleStickCount); - } - else - { - Interlocked.Increment(ref _tradeCandleStickCountIncomplete); - } - } - - static void OnQuoteCandleStick(QuoteCandleStick quoteCandleStick) - { - if (quoteCandleStick.QuoteType == QuoteType.Ask) - if (quoteCandleStick.Complete) - Interlocked.Increment(ref _AskCandleStickCount); - else - Interlocked.Increment(ref _AskCandleStickCountIncomplete); - else - if (quoteCandleStick.Complete) - Interlocked.Increment(ref _BidCandleStickCount); - else - Interlocked.Increment(ref _BidCandleStickCountIncomplete); - } - - static void TimerCallback(object obj) - { - IEquitiesWebSocketClient client = (IEquitiesWebSocketClient) obj; - ClientStats stats = client.GetStats(); - Log("Data Messages = {0}, Text Messages = {1}, Queue Depth = {2}, Individual Events = {3}, Trades = {4}, Quotes = {5}", - stats.SocketDataMessages(), stats.SocketTextMessages(), stats.QueueDepth(), stats.EventCount(), stats.TradeCount(), stats.QuoteCount()); - if (maxTradeCount > 0) - { - Log("Most active trade: {0} ({1} updates)", maxCountTrade, maxTradeCount); - } - if (maxQuoteCount > 0) - { - Log("Most active quote: {0} ({1} updates)", maxCountQuote, maxQuoteCount); - } - if (_useTradeCandleSticks) - Log("TRADE CANDLESTICK STATS - TradeCandleSticks = {0}, TradeCandleSticksIncomplete = {1}", _tradeCandleStickCount, _tradeCandleStickCountIncomplete); - if (_useQuoteCandleSticks) - Log("QUOTE CANDLESTICK STATS - Asks = {0}, Bids = {1}, AsksIncomplete = {2}, BidsIncomplete = {3}", _AskCandleStickCount, _BidCandleStickCount, _AskCandleStickCountIncomplete, _BidCandleStickCountIncomplete); - } - - static void Cancel(object sender, ConsoleCancelEventArgs args) - { - Log("Stopping sample app"); - timer.Dispose(); - client.Stop(); - if (_useTradeCandleSticks || _useQuoteCandleSticks) - { - _candleStickClient.Stop(); - } - Environment.Exit(0); - } - - [MessageTemplateFormatMethod("messageTemplate")] - static void Log(string messageTemplate, params object[] propertyValues) - { - Serilog.Log.Information(messageTemplate, propertyValues); - } - - static void Main(string[] _) - { - Log("Starting sample app"); - Action onTrade = OnTrade; - Action onQuote = OnQuote; - - // //Subscribe the candlestick client to trade and/or quote events as well. It's important any method subscribed this way handles exceptions so as to not cause issues for other subscribers! - // _useTradeCandleSticks = true; - // _useQuoteCandleSticks = true; - // _candleStickClient = new CandleStickClient(OnTradeCandleStick, OnQuoteCandleStick, IntervalType.OneMinute, true, null, null, 0, false); - // onTrade += _candleStickClient.OnTrade; - // onQuote += _candleStickClient.OnQuote; - // _candleStickClient.Start(); - - // //You can either automatically load the config.json by doing nothing, or you can specify your own config and pass it in. - // //If you don't have a config.json, don't forget to also give Serilog a config so it can write to console - // Log.Logger = new LoggerConfiguration().WriteTo.Console(restrictedToMinimumLevel: Serilog.Events.LogEventLevel.Information).CreateLogger(); - // Config.Config config = new Config.Config(); - // config.Provider = Provider.REALTIME; - // config.ApiKey = ""; - // config.Symbols = new[] { "AAPL", "MSFT" }; - // config.NumThreads = 2; - // client = new Client(onTrade, onQuote, config); - - client = new Client(onTrade, onQuote); - timer = new Timer(TimerCallback, client, 10000, 10000); - client.Join(); //Load symbols from your config or config.json - //client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime - - // //You can also simulate a trading day by replaying a particular day's data. You can do this with the actual time between events, or without. - // DateTime yesterday = DateTime.Today - TimeSpan.FromDays(1); - // replayClient = new ReplayClient(onTrade, onQuote, yesterday, false, true, false, "data.csv"); //A client to replay a previous day's data - // timer = new Timer(TimerCallback, replayClient, 10000, 10000); - // replayClient.Join(); //Load symbols from your config or config.json - // //client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime - - Console.CancelKeyPress += new ConsoleCancelEventHandler(Cancel); - } - } -} diff --git a/IntrinioRealTimeSDK/sample-config.json b/IntrinioRealTimeSDK/sample-config.json deleted file mode 100644 index 16f432a..0000000 --- a/IntrinioRealTimeSDK/sample-config.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - //This file is not used and is only here for example purposes - //Please edit or make a config.json file. By default, a config.json is copied over on build from the IntrinioRealtimeMultiExchange project, so you can edit that one. - "Config": { - "ApiKey": "", - "NumThreads": 2, - "Provider": "REALTIME", - //"Provider": "DELAYED_SIP", - //"Provider": "MANUAL", - "Symbols": [ "AAPL", "MSFT", "GOOG" ] - //"Symbols": [ "lobby" ] - //"IPAddress": "1.2.3.4" - }, - "Serilog": { - "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Warning", - "System": "Warning" - } - }, - "WriteTo": [ - { "Name": "Console" } - ] - } -} \ No newline at end of file diff --git a/IntrinioRealtimeMultiExchange/CandleStickClient.fs b/IntrinioRealtimeMultiExchange/CandleStickClient.fs deleted file mode 100644 index 374b806..0000000 --- a/IntrinioRealtimeMultiExchange/CandleStickClient.fs +++ /dev/null @@ -1,433 +0,0 @@ -namespace Intrinio.Realtime.Equities - -open Intrinio -open Serilog -open System -open System.Runtime.InteropServices -open System.Collections.Concurrent -open System.Collections.Generic -open System.Threading -open FSharp.NativeInterop -open System.Runtime.CompilerServices - -module private CandleStickClientInline = - - [] - let inline private stackalloc<'a when 'a: unmanaged> (length: int): Span<'a> = - let p = NativePtr.stackalloc<'a> length |> NativePtr.toVoidPtr - Span<'a>(p, length) - - let inline internal getCurrentTimestamp(delay : float) : float = - (DateTime.UtcNow - DateTime.UnixEpoch.ToUniversalTime()).TotalSeconds - delay - - let inline internal getNearestModInterval(timestamp : float, interval: IntervalType) : float = - System.Convert.ToDouble(System.Convert.ToUInt64(timestamp) / System.Convert.ToUInt64(int interval)) * System.Convert.ToDouble((int interval)) - - let inline internal mergeTradeCandles (a : TradeCandleStick, b : TradeCandleStick) : TradeCandleStick option = - a.Merge(b) - Some(a) - - let inline internal mergeQuoteCandles (a : QuoteCandleStick, b : QuoteCandleStick) : QuoteCandleStick option = - a.Merge(b) - Some(a) - - let inline internal isAnomalous(marketCenter : char, condition : string) : bool = - marketCenter.Equals('L') && (condition.Equals("@ Zo", StringComparison.InvariantCultureIgnoreCase) || condition.Equals("@ To", StringComparison.InvariantCultureIgnoreCase) || condition.Equals("@ TW", StringComparison.InvariantCultureIgnoreCase)) - - let inline internal isDarkPoolMarketCenter(marketCenter : char) : bool = - marketCenter.Equals((char)0) || Char.IsWhiteSpace(marketCenter) || marketCenter.Equals('D') || marketCenter.Equals('E') - - let inline internal shouldFilterTrade(incomingTrade : Trade, useFiltering: bool) : bool = - useFiltering && (isDarkPoolMarketCenter incomingTrade.MarketCenter || isAnomalous(incomingTrade.MarketCenter, incomingTrade.Condition)) - - let inline internal shouldFilterQuote(incomingQuote : Quote, useFiltering) : bool = - useFiltering && isDarkPoolMarketCenter incomingQuote.MarketCenter - - let inline internal convertToTimestamp (input : DateTime) : float = - (input.ToUniversalTime() - DateTime.UnixEpoch.ToUniversalTime()).TotalSeconds - -type internal SymbolBucket = - val mutable TradeCandleStick : TradeCandleStick option - val mutable AskCandleStick : QuoteCandleStick option - val mutable BidCandleStick : QuoteCandleStick option - val Locker : ReaderWriterLockSlim - - new (tradeCandleStick : TradeCandleStick option, askCandleStick : QuoteCandleStick option, bidCandleStick : QuoteCandleStick option) = - { - TradeCandleStick = tradeCandleStick - AskCandleStick = askCandleStick - BidCandleStick = bidCandleStick - Locker = new ReaderWriterLockSlim() - } - -type CandleStickClient( - [)>] onTradeCandleStick : Action, - [)>] onQuoteCandleStick : Action, - interval : IntervalType, - broadcastPartialCandles : bool, - [)>] getHistoricalTradeCandleStick : Func, - [)>] getHistoricalQuoteCandleStick : Func, - sourceDelaySeconds : float, - useTradeFiltering : bool) = - - let ctSource : CancellationTokenSource = new CancellationTokenSource() - let useOnTradeCandleStick : bool = not (obj.ReferenceEquals(onTradeCandleStick,null)) - let useOnQuoteCandleStick : bool = not (obj.ReferenceEquals(onQuoteCandleStick,null)) - let useGetHistoricalTradeCandleStick : bool = not (obj.ReferenceEquals(getHistoricalTradeCandleStick,null)) - let useGetHistoricalQuoteCandleStick : bool = not (obj.ReferenceEquals(getHistoricalQuoteCandleStick,null)) - let initialDictionarySize : int = 3_601_579 //a close prime number greater than 2x the max expected size. There are usually around 1.5m option contracts. - let symbolBucketsLock : ReaderWriterLockSlim = new ReaderWriterLockSlim() - let lostAndFoundLock : ReaderWriterLockSlim = new ReaderWriterLockSlim() - let symbolBuckets : Dictionary = new Dictionary(initialDictionarySize) - let lostAndFound : Dictionary = new Dictionary(initialDictionarySize) - let flushBufferSeconds : float = 30.0 - - static let getSlot(key : string, dict : Dictionary, locker : ReaderWriterLockSlim) : SymbolBucket = - match dict.TryGetValue(key) with - | (true, value) -> value - | (false, _) -> - locker.EnterWriteLock() - try - match dict.TryGetValue(key) with - | (true, value) -> value - | (false, _) -> - let bucket : SymbolBucket = new SymbolBucket(Option.None, Option.None, Option.None) - dict.Add(key, bucket) - bucket - finally locker.ExitWriteLock() - - static let removeSlot(key : string, dict : Dictionary, locker : ReaderWriterLockSlim) : unit = - match dict.TryGetValue(key) with - | (false, _) -> () - | (true, _) -> - locker.EnterWriteLock() - try - match dict.TryGetValue(key) with - | (false, _) -> () - | (true, _) -> - dict.Remove(key) |> ignore - finally locker.ExitWriteLock() - - let createNewTradeCandle(trade : Trade, timestamp : float) : TradeCandleStick option = - let start : float = CandleStickClientInline.getNearestModInterval(timestamp, interval) - let freshCandle : TradeCandleStick option = Some(new TradeCandleStick(trade.Symbol, trade.Size, trade.Price, start, (start + System.Convert.ToDouble(int interval)), interval, timestamp)) - - if (useGetHistoricalTradeCandleStick && useOnTradeCandleStick) - then - try - let historical = getHistoricalTradeCandleStick.Invoke(freshCandle.Value.Symbol, freshCandle.Value.OpenTimestamp, freshCandle.Value.CloseTimestamp, freshCandle.Value.Interval) - match not (obj.ReferenceEquals(historical,null)) with - | false -> - freshCandle - | true -> - historical.MarkIncomplete() - CandleStickClientInline.mergeTradeCandles(historical, freshCandle.Value) - with :? Exception as e -> - Log.Error("Error retrieving historical TradeCandleStick: {0}; trade: {1}", e.Message, trade) - freshCandle - else - freshCandle - - let createNewQuoteCandle(quote : Quote, timestamp : float) : QuoteCandleStick option = - let start : float = CandleStickClientInline.getNearestModInterval(timestamp, interval) - let freshCandle : QuoteCandleStick option = Some(new QuoteCandleStick(quote.Symbol, quote.Price, quote.Type, start, (start + System.Convert.ToDouble(int interval)), interval, timestamp)) - - if (useGetHistoricalQuoteCandleStick && useOnQuoteCandleStick) - then - try - let historical = getHistoricalQuoteCandleStick.Invoke(freshCandle.Value.Symbol, freshCandle.Value.OpenTimestamp, freshCandle.Value.CloseTimestamp, freshCandle.Value.QuoteType, freshCandle.Value.Interval) - match not (obj.ReferenceEquals(historical,null)) with - | false -> - freshCandle - | true -> - historical.MarkIncomplete() - CandleStickClientInline.mergeQuoteCandles(historical, freshCandle.Value) - with :? Exception as e -> - Log.Error("Error retrieving historical QuoteCandleStick: {0}; quote: {1}", e.Message, quote) - freshCandle - else - freshCandle - - let addAskToLostAndFound(ask: Quote) : unit = - let ts : float = CandleStickClientInline.convertToTimestamp(ask.Timestamp) - let key : string = String.Format("{0}|{1}|{2}", ask.Symbol, CandleStickClientInline.getNearestModInterval(ts, interval), interval) - let bucket : SymbolBucket = getSlot(key, lostAndFound, lostAndFoundLock) - try - if useGetHistoricalQuoteCandleStick && useOnQuoteCandleStick - then - bucket.Locker.EnterWriteLock() - try - if (bucket.AskCandleStick.IsSome) - then - bucket.AskCandleStick.Value.Update(ask.Price, ts) - else - let start = CandleStickClientInline.getNearestModInterval(ts, interval) - bucket.AskCandleStick <- Some(new QuoteCandleStick(ask.Symbol, ask.Price, QuoteType.Ask, start, (start + System.Convert.ToDouble(int interval)), interval, ts)) - finally - bucket.Locker.ExitWriteLock() - with ex -> - Log.Warning("Error on handling late ask in CandleStick Client: {0}", ex.Message) - - let addBidToLostAndFound(bid: Quote) : unit = - let ts : float = CandleStickClientInline.convertToTimestamp(bid.Timestamp) - let key : string = String.Format("{0}|{1}|{2}", bid.Symbol, CandleStickClientInline.getNearestModInterval(ts, interval), interval) - let bucket : SymbolBucket = getSlot(key, lostAndFound, lostAndFoundLock) - try - if useGetHistoricalQuoteCandleStick && useOnQuoteCandleStick - then - bucket.Locker.EnterWriteLock() - try - if (bucket.BidCandleStick.IsSome) - then - bucket.BidCandleStick.Value.Update(bid.Price, ts) - else - let start = CandleStickClientInline.getNearestModInterval(ts, interval) - bucket.BidCandleStick <- Some(new QuoteCandleStick(bid.Symbol, bid.Price, QuoteType.Bid, start, (start + System.Convert.ToDouble(int interval)), interval, ts)) - finally - bucket.Locker.ExitWriteLock() - with ex -> - Log.Warning("Error on handling late bid in CandleStick Client: {0}", ex.Message) - - let addTradeToLostAndFound (trade: Trade) : unit = - let ts : float = CandleStickClientInline.convertToTimestamp(trade.Timestamp) - let key : string = String.Format("{0}|{1}|{2}", trade.Symbol, CandleStickClientInline.getNearestModInterval(ts, interval), interval) - let bucket : SymbolBucket = getSlot(key, lostAndFound, lostAndFoundLock) - try - if useGetHistoricalTradeCandleStick && useOnTradeCandleStick - then - bucket.Locker.EnterWriteLock() - try - if (bucket.TradeCandleStick.IsSome) - then - bucket.TradeCandleStick.Value.Update(trade.Size, trade.Price, ts) - else - let start = CandleStickClientInline.getNearestModInterval(ts, interval) - bucket.TradeCandleStick <- Some(new TradeCandleStick(trade.Symbol, trade.Size, trade.Price, start, (start + System.Convert.ToDouble(int interval)), interval, ts)) - finally - bucket.Locker.ExitWriteLock() - with ex -> - Log.Warning("Error on handling late trade in CandleStick Client: {0}", ex.Message) - - let onAsk(quote: Quote, bucket: SymbolBucket) : unit = - let ts : float = CandleStickClientInline.convertToTimestamp(quote.Timestamp) - if (bucket.AskCandleStick.IsSome && not (Double.IsNaN(quote.Price))) - then - if (bucket.AskCandleStick.Value.CloseTimestamp < ts) - then - bucket.AskCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - bucket.AskCandleStick <- createNewQuoteCandle(quote, ts) - elif (bucket.AskCandleStick.Value.OpenTimestamp <= ts) - then - bucket.AskCandleStick.Value.Update(quote.Price, ts) - if broadcastPartialCandles then onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - else //This is a late event. We already shipped the candle, so add to lost and found - addAskToLostAndFound(quote) - elif (bucket.AskCandleStick.IsNone && not (Double.IsNaN(quote.Price))) - then - bucket.AskCandleStick <- createNewQuoteCandle(quote, ts) - if broadcastPartialCandles then onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - - let onBid(quote: Quote, bucket : SymbolBucket) : unit = - let ts : float = CandleStickClientInline.convertToTimestamp(quote.Timestamp) - if (bucket.BidCandleStick.IsSome && not (Double.IsNaN(quote.Price))) - then - if (bucket.BidCandleStick.Value.CloseTimestamp < ts) - then - bucket.BidCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - bucket.BidCandleStick <- createNewQuoteCandle(quote, ts) - elif (bucket.BidCandleStick.Value.OpenTimestamp <= ts) - then - bucket.BidCandleStick.Value.Update(quote.Price, ts) - if broadcastPartialCandles then onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - else //This is a late event. We already shipped the candle, so add to lost and found - addBidToLostAndFound(quote) - elif (bucket.BidCandleStick.IsNone && not (Double.IsNaN(quote.Price))) - then - bucket.BidCandleStick <- createNewQuoteCandle(quote, ts) - if broadcastPartialCandles then onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - - let flushFn () : unit = - Log.Information("Starting candlestick expiration watcher...") - let ct = ctSource.Token - while not (ct.IsCancellationRequested) do - try - symbolBucketsLock.EnterReadLock() - let mutable keys : string list = [] - for key in symbolBuckets.Keys do - keys <- key::keys - symbolBucketsLock.ExitReadLock() - for key in keys do - let bucket : SymbolBucket = getSlot(key, symbolBuckets, symbolBucketsLock) - let flushThresholdTime : float = CandleStickClientInline.getCurrentTimestamp(sourceDelaySeconds) - flushBufferSeconds - bucket.Locker.EnterWriteLock() - try - if (useOnTradeCandleStick && bucket.TradeCandleStick.IsSome && (bucket.TradeCandleStick.Value.CloseTimestamp < flushThresholdTime)) - then - bucket.TradeCandleStick.Value.MarkComplete() - onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - bucket.TradeCandleStick <- Option.None - if (useOnQuoteCandleStick && bucket.AskCandleStick.IsSome && (bucket.AskCandleStick.Value.CloseTimestamp < flushThresholdTime)) - then - bucket.AskCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - bucket.AskCandleStick <- Option.None - if (useOnQuoteCandleStick && bucket.BidCandleStick.IsSome && (bucket.BidCandleStick.Value.CloseTimestamp < flushThresholdTime)) - then - bucket.BidCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - bucket.BidCandleStick <- Option.None - finally - bucket.Locker.ExitWriteLock() - if not (ct.IsCancellationRequested) - then - Thread.Sleep 1000 - with :? OperationCanceledException -> () - Log.Information("Stopping candlestick expiration watcher...") - - let flushThread : Thread = new Thread(new ThreadStart(flushFn)) - - let lostAndFoundFn () : unit = - Log.Information("Starting candlestick late event watcher...") - let ct = ctSource.Token - while not (ct.IsCancellationRequested) do - try - lostAndFoundLock.EnterReadLock() - let mutable keys : string list = [] - for key in lostAndFound.Keys do - keys <- key::keys - lostAndFoundLock.ExitReadLock() - for key in keys do - let bucket : SymbolBucket = getSlot(key, lostAndFound, lostAndFoundLock) - bucket.Locker.EnterWriteLock() - try - if (useGetHistoricalTradeCandleStick && useOnTradeCandleStick && bucket.TradeCandleStick.IsSome) - then - try - let historical = getHistoricalTradeCandleStick.Invoke(bucket.TradeCandleStick.Value.Symbol, bucket.TradeCandleStick.Value.OpenTimestamp, bucket.TradeCandleStick.Value.CloseTimestamp, bucket.TradeCandleStick.Value.Interval) - match not (obj.ReferenceEquals(historical,null)) with - | false -> - bucket.TradeCandleStick.Value.MarkComplete() - onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - bucket.TradeCandleStick <- Option.None - | true -> - bucket.TradeCandleStick <- CandleStickClientInline.mergeTradeCandles(historical, bucket.TradeCandleStick.Value) - bucket.TradeCandleStick.Value.MarkComplete() - onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - bucket.TradeCandleStick <- Option.None - with :? Exception as e -> - Log.Error("Error retrieving historical TradeCandleStick: {0}", e.Message) - bucket.TradeCandleStick.Value.MarkComplete() - onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - bucket.TradeCandleStick <- Option.None - else - bucket.TradeCandleStick <- Option.None - if (useGetHistoricalQuoteCandleStick && useOnQuoteCandleStick && bucket.AskCandleStick.IsSome) - then - try - let historical = getHistoricalQuoteCandleStick.Invoke(bucket.AskCandleStick.Value.Symbol, bucket.AskCandleStick.Value.OpenTimestamp, bucket.AskCandleStick.Value.CloseTimestamp, bucket.AskCandleStick.Value.QuoteType, bucket.AskCandleStick.Value.Interval) - match not (obj.ReferenceEquals(historical,null)) with - | false -> - bucket.AskCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - bucket.AskCandleStick <- Option.None - | true -> - bucket.AskCandleStick <- CandleStickClientInline.mergeQuoteCandles(historical, bucket.AskCandleStick.Value) - bucket.AskCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - bucket.AskCandleStick <- Option.None - with :? Exception as e -> - Log.Error("Error retrieving historical QuoteCandleStick: {0}", e.Message) - bucket.AskCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.AskCandleStick.Value) - bucket.AskCandleStick <- Option.None - else - bucket.AskCandleStick <- Option.None - if (useGetHistoricalQuoteCandleStick && useOnQuoteCandleStick && bucket.BidCandleStick.IsSome) - then - try - let historical = getHistoricalQuoteCandleStick.Invoke(bucket.BidCandleStick.Value.Symbol, bucket.BidCandleStick.Value.OpenTimestamp, bucket.BidCandleStick.Value.CloseTimestamp, bucket.BidCandleStick.Value.QuoteType, bucket.BidCandleStick.Value.Interval) - match not (obj.ReferenceEquals(historical,null)) with - | false -> - bucket.BidCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - bucket.BidCandleStick <- Option.None - | true -> - bucket.BidCandleStick <- CandleStickClientInline.mergeQuoteCandles(historical, bucket.BidCandleStick.Value) - bucket.BidCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - bucket.BidCandleStick <- Option.None - with :? Exception as e -> - Log.Error("Error retrieving historical QuoteCandleStick: {0}", e.Message) - bucket.BidCandleStick.Value.MarkComplete() - onQuoteCandleStick.Invoke(bucket.BidCandleStick.Value) - bucket.BidCandleStick <- Option.None - else - bucket.BidCandleStick <- Option.None - if bucket.TradeCandleStick.IsNone && bucket.AskCandleStick.IsNone && bucket.BidCandleStick.IsNone - then - removeSlot(key, lostAndFound, lostAndFoundLock) - finally - bucket.Locker.ExitWriteLock() - if not (ct.IsCancellationRequested) - then - Thread.Sleep 1000 - with :? OperationCanceledException -> () - Log.Information("Stopping candlestick late event watcher...") - - let lostAndFoundThread : Thread = new Thread(new ThreadStart(lostAndFoundFn)) - - member _.OnTrade(trade: Trade) : unit = - try - if useOnTradeCandleStick && (not (CandleStickClientInline.shouldFilterTrade(trade, useTradeFiltering))) - then - let bucket : SymbolBucket = getSlot(trade.Symbol, symbolBuckets, symbolBucketsLock) - try - let ts : float = CandleStickClientInline.convertToTimestamp(trade.Timestamp) - bucket.Locker.EnterWriteLock() - if (bucket.TradeCandleStick.IsSome) - then - if (bucket.TradeCandleStick.Value.CloseTimestamp < ts) - then - bucket.TradeCandleStick.Value.MarkComplete() - onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - bucket.TradeCandleStick <- createNewTradeCandle(trade, ts) - elif (bucket.TradeCandleStick.Value.OpenTimestamp <= ts) - then - bucket.TradeCandleStick.Value.Update(trade.Size, trade.Price, ts) - if broadcastPartialCandles then onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - else //This is a late trade. We already shipped the candle, so add to lost and found - addTradeToLostAndFound(trade) - else - bucket.TradeCandleStick <- createNewTradeCandle(trade, ts) - if broadcastPartialCandles then onTradeCandleStick.Invoke(bucket.TradeCandleStick.Value) - finally bucket.Locker.ExitWriteLock() - with ex -> - Log.Warning("Error on handling trade in CandleStick Client: {0}", ex.Message) - - member _.OnQuote(quote: Quote) : unit = - try - if useOnQuoteCandleStick && (not (CandleStickClientInline.shouldFilterQuote(quote, useTradeFiltering))) - then - let bucket : SymbolBucket = getSlot(quote.Symbol, symbolBuckets, symbolBucketsLock) - try - bucket.Locker.EnterWriteLock() - match quote.Type with - | QuoteType.Ask -> onAsk(quote, bucket) - | QuoteType.Bid -> onBid(quote, bucket) - | _ -> () - finally bucket.Locker.ExitWriteLock() - with ex -> - Log.Warning("Error on handling trade in CandleStick Client: {0}", ex.Message) - - member _.Start() : unit = - if not flushThread.IsAlive - then - flushThread.Start() - if not lostAndFoundThread.IsAlive - then - lostAndFoundThread.Start() - - member _.Stop() : unit = - ctSource.Cancel() \ No newline at end of file diff --git a/IntrinioRealtimeMultiExchange/Client.fs b/IntrinioRealtimeMultiExchange/Client.fs deleted file mode 100644 index 9c051ad..0000000 --- a/IntrinioRealtimeMultiExchange/Client.fs +++ /dev/null @@ -1,458 +0,0 @@ -namespace Intrinio.Realtime.Equities - -open Intrinio.Realtime.Equities -open Serilog -open System -open System.Runtime.InteropServices -open System.IO -open System.Net.Http -open System.Text -open System.Collections.Concurrent -open System.Collections.Generic -open System.Threading -open System.Threading.Tasks -open System.Net.Sockets -open WebSocket4Net -open Intrinio.Realtime.Equities.Config -open Serilog.Core - -type internal WebSocketState(ws: WebSocket) = - let mutable webSocket : WebSocket = ws - let mutable isReady : bool = false - let mutable isReconnecting : bool = false - let mutable lastReset : DateTime = DateTime.Now - - member _.WebSocket - with get() : WebSocket = webSocket - and set (ws:WebSocket) = webSocket <- ws - - member _.IsReady - with get() : bool = isReady - and set (ir:bool) = isReady <- ir - - member _.IsReconnecting - with get() : bool = isReconnecting - and set (ir:bool) = isReconnecting <- ir - - member _.LastReset : DateTime = lastReset - - member _.Reset() : unit = lastReset <- DateTime.Now - -type Client( - [)>] onTrade: Action, - [)>] onQuote : Action, - config : Config) = - let selfHealBackoffs : int[] = [| 10_000; 30_000; 60_000; 300_000; 600_000 |] - let empty : byte[] = Array.empty - let tLock : ReaderWriterLockSlim = new ReaderWriterLockSlim() - let wsLock : ReaderWriterLockSlim = new ReaderWriterLockSlim() - let mutable token : (string * DateTime) = (null, DateTime.Now) - let mutable wsState : WebSocketState = Unchecked.defaultof - let mutable dataMsgCount : uint64 = 0UL - let mutable dataEventCount : uint64 = 0UL - let mutable dataTradeCount : uint64 = 0UL - let mutable dataQuoteCount : uint64 = 0UL - let mutable textMsgCount : uint64 = 0UL - let channels : HashSet<(string*bool)> = new HashSet<(string*bool)>() - let ctSource : CancellationTokenSource = new CancellationTokenSource() - let data : ConcurrentQueue = new ConcurrentQueue() - let mutable tryReconnect : (unit -> unit) = fun () -> () - let httpClient : HttpClient = new HttpClient() - let useOnTrade : bool = not (obj.ReferenceEquals(onTrade,null)) - let useOnQuote : bool = not (obj.ReferenceEquals(onQuote,null)) - let logPrefix : string = String.Format("{0}: ", config.Provider.ToString()) - let clientInfoHeaderKey : string = "Client-Information" - let clientInfoHeaderValue : string = "IntrinioDotNetSDKv10.0" - let messageVersionHeaderKey : string = "UseNewEquitiesFormat" - let messageVersionHeaderValue : string = "v2" - let mainThreadPriority = Thread.CurrentThread.Priority //this is set outside of our scope - let's not interfere. - - [] - let logMessage(logLevel:LogLevel, messageTemplate:string, [] propertyValues:obj[]) : unit = - match logLevel with - | LogLevel.DEBUG -> Log.Debug(logPrefix + messageTemplate, propertyValues) - | LogLevel.INFORMATION -> Log.Information(logPrefix + messageTemplate, propertyValues) - | LogLevel.WARNING -> Log.Warning(logPrefix + messageTemplate, propertyValues) - | LogLevel.ERROR -> Log.Error(logPrefix + messageTemplate, propertyValues) - | _ -> failwith "LogLevel not specified!" - - let isReady() : bool = - wsLock.EnterReadLock() - try - if not (obj.ReferenceEquals(null, wsState)) && wsState.IsReady - then true - else false - finally wsLock.ExitReadLock() - - let getAuthUrl () : string = - match config.Provider with - | Provider.REALTIME -> "https://realtime-mx.intrinio.com/auth?api_key=" + config.ApiKey - | Provider.DELAYED_SIP -> "https://realtime-delayed-sip.intrinio.com/auth?api_key=" + config.ApiKey - | Provider.NASDAQ_BASIC -> "https://realtime-nasdaq-basic.intrinio.com/auth?api_key=" + config.ApiKey - | Provider.MANUAL -> "http://" + config.IPAddress + "/auth?api_key=" + config.ApiKey - | _ -> failwith "Provider not specified!" - - let getWebSocketUrl (token: string) : string = - match config.Provider with - | Provider.REALTIME -> "wss://realtime-mx.intrinio.com/socket/websocket?vsn=1.0.0&token=" + token - | Provider.DELAYED_SIP -> "wss://realtime-delayed-sip.intrinio.com/socket/websocket?vsn=1.0.0&token=" + token - | Provider.NASDAQ_BASIC -> "wss://realtime-nasdaq-basic.intrinio.com/socket/websocket?vsn=1.0.0&token=" + token - | Provider.MANUAL -> "ws://" + config.IPAddress + "/socket/websocket?vsn=1.0.0&token=" + token - | _ -> failwith "Provider not specified!" - - let getCustomSocketHeaders() : List> = - let headers : List> = new List>() - headers.Add(new KeyValuePair(clientInfoHeaderKey, clientInfoHeaderValue)) - headers.Add(new KeyValuePair(messageVersionHeaderKey, messageVersionHeaderValue)) - headers - - let parseTrade (bytes: ReadOnlySpan) : Trade = - let symbolLength : int = int32 (bytes.Item(2)) - let conditionLength : int = int32 (bytes.Item(26 + symbolLength)) - { - Symbol = Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)) - Price = (float (BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4)))) - Size = BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)) - Timestamp = DateTime.UnixEpoch + TimeSpan.FromTicks(int64 (BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)) - TotalVolume = BitConverter.ToUInt32(bytes.Slice(22 + symbolLength, 4)) - SubProvider = enum (int32 (bytes.Item(3 + symbolLength))) - MarketCenter = BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)) - Condition = if (conditionLength > 0) then Encoding.ASCII.GetString(bytes.Slice(27 + symbolLength, conditionLength)) else String.Empty - } - - let parseQuote (bytes: ReadOnlySpan) : Quote = - let symbolLength : int = int32 (bytes.Item(2)) - let conditionLength : int = int32 (bytes.Item(22 + symbolLength)) - { - Type = enum (int32 (bytes.Item(0))) - Symbol = Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)) - Price = (float (BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4)))) - Size = BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)) - Timestamp = DateTime.UnixEpoch + TimeSpan.FromTicks(int64 (BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)) - SubProvider = enum (int32 (bytes.Item(3 + symbolLength))) - MarketCenter = BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)) - Condition = if (conditionLength > 0) then Encoding.ASCII.GetString(bytes.Slice(23 + symbolLength, conditionLength)) else String.Empty - } - - let parseSocketMessage (bytes: byte[], startIndex: byref) : unit = - let mutable msgLength : int = 1 //default value in case corrupt array so we don't reprocess same bytes over and over. - try - let msgType : MessageType = enum (int32 bytes.[startIndex]) - msgLength <- int32 bytes.[startIndex + 1] - let chunk: ReadOnlySpan = new ReadOnlySpan(bytes, startIndex, msgLength) - match msgType with - | MessageType.Trade -> - if useOnTrade - then - let trade: Trade = parseTrade(chunk) - Interlocked.Increment(&dataTradeCount) |> ignore - trade |> onTrade.Invoke - | MessageType.Ask | MessageType.Bid -> - if useOnQuote - then - let quote: Quote = parseQuote(chunk) - Interlocked.Increment(&dataQuoteCount) |> ignore - quote |> onQuote.Invoke - | _ -> logMessage(LogLevel.WARNING, "Invalid MessageType: {0}", [|(int32 bytes.[startIndex])|]) - finally - startIndex <- startIndex + msgLength - - let threadFn () : unit = - let ct = ctSource.Token - Thread.CurrentThread.Priority <- enum (Math.Max(((int mainThreadPriority) - 1), 0)) //Set below main thread priority so doesn't interfere with main thread accepting messages. - let mutable datum : byte[] = Array.empty - while not (ct.IsCancellationRequested) do - try - if data.TryDequeue(&datum) then - // These are grouped (many) messages. - // The first byte tells us how many there are. - // From there, check the type at index 0 of each chunk to know how many bytes each message has. - let cnt = datum.[0] |> uint64 - Interlocked.Add(&dataEventCount, cnt) |> ignore - let mutable startIndex = 1 - for _ in 1UL .. cnt do - parseSocketMessage(datum, &startIndex) - else - Thread.Sleep(10) - with - | :? OperationCanceledException -> () - | exn -> logMessage(LogLevel.ERROR, "Error parsing message: {0}; {1}", [|exn.Message, exn.StackTrace|]) - - let threads : Thread[] = Array.init config.NumThreads (fun _ -> new Thread(new ThreadStart(threadFn))) - - let doBackoff(fn: unit -> bool) : unit = - let mutable i : int = 0 - let mutable backoff : int = selfHealBackoffs.[i] - let mutable success : bool = fn() - while not success do - Thread.Sleep(backoff) - i <- Math.Min(i + 1, selfHealBackoffs.Length - 1) - backoff <- selfHealBackoffs.[i] - success <- fn() - - let trySetToken() : bool = - logMessage(LogLevel.INFORMATION, "Authorizing...", [||]) - let authUrl : string = getAuthUrl() - async { - try - let! response = httpClient.GetAsync(authUrl) |> Async.AwaitTask - if (response.IsSuccessStatusCode) - then - let! _token = response.Content.ReadAsStringAsync() |> Async.AwaitTask - Interlocked.Exchange(&token, (_token, DateTime.Now)) |> ignore - return true - else - logMessage(LogLevel.WARNING, "Authorization Failure. Authorization server status code = {0}", [|response.StatusCode|]) - return false - with - | :? System.InvalidOperationException as exn -> - logMessage(LogLevel.ERROR, "Authorization Failure (bad URI): {0}", [|exn.Message|]) - return false - | :? System.Net.Http.HttpRequestException as exn -> - logMessage(LogLevel.ERROR, "Authoriztion Failure (bad network connection): {0}", [|exn.Message|]) - return false - | :? System.Threading.Tasks.TaskCanceledException as exn -> - logMessage(LogLevel.ERROR, "Authorization Failure (timeout): {0}", [|exn.Message|]) - return false - } |> Async.RunSynchronously - - let getToken() : string = - tLock.EnterUpgradeableReadLock() - try - tLock.EnterWriteLock() - try doBackoff(trySetToken) - finally tLock.ExitWriteLock() - fst token - finally tLock.ExitUpgradeableReadLock() - - let makeJoinMessage(tradesOnly: bool, symbol: string) : byte[] = - match symbol with - | "lobby" -> - let message : byte[] = Array.zeroCreate 11 //1 + 1 + 9 - message.[0] <- 74uy //type: join (74uy) or leave (76uy) - message.[1] <- (if tradesOnly then 1uy else 0uy) - Encoding.ASCII.GetBytes("$FIREHOSE").CopyTo(message, 2) - message - | _ -> - let message : byte[] = Array.zeroCreate (2 + symbol.Length) //1 + 1 + symbol.Length - message.[0] <- 74uy //type: join (74uy) or leave (76uy) - message.[1] <- (if tradesOnly then 1uy else 0uy) - Encoding.ASCII.GetBytes(symbol).CopyTo(message, 2) - message - - let makeLeaveMessage(symbol: string) : byte[] = - match symbol with - | "lobby" -> - let message : byte[] = Array.zeroCreate 10 // 1 (type = join) + 9 (symbol = $FIREHOSE) - message.[0] <- 76uy //type: join (74uy) or leave (76uy) - Encoding.ASCII.GetBytes("$FIREHOSE").CopyTo(message, 1) - message - | _ -> - let message : byte[] = Array.zeroCreate (1 + symbol.Length) //1 + symbol.Length - message.[0] <- 76uy //type: join (74uy) or leave (76uy) - Encoding.ASCII.GetBytes(symbol).CopyTo(message, 1) - message - - let onOpen (_ : EventArgs) : unit = - logMessage(LogLevel.INFORMATION, "Websocket - Connected", [||]) - wsLock.EnterWriteLock() - try - wsState.IsReady <- true - wsState.IsReconnecting <- false - for thread in threads do - if not thread.IsAlive - then thread.Start() - finally wsLock.ExitWriteLock() - if channels.Count > 0 - then - channels |> Seq.iter (fun (symbol: string, tradesOnly:bool) -> - let lastOnly : string = if tradesOnly then "true" else "false" - let message : byte[] = makeJoinMessage(tradesOnly, symbol) - logMessage(LogLevel.INFORMATION, "Websocket - Joining channel: {0} (trades only = {1})", [|symbol, lastOnly|]) - wsState.WebSocket.Send(message, 0, message.Length) ) - - let onClose (_ : EventArgs) : unit = - wsLock.EnterUpgradeableReadLock() - try - if not wsState.IsReconnecting - then - logMessage(LogLevel.INFORMATION, "Websocket - Closed", [||]) - wsLock.EnterWriteLock() - try wsState.IsReady <- false - finally wsLock.ExitWriteLock() - if (not ctSource.IsCancellationRequested) - then Task.Factory.StartNew(Action(tryReconnect)) |> ignore - finally wsLock.ExitUpgradeableReadLock() - - let (|Closed|Refused|Unavailable|Other|) (input:exn) = - if (input.GetType() = typeof) && - input.Message.StartsWith("A connection attempt failed because the connected party did not properly respond after a period of time") - then Closed - elif (input.GetType() = typeof) && - (input.Message = "No connection could be made because the target machine actively refused it.") - then Refused - elif input.Message.StartsWith("HTTP/1.1 503") - then Unavailable - else Other - - let onError (args : SuperSocket.ClientEngine.ErrorEventArgs) : unit = - let exn = args.Exception - match exn with - | Closed -> logMessage(LogLevel.WARNING, "Websocket - Error - Connection failed", [||]) - | Refused -> logMessage(LogLevel.WARNING, "Websocket - Error - Connection refused", [||]) - | Unavailable -> logMessage(LogLevel.WARNING, "Websocket - Error - Server unavailable", [||]) - | _ -> logMessage(LogLevel.ERROR, "Websocket - Error - {0}:{1}", [|exn.GetType(), exn.Message|]) - - let onDataReceived (args: DataReceivedEventArgs) : unit = - logMessage(LogLevel.DEBUG, "Websocket - Data received", [||]) - Interlocked.Increment(&dataMsgCount) |> ignore - data.Enqueue(args.Data) - - let onMessageReceived (args : MessageReceivedEventArgs) : unit = - logMessage(LogLevel.DEBUG, "Websocket - Message received", [||]) - Interlocked.Increment(&textMsgCount) |> ignore - logMessage(LogLevel.ERROR, "Error received: {0}", [|args.Message|]) - - let resetWebSocket(token: string) : unit = - logMessage(LogLevel.INFORMATION, "Websocket - Resetting", [||]) - let wsUrl : string = getWebSocketUrl(token) - let headers : List> = getCustomSocketHeaders() - //let ws : WebSocket = new WebSocket(wsUrl, customHeaderItems = headers) - let ws : WebSocket = new WebSocket(wsUrl, null, null, headers) - ws.Opened.Add(onOpen) - ws.Closed.Add(onClose) - ws.Error.Add(onError) - ws.DataReceived.Add(onDataReceived) - ws.MessageReceived.Add(onMessageReceived) - wsLock.EnterWriteLock() - try - wsState.WebSocket <- ws - wsState.Reset() - finally wsLock.ExitWriteLock() - ws.Open() - - let initializeWebSockets(token: string) : unit = - wsLock.EnterWriteLock() - try - logMessage(LogLevel.INFORMATION, "Websocket - Connecting...", [||]) - let wsUrl : string = getWebSocketUrl(token) - let headers : List> = getCustomSocketHeaders() - //let ws : WebSocket = new WebSocket(wsUrl, customHeaderItems = headers) - let ws : WebSocket = new WebSocket(wsUrl, null, null, headers) - ws.Opened.Add(onOpen) - ws.Closed.Add(onClose) - ws.Error.Add(onError) - ws.DataReceived.Add(onDataReceived) - ws.MessageReceived.Add(onMessageReceived) - wsState <- new WebSocketState(ws) - finally wsLock.ExitWriteLock() - wsState.WebSocket.Open() - - let join(symbol: string, tradesOnly: bool) : unit = - let lastOnly : string = if tradesOnly then "true" else "false" - if channels.Add((symbol, tradesOnly)) - then - let message : byte[] = makeJoinMessage(tradesOnly, symbol) - logMessage(LogLevel.INFORMATION, "Websocket - Joining channel: {0} (trades only = {1})", [|symbol, lastOnly|]) - try wsState.WebSocket.Send(message, 0, message.Length) - with _ -> channels.Remove((symbol, tradesOnly)) |> ignore - - let leave(symbol: string, tradesOnly: bool) : unit = - let lastOnly : string = if tradesOnly then "true" else "false" - if channels.Remove((symbol, tradesOnly)) - then - let message : byte[] = makeLeaveMessage(symbol) - logMessage(LogLevel.INFORMATION, "Websocket - Leaving channel: {0} (trades only = {1})", [|symbol, lastOnly|]) - try wsState.WebSocket.Send(message, 0, message.Length) - with _ -> () - - do - config.Validate() - httpClient.Timeout <- TimeSpan.FromSeconds(5.0) - httpClient.DefaultRequestHeaders.Add(clientInfoHeaderKey, clientInfoHeaderValue) - tryReconnect <- fun () -> - let reconnectFn () : bool = - logMessage(LogLevel.INFORMATION, "Websocket - Reconnecting...", [||]) - if wsState.IsReady then true - else - wsLock.EnterWriteLock() - try wsState.IsReconnecting <- true - finally wsLock.ExitWriteLock() - let _token : string = getToken() - resetWebSocket(_token) - false - doBackoff(reconnectFn) - let _token : string = getToken() - initializeWebSockets(_token) - - new ([)>] onTrade: Action) = - Client(onTrade, null, LoadConfig()) - - new ([)>] onQuote : Action) = - Client(null, onQuote, LoadConfig()) - - new ([)>] onTrade: Action, [)>] onQuote : Action) = - Client(onTrade, onQuote, LoadConfig()) - - interface IEquitiesWebSocketClient with - member this.Join() : unit = - while not(isReady()) do Thread.Sleep(1000) - let symbolsToAdd : HashSet<(string*bool)> = - config.Symbols - |> Seq.map(fun (symbol:string) -> (symbol, config.TradesOnly)) - |> fun (symbols:seq<(string*bool)>) -> new HashSet<(string*bool)>(symbols) - symbolsToAdd.ExceptWith(channels) - for symbol in symbolsToAdd do join(symbol) - - member this.Join(symbol: string, ?tradesOnly: bool) : unit = - let t: bool = - match tradesOnly with - | Some(v:bool) -> v || config.TradesOnly - | None -> false || config.TradesOnly - while not(isReady()) do Thread.Sleep(1000) - if not (channels.Contains((symbol, t))) - then join(symbol, t) - - member this.Join(symbols: string[], ?tradesOnly: bool) : unit = - let t: bool = - match tradesOnly with - | Some(v:bool) -> v || config.TradesOnly - | None -> false || config.TradesOnly - while not(isReady()) do Thread.Sleep(1000) - let symbolsToAdd : HashSet<(string*bool)> = - symbols - |> Seq.map(fun (symbol:string) -> (symbol,t)) - |> fun (_symbols:seq<(string*bool)>) -> new HashSet<(string*bool)>(_symbols) - symbolsToAdd.ExceptWith(channels) - for symbol in symbolsToAdd do join(symbol) - - member this.Leave() : unit = - for channel in channels do leave(channel) - - member this.Leave(symbol: string) : unit = - let matchingChannels : seq<(string*bool)> = channels |> Seq.where (fun (_symbol:string, _:bool) -> _symbol = symbol) - for channel in matchingChannels do leave(channel) - - member this.Leave(symbols: string[]) : unit = - let _symbols : HashSet = new HashSet(symbols) - let matchingChannels : seq<(string*bool)> = channels |> Seq.where(fun (symbol:string, _:bool) -> _symbols.Contains(symbol)) - for channel in matchingChannels do leave(channel) - - member this.Stop() : unit = - for channel in channels do leave(channel) - Thread.Sleep(1000) - wsLock.EnterWriteLock() - try wsState.IsReady <- false - finally wsLock.ExitWriteLock() - ctSource.Cancel () - logMessage(LogLevel.INFORMATION, "Websocket - Closing...", [||]) - wsState.WebSocket.Close() - for thread in threads do thread.Join() - logMessage(LogLevel.INFORMATION, "Stopped", [||]) - - member this.GetStats() : ClientStats = - new ClientStats(Interlocked.Read(&dataMsgCount), Interlocked.Read(&textMsgCount), data.Count, Interlocked.Read(&dataEventCount), Interlocked.Read(&dataTradeCount), Interlocked.Read(&dataQuoteCount)) - - [] - member this.Log(messageTemplate:string, [] propertyValues:obj[]) : unit = - Log.Information(messageTemplate, propertyValues) \ No newline at end of file diff --git a/IntrinioRealtimeMultiExchange/Config.fs b/IntrinioRealtimeMultiExchange/Config.fs deleted file mode 100644 index 8a1549d..0000000 --- a/IntrinioRealtimeMultiExchange/Config.fs +++ /dev/null @@ -1,41 +0,0 @@ -namespace Intrinio.Realtime.Equities - -module Config = - - open System - open Serilog - open System.IO - open Microsoft.Extensions.Configuration - - type Config () = - let mutable apiKey : string = String.Empty - let mutable provider : Provider = Provider.NONE - let mutable ipAddress : string = String.Empty - let mutable numThreads : int = 2 - - member this.ApiKey with get () : string = apiKey and set (value : string) = apiKey <- value - member this.Provider with get () : Provider = provider and set (value : Provider) = provider <- value - member this.IPAddress with get () : string = ipAddress and set (value : string) = ipAddress <- value - member val Symbols: string[] = [||] with get, set - member val TradesOnly: bool = false with get, set - member this.NumThreads with get () : int = numThreads and set (value : int) = numThreads <- value - - member _.Validate() : unit = - if String.IsNullOrWhiteSpace(apiKey) - then failwith "You must provide a valid API key" - if (provider = Provider.NONE) - then failwith "You must specify a valid 'provider'" - if ((provider = Provider.MANUAL) && (String.IsNullOrWhiteSpace(ipAddress))) - then failwith "You must specify an IP address for manual configuration" - if (numThreads <= 0) - then failwith "You must specify a valid 'NumThreads'" - - let LoadConfig() = - Log.Information("Loading application configuration") - let _config = ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("config.json").Build() - Log.Logger <- LoggerConfiguration().ReadFrom.Configuration(_config).CreateLogger() - let mutable config = new Config() - for (KeyValue(key,value)) in _config.AsEnumerable() do Log.Debug("Key: {0}, Value:{1}", key, value) - _config.Bind("Config", config) - config.Validate() - config diff --git a/IntrinioRealtimeMultiExchange/IntrinioRealtimeMultiExchange.fsproj b/IntrinioRealtimeMultiExchange/IntrinioRealtimeMultiExchange.fsproj deleted file mode 100644 index 641e672..0000000 --- a/IntrinioRealtimeMultiExchange/IntrinioRealtimeMultiExchange.fsproj +++ /dev/null @@ -1,41 +0,0 @@ - - - - net8.0 - true - - - - true - - - - - - - - - - - - - PreserveNewest - - - - - - - - - - - - - - - - - - - diff --git a/IntrinioRealtimeMultiExchange/ReplayClient.fs b/IntrinioRealtimeMultiExchange/ReplayClient.fs deleted file mode 100644 index e2c78ff..0000000 --- a/IntrinioRealtimeMultiExchange/ReplayClient.fs +++ /dev/null @@ -1,471 +0,0 @@ -namespace Intrinio.Realtime.Equities - -open Intrinio.Realtime.Equities -open Intrinio.SDK.Model -open Serilog -open System -open System.Runtime.InteropServices -open System.IO -open System.Net -open System.Net.Http -open System.Text -open System.Collections.Concurrent -open System.Collections.Generic -open System.Threading -open System.Threading.Tasks -open Intrinio.Realtime.Equities.Config -open Serilog.Core - -type ReplayClient( - [)>] onTrade: Action, - [)>] onQuote : Action, - config : Config, - date : DateTime, - withSimulatedDelay : bool, - deleteFileWhenDone : bool, - writeToCsv : bool, - csvFilePath : string) = - let empty : byte[] = Array.empty - let mutable dataMsgCount : uint64 = 0UL - let mutable dataEventCount : uint64 = 0UL - let mutable dataTradeCount : uint64 = 0UL - let mutable dataQuoteCount : uint64 = 0UL - let mutable textMsgCount : uint64 = 0UL - let channels : HashSet<(string*bool)> = new HashSet<(string*bool)>() - let ctSource : CancellationTokenSource = new CancellationTokenSource() - let data : ConcurrentQueue = new ConcurrentQueue() - let useOnTrade : bool = not (obj.ReferenceEquals(onTrade,null)) - let useOnQuote : bool = not (obj.ReferenceEquals(onQuote,null)) - let logPrefix : string = String.Format("{0}: ", config.Provider.ToString()) - let csvLock : Object = new Object(); - - let logMessage(logLevel:LogLevel, messageTemplate:string, [] propertyValues:obj[]) : unit = - match logLevel with - | LogLevel.DEBUG -> Log.Debug(logPrefix + messageTemplate, propertyValues) - | LogLevel.INFORMATION -> Log.Information(logPrefix + messageTemplate, propertyValues) - | LogLevel.WARNING -> Log.Warning(logPrefix + messageTemplate, propertyValues) - | LogLevel.ERROR -> Log.Error(logPrefix + messageTemplate, propertyValues) - | _ -> failwith "LogLevel not specified!" - - let parseTimeReceived(bytes: ReadOnlySpan) : DateTime = - DateTime.UnixEpoch + TimeSpan.FromTicks((int64)(BitConverter.ToUInt64(bytes) / 100UL)); - - let parseTrade (bytes: ReadOnlySpan) : Trade = - let symbolLength : int = int32 (bytes.Item(2)) - let conditionLength : int = int32 (bytes.Item(26 + symbolLength)) - { - Symbol = Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)) - Price = (float (BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4)))) - Size = BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)) - Timestamp = DateTime.UnixEpoch + TimeSpan.FromTicks(int64 (BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)) - TotalVolume = BitConverter.ToUInt32(bytes.Slice(22 + symbolLength, 4)) - SubProvider = enum (int32 (bytes.Item(3 + symbolLength))) - MarketCenter = BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)) - Condition = if (conditionLength > 0) then Encoding.ASCII.GetString(bytes.Slice(27 + symbolLength, conditionLength)) else String.Empty - } - - let parseQuote (bytes: ReadOnlySpan) : Quote = - let symbolLength : int = int32 (bytes.Item(2)) - let conditionLength : int = int32 (bytes.Item(22 + symbolLength)) - { - Type = enum (int32 (bytes.Item(0))) - Symbol = Encoding.ASCII.GetString(bytes.Slice(3, symbolLength)) - Price = (float (BitConverter.ToSingle(bytes.Slice(6 + symbolLength, 4)))) - Size = BitConverter.ToUInt32(bytes.Slice(10 + symbolLength, 4)) - Timestamp = DateTime.UnixEpoch + TimeSpan.FromTicks(int64 (BitConverter.ToUInt64(bytes.Slice(14 + symbolLength, 8)) / 100UL)) - SubProvider = enum (int32 (bytes.Item(3 + symbolLength))) - MarketCenter = BitConverter.ToChar(bytes.Slice(4 + symbolLength, 2)) - Condition = if (conditionLength > 0) then Encoding.ASCII.GetString(bytes.Slice(23 + symbolLength, conditionLength)) else String.Empty - } - - let writeRowToOpenCsvWithoutLock(row : IEnumerable) : unit = - let mutable first : bool = true - use fs : FileStream = new FileStream(csvFilePath, FileMode.Append); - use tw : TextWriter = new StreamWriter(fs); - for s : string in row do - if (not first) - then - tw.Write(","); - else - first <- false; - tw.Write($"\"{s}\""); - tw.WriteLine(); - - let writeRowToOpenCsvWithLock(row : IEnumerable) : unit = - lock csvLock (fun () -> writeRowToOpenCsvWithoutLock(row)) - - let doubleRoundSecRule612(value : float) : string = - if (value >= 1.0) - then - value.ToString("0.00") - else - value.ToString("0.0000"); - - let mapTradeToRow(trade : Trade) : IEnumerable = - seq{ - yield MessageType.Trade.ToString(); - yield trade.Symbol; - yield doubleRoundSecRule612(trade.Price); - yield trade.Size.ToString(); - yield trade.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffK"); - yield trade.SubProvider.ToString(); - yield trade.MarketCenter.ToString(); - yield trade.Condition; - yield trade.TotalVolume.ToString(); - } - - let writeTradeToCsv(trade : Trade) : unit = - writeRowToOpenCsvWithLock(mapTradeToRow(trade)) - - let mapQuoteToRow(quote : Quote) : IEnumerable = - seq{ - yield quote.Type.ToString(); - yield quote.Symbol; - yield doubleRoundSecRule612(quote.Price); - yield quote.Size.ToString(); - yield quote.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffK"); - yield quote.SubProvider.ToString(); - yield quote.MarketCenter.ToString(); - yield quote.Condition; - } - - let writeQuoteToCsv(quote : Quote) : unit = - writeRowToOpenCsvWithLock(mapQuoteToRow(quote)); - - let writeHeaderRow() : unit = - writeRowToOpenCsvWithLock ([|"Type"; "Symbol"; "Price"; "Size"; "Timestamp"; "SubProvider"; "MarketCenter"; "Condition"; "TotalVolume"|]); - - let threadFn () : unit = - let ct = ctSource.Token - let mutable datum : Tick = new Tick(DateTime.Now, Option.None, Option.None) //initial throw away value - while not (ct.IsCancellationRequested) do - try - if data.TryDequeue(&datum) then - match datum.IsTrade() with - | true -> - if useOnTrade - then - Interlocked.Increment(&dataTradeCount) |> ignore - datum.Trade() |> onTrade.Invoke - | false -> - if useOnQuote - then - Interlocked.Increment(&dataQuoteCount) |> ignore - datum.Quote() |> onQuote.Invoke - else - Thread.Sleep(1) - with - | :? OperationCanceledException -> () - | exn -> logMessage(LogLevel.ERROR, "Error parsing message: {0}; {1}", [|exn.Message, exn.StackTrace|]) - - /// - /// The results of this should be streamed and not ToList-ed. - /// - /// - /// - /// - let replayTickFileWithoutDelay(fullFilePath : string, byteBufferSize : int, ct : CancellationToken) : IEnumerable = - if File.Exists(fullFilePath) - then - seq { - use fRead : FileStream = new FileStream(fullFilePath, FileMode.Open, FileAccess.Read, FileShare.None) - - if (fRead.CanRead) - then - let mutable readResult : int = fRead.ReadByte() //This is message type - while (readResult <> -1) do - if not ct.IsCancellationRequested - then - let eventBuffer : byte[] = Array.zeroCreate byteBufferSize - let timeReceivedBuffer: byte[] = Array.zeroCreate 8 - let eventSpanBuffer : ReadOnlySpan = new ReadOnlySpan(eventBuffer) - let timeReceivedSpanBuffer : ReadOnlySpan = new ReadOnlySpan(timeReceivedBuffer) - eventBuffer[0] <- (byte) readResult //This is message type - eventBuffer[1] <- (byte) (fRead.ReadByte()) //This is message length, including this and the previous byte. - let bytesRead : int = fRead.Read(eventBuffer, 2, (System.Convert.ToInt32(eventBuffer[1])-2)) //read the rest of the message - let timeBytesRead : int = fRead.Read(timeReceivedBuffer, 0, 8) //get the time received - let timeReceived : DateTime = parseTimeReceived(timeReceivedSpanBuffer) - - match (enum (System.Convert.ToInt32(eventBuffer[0]))) with - | MessageType.Trade -> - let trade : Trade = parseTrade(eventSpanBuffer); - if (channels.Contains ("lobby", true) || channels.Contains ("lobby", false) || channels.Contains (trade.Symbol, true) || channels.Contains (trade.Symbol, false)) - then - if writeToCsv - then - writeTradeToCsv trade; - yield new Tick(timeReceived, Some(trade), Option.None); - | MessageType.Ask - | MessageType.Bid -> - let quote : Quote = parseQuote(eventSpanBuffer); - if (channels.Contains ("lobby", false) || channels.Contains (quote.Symbol, false)) - then - if writeToCsv - then - writeQuoteToCsv quote; - yield new Tick(timeReceived, Option.None, Some(quote)); - | _ -> logMessage(LogLevel.ERROR, "Invalid MessageType: {0}", [|eventBuffer[0]|]); - - //Set up the next iteration - readResult <- fRead.ReadByte(); - else readResult <- -1; - else - raise (FileLoadException("Unable to read replay file.")); - } - else - Array.Empty() - - /// - /// The results of this should be streamed and not ToList-ed. - /// - /// - /// - /// - let replayTickFileWithDelay(fullFilePath : string, byteBufferSize : int, ct : CancellationToken) : IEnumerable = - let start : int64 = DateTime.UtcNow.Ticks; - let mutable offset : int64 = 0L; - seq { - for tick : Tick in replayTickFileWithoutDelay(fullFilePath, byteBufferSize, ct) do - if (offset = 0L) - then - offset <- start - tick.TimeReceived().Ticks - - if not ct.IsCancellationRequested - then - System.Threading.SpinWait.SpinUntil(fun () -> ((tick.TimeReceived().Ticks + offset) <= DateTime.UtcNow.Ticks)); - yield tick - } - - let mapSubProviderToApiValue(subProvider : SubProvider) : string = - match subProvider with - | SubProvider.IEX -> "iex" - | SubProvider.UTP -> "utp_delayed" - | SubProvider.CTA_A -> "cta_a_delayed" - | SubProvider.CTA_B -> "cta_b_delayed" - | SubProvider.OTC -> "otc_delayed" - | SubProvider.NASDAQ_BASIC -> "nasdaq_basic" - | _ -> "iex" - - let mapProviderToSubProviders(provider : Intrinio.Realtime.Equities.Provider) : SubProvider[] = - match provider with - | Provider.NONE -> [||] - | Provider.MANUAL -> [||] - | Provider.REALTIME -> [|SubProvider.IEX|] - | Provider.DELAYED_SIP -> [|SubProvider.UTP; SubProvider.CTA_A; SubProvider.CTA_B; SubProvider.OTC|] - | Provider.NASDAQ_BASIC -> [|SubProvider.NASDAQ_BASIC|] - | _ -> [||] - - let fetchReplayFile(subProvider : SubProvider) : string = - let api : Intrinio.SDK.Api.SecurityApi = new Intrinio.SDK.Api.SecurityApi() - if not (api.Configuration.ApiKey.ContainsKey("api_key")) - then - api.Configuration.ApiKey.Add("api_key", config.ApiKey) - - try - let result : SecurityReplayFileResult = api.GetSecurityReplayFile(mapSubProviderToApiValue(subProvider), date) - let decodedUrl : string = result.Url.Replace(@"\u0026", "&") - let tempDir : string = System.IO.Path.GetTempPath() - let fileName : string = Path.Combine(tempDir, result.Name) - - use outputFile = new System.IO.FileStream(fileName,System.IO.FileMode.Create) - ( - use httpClient = new HttpClient() - ( - httpClient.Timeout <- TimeSpan.FromHours(1) - httpClient.BaseAddress <- new Uri(decodedUrl) - use response : HttpResponseMessage = httpClient.GetAsync(decodedUrl, HttpCompletionOption.ResponseHeadersRead).Result - ( - use streamToReadFrom : Stream = response.Content.ReadAsStreamAsync().Result - ( - streamToReadFrom.CopyTo outputFile - ) - ) - ) - ) - - fileName - with | :? Exception as e -> - logMessage(LogLevel.ERROR, "Error while fetching {0} file: {1}", [|subProvider.ToString(), e.Message|]) - null - - let fillNextTicks(enumerators : IEnumerator[], nextTicks : Option[]) : unit = - for i = 0 to (nextTicks.Length-1) do - if nextTicks.[i].IsNone && enumerators.[i].MoveNext() - then - nextTicks.[i] <- Some(enumerators.[i].Current) - - let pullNextTick(nextTicks : Option[]) : Option = - let mutable pullIndex : int = 0 - let mutable t : DateTime = DateTime.MaxValue - for i = 0 to (nextTicks.Length-1) do - if nextTicks.[i].IsSome && nextTicks.[i].Value.TimeReceived() < t - then - pullIndex <- i - t <- nextTicks.[i].Value.TimeReceived() - - let pulledTick = nextTicks.[pullIndex] - nextTicks.[pullIndex] <- Option.None - pulledTick - - let hasAnyValue(nextTicks : Option[]) : bool = - let mutable hasValue : bool = false - for i = 0 to (nextTicks.Length-1) do - if nextTicks.[i].IsSome - then - hasValue <- true - hasValue - - let replayFileGroupWithoutDelay(tickGroup : IEnumerable[], ct : CancellationToken) : IEnumerable = - seq{ - let nextTicks : Option[] = Array.zeroCreate(tickGroup.Length) - let enumerators : IEnumerator[] = Array.zeroCreate(tickGroup.Length) - for i = 0 to (tickGroup.Length-1) do - enumerators.[i] <- tickGroup.[i].GetEnumerator() - - fillNextTicks(enumerators, nextTicks) - while hasAnyValue(nextTicks) do - let nextTick : Option = pullNextTick(nextTicks) - if nextTick.IsSome - then yield nextTick.Value - fillNextTicks(enumerators, nextTicks) - } - - let replayFileGroupWithDelay(tickGroup : IEnumerable[], ct : CancellationToken) : IEnumerable = - seq { - let start : int64 = DateTime.UtcNow.Ticks; - let mutable offset : int64 = 0L; - for tick : Tick in replayFileGroupWithoutDelay(tickGroup, ct) do - if (offset = 0L) - then - offset <- start - tick.TimeReceived().Ticks - - if not ct.IsCancellationRequested - then - System.Threading.SpinWait.SpinUntil(fun () -> ((tick.TimeReceived().Ticks + offset) <= DateTime.UtcNow.Ticks)); - yield tick - } - - let replayThreadFn () : unit = - let ct : CancellationToken = ctSource.Token - let subProviders : SubProvider[] = mapProviderToSubProviders(config.Provider) - let replayFiles : string[] = Array.zeroCreate(subProviders.Length) - let allTicks : IEnumerable[] = Array.zeroCreate(subProviders.Length) - - try - for i = 0 to subProviders.Length-1 do - logMessage(LogLevel.INFORMATION, "Downloading Replay file for {0} on {1}...", [|subProviders.[i].ToString(); date.Date.ToString()|]) - replayFiles.[i] <- fetchReplayFile(subProviders.[i]) - logMessage(LogLevel.INFORMATION, "Downloaded Replay file to: {0}", [|replayFiles.[i]|]) - allTicks.[i] <- replayTickFileWithoutDelay(replayFiles.[i], 100, ct) - - let aggregatedTicks : IEnumerable = - if withSimulatedDelay - then replayFileGroupWithDelay(allTicks, ct) - else replayFileGroupWithoutDelay(allTicks, ct) - - for tick : Tick in aggregatedTicks do - if not ct.IsCancellationRequested - then - Interlocked.Increment(&dataEventCount) |> ignore - Interlocked.Increment(&dataMsgCount) |> ignore - data.Enqueue(tick) - - with | :? Exception as e -> logMessage(LogLevel.ERROR, "Error while replaying file: {0}", [|e.Message|]) - - if deleteFileWhenDone - then - for deleteFilePath in replayFiles do - if File.Exists deleteFilePath - then - logMessage(LogLevel.INFORMATION, "Deleting Replay file: {0}", [|deleteFilePath|]) - File.Delete(deleteFilePath) - - let threads : Thread[] = Array.init config.NumThreads (fun _ -> new Thread(new ThreadStart(threadFn))) - - let replayThread : Thread = new Thread(new ThreadStart(replayThreadFn)) - - let join(symbol: string, tradesOnly: bool) : unit = - let lastOnly : string = if tradesOnly then "true" else "false" - if channels.Add((symbol, tradesOnly)) - then - logMessage(LogLevel.INFORMATION, "Websocket - Joining channel: {0} (trades only = {1})", [|symbol, lastOnly|]) - - let leave(symbol: string, tradesOnly: bool) : unit = - let lastOnly : string = if tradesOnly then "true" else "false" - if channels.Remove((symbol, tradesOnly)) - then - logMessage(LogLevel.INFORMATION, "Websocket - Leaving channel: {0} (trades only = {1})", [|symbol, lastOnly|]) - - do - config.Validate() - for thread : Thread in threads do - thread.Start() - if writeToCsv - then - writeHeaderRow(); - replayThread.Start() - - new ([)>] onTrade: Action, date : DateTime, withSimulatedDelay : bool, deleteFileWhenDone : bool, writeToCsv : bool, csvFilePath : string) = - ReplayClient(onTrade, null, LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) - - new ([)>] onQuote : Action, date : DateTime, withSimulatedDelay : bool, deleteFileWhenDone : bool, writeToCsv : bool, csvFilePath : string) = - ReplayClient(null, onQuote, LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) - - new ([)>] onTrade: Action, [)>] onQuote : Action, date : DateTime, withSimulatedDelay : bool, deleteFileWhenDone : bool, writeToCsv : bool, csvFilePath : string) = - ReplayClient(onTrade, onQuote, LoadConfig(), date, withSimulatedDelay, deleteFileWhenDone, writeToCsv, csvFilePath) - - interface IEquitiesWebSocketClient with - member this.Join() : unit = - let symbolsToAdd : HashSet<(string*bool)> = - config.Symbols - |> Seq.map(fun (symbol:string) -> (symbol, config.TradesOnly)) - |> fun (symbols:seq<(string*bool)>) -> new HashSet<(string*bool)>(symbols) - symbolsToAdd.ExceptWith(channels) - for symbol in symbolsToAdd do join(symbol) - - member this.Join(symbol: string, ?tradesOnly: bool) : unit = - let t: bool = - match tradesOnly with - | Some(v:bool) -> v || config.TradesOnly - | None -> false || config.TradesOnly - if not (channels.Contains((symbol, t))) - then join(symbol, t) - - member this.Join(symbols: string[], ?tradesOnly: bool) : unit = - let t: bool = - match tradesOnly with - | Some(v:bool) -> v || config.TradesOnly - | None -> false || config.TradesOnly - let symbolsToAdd : HashSet<(string*bool)> = - symbols - |> Seq.map(fun (symbol:string) -> (symbol,t)) - |> fun (_symbols:seq<(string*bool)>) -> new HashSet<(string*bool)>(_symbols) - symbolsToAdd.ExceptWith(channels) - for symbol in symbolsToAdd do join(symbol) - - member this.Leave() : unit = - for channel in channels do leave(channel) - - member this.Leave(symbol: string) : unit = - let matchingChannels : seq<(string*bool)> = channels |> Seq.where (fun (_symbol:string, _:bool) -> _symbol = symbol) - for channel in matchingChannels do leave(channel) - - member this.Leave(symbols: string[]) : unit = - let _symbols : HashSet = new HashSet(symbols) - let matchingChannels : seq<(string*bool)> = channels |> Seq.where(fun (symbol:string, _:bool) -> _symbols.Contains(symbol)) - for channel in matchingChannels do leave(channel) - - member this.Stop() : unit = - for channel in channels do leave(channel) - ctSource.Cancel () - logMessage(LogLevel.INFORMATION, "Websocket - Closing...", [||]) - for thread in threads do thread.Join() - replayThread.Join() - logMessage(LogLevel.INFORMATION, "Stopped", [||]) - - member this.GetStats() : ClientStats = - new ClientStats(Interlocked.Read(&dataMsgCount), Interlocked.Read(&textMsgCount), data.Count, Interlocked.Read(&dataEventCount), Interlocked.Read(&dataTradeCount), Interlocked.Read(&dataQuoteCount)) - - [] - member this.Log(messageTemplate:string, [] propertyValues:obj[]) : unit = - Log.Information(messageTemplate, propertyValues) \ No newline at end of file diff --git a/IntrinioRealtimeMultiExchange/Types.fs b/IntrinioRealtimeMultiExchange/Types.fs deleted file mode 100644 index 843ff07..0000000 --- a/IntrinioRealtimeMultiExchange/Types.fs +++ /dev/null @@ -1,587 +0,0 @@ -namespace Intrinio.Realtime.Equities - -open System -open System.Text -open System.Runtime.InteropServices - -type Provider = - | NONE = 0 - | REALTIME = 1 - | MANUAL = 2 - | DELAYED_SIP = 3 - | NASDAQ_BASIC = 4 - -type SubProvider = - | NONE = 0 - | CTA_A = 1 - | CTA_B = 2 - | UTP = 3 - | OTC = 4 - | NASDAQ_BASIC = 5 - | IEX = 6 - -type MessageType = - | Trade = 0 - | Ask = 1 - | Bid = 2 - -type QuoteType = - | Ask = 1 - | Bid = 2 - -type IntervalType = - | OneMinute = 60 - | TwoMinute = 120 - | ThreeMinute = 180 - | FourMinute = 240 - | FiveMinute = 300 - | TenMinute = 600 - | FifteenMinute = 900 - | ThirtyMinute = 1800 - | SixtyMinute = 3600 - -type LogLevel = - | DEBUG = 0 - | INFORMATION = 1 - | WARNING = 2 - | ERROR = 3 - -/// Type: the type of the quote (can be 'ask' or 'bid') -/// Symbol: the 'ticker' symbol -/// Price: the dollar price of the quote -/// Size: the number of shares that were offered as part of the quote -/// Timestamp: the time that the quote was placed (a unix timestamp representing the number of milliseconds (or better) since the unix epoch) -/// SubProvider: the specific provider this trade came from under the parent provider grouping. -type [] Quote = - { - Type : QuoteType - Symbol : string - Price : float - Size : uint32 - Timestamp : DateTime - SubProvider: SubProvider - MarketCenter: char - Condition: string - } - - override this.ToString() : string = - "Quote (" + - "Type: " + MessageType.GetName(this.Type) + - ", Symbol: " + this.Symbol + - ", Price: " + this.Price.ToString("F2") + - ", Size: " + this.Size.ToString() + - ", Timestamp: " + this.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffff") + - ", SubProvider: " + this.SubProvider.ToString() + - ", MarketCenter: " + this.MarketCenter.ToString() + - ", Condition: " + this.Condition + - ")" - -/// Symbol: the 'ticker' symbol -/// Price: the dollar price of the last trade -/// Size: the number of shares that were exchanged in the last trade -/// TotalVolume: the total number of shares that have been traded since market open -/// Timestamp: the time that the trade was executed (a unix timestamp representing the number of milliseconds (or better) since the unix epoch) -/// SubProvider: the specific provider this trade came from under the parent provider grouping. -type [] Trade = - { - Symbol : string - Price : float - Size : uint32 - TotalVolume : uint32 - Timestamp : DateTime - SubProvider: SubProvider - MarketCenter: char - Condition: string - } - - member this.IsDarkpool() : bool = - this.MarketCenter.Equals((char)0) || this.MarketCenter.Equals('D') || this.MarketCenter.Equals('E') || Char.IsWhiteSpace(this.MarketCenter) - - override this.ToString() : string = - "Trade (" + - "Symbol: " + this.Symbol + - ", Price: " + this.Price.ToString("F2") + - ", Size: " + this.Size.ToString() + - ", TotalVolume: " + this.TotalVolume.ToString() + - ", Timestamp: " + this.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffff") + - ", SubProvider: " + this.SubProvider.ToString() + - ", MarketCenter: " + this.MarketCenter.ToString() + - ", Condition: " + this.Condition + - ")" - -type TradeCandleStick = - val Symbol: string - val mutable Volume: uint32 - val mutable High: float - val mutable Low: float - val mutable Close: float - val mutable Open: float - val OpenTimestamp: float - val CloseTimestamp: float - val mutable FirstTimestamp: float - val mutable LastTimestamp: float - val mutable Complete: bool - val mutable Average: float - val mutable Change: float - val Interval: IntervalType - - new(symbol: string, volume: uint32, price: float, openTimestamp: float, closeTimestamp : float, interval : IntervalType, tradeTime : float) = - { - Symbol = symbol - Volume = volume - High = price - Low = price - Close = price - Open = price - OpenTimestamp = openTimestamp - CloseTimestamp = closeTimestamp - FirstTimestamp = tradeTime - LastTimestamp = tradeTime - Complete = false - Average = price - Change = 0.0 - Interval = interval - } - - new(symbol: string, volume: uint32, high: float, low: float, closePrice: float, openPrice: float, openTimestamp: float, closeTimestamp : float, firstTimestamp: float, lastTimestamp: float, complete: bool, average: float, change: float, interval : IntervalType) = - { - Symbol = symbol - Volume = volume - High = high - Low = low - Close = closePrice - Open = openPrice - OpenTimestamp = openTimestamp - CloseTimestamp = closeTimestamp - FirstTimestamp = firstTimestamp - LastTimestamp = lastTimestamp - Complete = complete - Average = average - Change = change - Interval = interval - } - - override this.Equals(other: Object) : bool = - ((not (Object.ReferenceEquals(other, null))) && Object.ReferenceEquals(this, other)) - || ( - (not (Object.ReferenceEquals(other, null))) - && (not (Object.ReferenceEquals(this, other))) - && (other :? TradeCandleStick) - && (this.Symbol.Equals((other :?> TradeCandleStick).Symbol)) - && (this.Interval.Equals((other :?> TradeCandleStick).Interval)) - && (this.OpenTimestamp.Equals((other :?> TradeCandleStick).OpenTimestamp)) - ) - - override this.GetHashCode() : int = - this.Symbol.GetHashCode() ^^^ this.Interval.GetHashCode() ^^^ this.OpenTimestamp.GetHashCode() - - interface IEquatable with - member this.Equals(other: TradeCandleStick) : bool = - ((not (Object.ReferenceEquals(other, null))) && Object.ReferenceEquals(this, other)) - || ( - (not (Object.ReferenceEquals(other, null))) - && (not (Object.ReferenceEquals(this, other))) - && (this.Symbol.Equals(other.Symbol)) - && (this.Interval.Equals(other.Interval)) - && (this.OpenTimestamp.Equals(other.OpenTimestamp)) - ) - - interface IComparable with - member this.CompareTo(other: Object) : int = - match this.Equals(other) with - | true -> 0 - | false -> - match Object.ReferenceEquals(other, null) with - | true -> 1 - | false -> - match (other :? TradeCandleStick) with - | false -> -1 - | true -> - match this.Symbol.CompareTo((other :?> TradeCandleStick).Symbol) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> - match this.Interval.CompareTo((other :?> TradeCandleStick).Interval) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> this.OpenTimestamp.CompareTo((other :?> TradeCandleStick).OpenTimestamp) - - interface IComparable with - member this.CompareTo(other: TradeCandleStick) : int = - match this.Equals(other) with - | true -> 0 - | false -> - match Object.ReferenceEquals(other, null) with - | true -> 1 - | false -> - match this.Symbol.CompareTo(other.Symbol) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> - match this.Interval.CompareTo(other.Interval) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> this.OpenTimestamp.CompareTo(other.OpenTimestamp) - - override this.ToString() : string = - sprintf "TradeCandleStick (Symbol: %s, Volume: %s, High: %s, Low: %s, Close: %s, Open: %s, OpenTimestamp: %s, CloseTimestamp: %s, AveragePrice: %s, Change: %s, Complete: %s)" - this.Symbol - (this.Volume.ToString()) - (this.High.ToString("f3")) - (this.Low.ToString("f3")) - (this.Close.ToString("f3")) - (this.Open.ToString("f3")) - (this.OpenTimestamp.ToString("f6")) - (this.CloseTimestamp.ToString("f6")) - (this.Average.ToString("f3")) - (this.Change.ToString("f6")) - (this.Complete.ToString()) - - member this.Merge(candle: TradeCandleStick) : unit = - this.Average <- ((System.Convert.ToDouble(this.Volume) * this.Average) + (System.Convert.ToDouble(candle.Volume) * candle.Average)) / (System.Convert.ToDouble(this.Volume + candle.Volume)) - this.Volume <- this.Volume + candle.Volume - this.High <- if this.High > candle.High then this.High else candle.High - this.Low <- if this.Low < candle.Low then this.Low else candle.Low - this.Close <- if this.LastTimestamp > candle.LastTimestamp then this.Close else candle.Close - this.Open <- if this.FirstTimestamp < candle.FirstTimestamp then this.Open else candle.Open - this.FirstTimestamp <- if candle.FirstTimestamp < this.FirstTimestamp then candle.FirstTimestamp else this.FirstTimestamp - this.LastTimestamp <- if candle.LastTimestamp > this.LastTimestamp then candle.LastTimestamp else this.LastTimestamp - this.Change <- (this.Close - this.Open) / this.Open - - member internal this.Update(volume: uint32, price: float, time: float) : unit = - this.Average <- ((System.Convert.ToDouble(this.Volume) * this.Average) + (System.Convert.ToDouble(volume) * price)) / (System.Convert.ToDouble(this.Volume + volume)) - this.Volume <- this.Volume + volume - this.High <- if price > this.High then price else this.High - this.Low <- if price < this.Low then price else this.Low - this.Close <- if time > this.LastTimestamp then price else this.Close - this.Open <- if time < this.FirstTimestamp then price else this.Open - this.FirstTimestamp <- if time < this.FirstTimestamp then time else this.FirstTimestamp - this.LastTimestamp <- if time > this.LastTimestamp then time else this.LastTimestamp - this.Change <- (this.Close - this.Open) / this.Open - - member internal this.MarkComplete() : unit = - this.Complete <- true - - member internal this.MarkIncomplete() : unit = - this.Complete <- false - -type QuoteCandleStick = - val Symbol: string - val mutable High: float - val mutable Low: float - val mutable Close: float - val mutable Open: float - val QuoteType: QuoteType - val OpenTimestamp: float - val CloseTimestamp: float - val mutable FirstTimestamp: float - val mutable LastTimestamp: float - val mutable Complete: bool - val mutable Change: float - val Interval: IntervalType - - new(symbol: string, - price: float, - quoteType: QuoteType, - openTimestamp: float, - closeTimestamp: float, - interval: IntervalType, - tradeTime: float) = - { - Symbol = symbol - High = price - Low = price - Close = price - Open = price - QuoteType = quoteType - OpenTimestamp = openTimestamp - CloseTimestamp = closeTimestamp - FirstTimestamp = tradeTime - LastTimestamp = tradeTime - Complete = false - Change = 0.0 - Interval = interval - } - - new(symbol: string, - high: float, - low: float, - closePrice: float, - openPrice: float, - quoteType: QuoteType, - openTimestamp: float, - closeTimestamp: float, - firstTimestamp: float, - lastTimestamp: float, - complete: bool, - change: float, - interval: IntervalType) = - { - Symbol = symbol - High = high - Low = low - Close = closePrice - Open = openPrice - QuoteType = quoteType - OpenTimestamp = openTimestamp - CloseTimestamp = closeTimestamp - FirstTimestamp = firstTimestamp - LastTimestamp = lastTimestamp - Complete = complete - Change = change - Interval = interval - } - - override this.Equals(other: Object) : bool = - ((not (Object.ReferenceEquals(other, null))) && Object.ReferenceEquals(this, other)) - || ( - (not (Object.ReferenceEquals(other, null))) - && (not (Object.ReferenceEquals(this, other))) - && (other :? QuoteCandleStick) - && (this.Symbol.Equals((other :?> QuoteCandleStick).Symbol)) - && (this.Interval.Equals((other :?> QuoteCandleStick).Interval)) - && (this.QuoteType.Equals((other :?> QuoteCandleStick).QuoteType)) - && (this.OpenTimestamp.Equals((other :?> QuoteCandleStick).OpenTimestamp)) - ) - - override this.GetHashCode() : int = - this.Symbol.GetHashCode() ^^^ this.Interval.GetHashCode() ^^^ this.OpenTimestamp.GetHashCode() ^^^ this.QuoteType.GetHashCode() - - interface IEquatable with - member this.Equals(other: QuoteCandleStick) : bool = - ((not (Object.ReferenceEquals(other, null))) && Object.ReferenceEquals(this, other)) - || ( - (not (Object.ReferenceEquals(other, null))) - && (not (Object.ReferenceEquals(this, other))) - && (this.Symbol.Equals(other.Symbol)) - && (this.Interval.Equals(other.Interval)) - && (this.QuoteType.Equals(other.QuoteType)) - && (this.OpenTimestamp.Equals(other.OpenTimestamp)) - ) - - interface IComparable with - member this.CompareTo(other: Object) : int = - match this.Equals(other) with - | true -> 0 - | false -> - match Object.ReferenceEquals(other, null) with - | true -> 1 - | false -> - match (other :? QuoteCandleStick) with - | false -> -1 - | true -> - match this.Symbol.CompareTo((other :?> QuoteCandleStick).Symbol) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> - match this.Interval.CompareTo((other :?> QuoteCandleStick).Interval) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> - match this.QuoteType.CompareTo((other :?> QuoteCandleStick).QuoteType) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> this.OpenTimestamp.CompareTo((other :?> QuoteCandleStick).OpenTimestamp) - - interface IComparable with - member this.CompareTo(other: QuoteCandleStick) : int = - match this.Equals(other) with - | true -> 0 - | false -> - match Object.ReferenceEquals(other, null) with - | true -> 1 - | false -> - match this.Symbol.CompareTo(other.Symbol) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> - match this.Interval.CompareTo(other.Interval) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> - match this.QuoteType.CompareTo(other.QuoteType) with - | x when x < 0 -> -1 - | x when x > 0 -> 1 - | 0 -> this.OpenTimestamp.CompareTo(other.OpenTimestamp) - - override this.ToString() : string = - sprintf "QuoteCandleStick (Symbol: %s, QuoteType: %s, High: %s, Low: %s, Close: %s, Open: %s, OpenTimestamp: %s, CloseTimestamp: %s, Change: %s, Complete: %s)" - this.Symbol - (this.QuoteType.ToString()) - (this.High.ToString("f3")) - (this.Low.ToString("f3")) - (this.Close.ToString("f3")) - (this.Open.ToString("f3")) - (this.OpenTimestamp.ToString("f6")) - (this.CloseTimestamp.ToString("f6")) - (this.Change.ToString("f6")) - (this.Complete.ToString()) - - member this.Merge(candle: QuoteCandleStick) : unit = - this.High <- if this.High > candle.High then this.High else candle.High - this.Low <- if this.Low < candle.Low then this.Low else candle.Low - this.Close <- if this.LastTimestamp > candle.LastTimestamp then this.Close else candle.Close - this.Open <- if this.FirstTimestamp < candle.FirstTimestamp then this.Open else candle.Open - this.FirstTimestamp <- if candle.FirstTimestamp < this.FirstTimestamp then candle.FirstTimestamp else this.FirstTimestamp - this.LastTimestamp <- if candle.LastTimestamp > this.LastTimestamp then candle.LastTimestamp else this.LastTimestamp - this.Change <- (this.Close - this.Open) / this.Open - - member this.Update(price: float, time: float) : unit = - this.High <- if price > this.High then price else this.High - this.Low <- if price < this.Low then price else this.Low - this.Close <- if time > this.LastTimestamp then price else this.Close - this.Open <- if time < this.FirstTimestamp then price else this.Open - this.FirstTimestamp <- if time < this.FirstTimestamp then time else this.FirstTimestamp - this.LastTimestamp <- if time > this.LastTimestamp then time else this.LastTimestamp - this.Change <- (this.Close - this.Open) / this.Open - - member internal this.MarkComplete() : unit = - this.Complete <- true - - member internal this.MarkIncomplete() : unit = - this.Complete <- false - -type internal Tick( - timeReceived : DateTime, - trade: Option, - quote : Option) = - - let getTradeBytes(trade : Trade) : byte[] = - let symbolBytes : byte[] = Encoding.ASCII.GetBytes(trade.Symbol) - let symbolLength : byte = System.Convert.ToByte(symbolBytes.Length) - let symbolLengthInt32 : int = System.Convert.ToInt32 symbolLength - let marketCenterBytes : byte[] = BitConverter.GetBytes(trade.MarketCenter) - let tradePrice : byte[] = BitConverter.GetBytes(System.Convert.ToSingle(trade.Price)) - let tradeSize : byte[] = BitConverter.GetBytes(trade.Size) - let timeStamp : byte[] = BitConverter.GetBytes(System.Convert.ToUInt64((trade.Timestamp - DateTime.UnixEpoch).Ticks) * 100UL) - let tradeTotalVolume : byte[] = BitConverter.GetBytes(trade.TotalVolume) - let condition : byte[] = Encoding.ASCII.GetBytes(trade.Condition) - let conditionLength : byte = System.Convert.ToByte(condition.Length) - let messageLength : byte = 27uy + symbolLength + conditionLength - - let bytes : byte[] = Array.zeroCreate (System.Convert.ToInt32(messageLength)) - bytes[0] <- System.Convert.ToByte((int)(MessageType.Trade)); - bytes[1] <- messageLength; - bytes[2] <- symbolLength; - Array.Copy(symbolBytes, 0, bytes, 3, symbolLengthInt32); - bytes[3 + symbolLengthInt32] <- System.Convert.ToByte((int)(trade.SubProvider)); - Array.Copy(marketCenterBytes, 0, bytes, 4 + symbolLengthInt32, marketCenterBytes.Length); - Array.Copy(tradePrice, 0, bytes, 6 + symbolLengthInt32, tradePrice.Length); - Array.Copy(tradeSize, 0, bytes, 10 + symbolLengthInt32, tradeSize.Length); - Array.Copy(timeStamp, 0, bytes, 14 + symbolLengthInt32, timeStamp.Length); - Array.Copy(tradeTotalVolume, 0, bytes, 22 + symbolLengthInt32, tradeTotalVolume.Length); - bytes[26 + symbolLengthInt32] <- conditionLength; - Array.Copy(condition, 0, bytes, 27 + symbolLengthInt32, System.Convert.ToInt32(conditionLength)); - - // byte 0: message type (hasn't changed) - // byte 1: message length (in bytes, including bytes 0 and 1) - // byte 2: symbol length (in bytes) - // bytes[3...]: symbol string (ascii) - // next byte: source - // next 2 bytes: market center (as 1 char) - // next 4 bytes: trade price (float) - // next 4 bytes: trade size (uint) - // next 8 bytes: timestamp (uint64) - // next 4 bytes: trade total volume ((uint) - // next byte: condition len - // next bytes: condition string (ascii) - - bytes; - - let getQuoteBytes(quote : Quote) : byte[] = - let symbolBytes : byte[] = Encoding.ASCII.GetBytes(quote.Symbol) - let symbolLength : byte = System.Convert.ToByte(symbolBytes.Length) - let symbolLengthInt32 : int = System.Convert.ToInt32 symbolLength - let marketCenterBytes : byte[] = BitConverter.GetBytes(quote.MarketCenter) - let tradePrice : byte[] = BitConverter.GetBytes(System.Convert.ToSingle(quote.Price)) - let tradeSize : byte[] = BitConverter.GetBytes(quote.Size) - let timeStamp : byte[] = BitConverter.GetBytes(System.Convert.ToUInt64((quote.Timestamp - DateTime.UnixEpoch).Ticks) * 100UL) - let condition : byte[] = Encoding.ASCII.GetBytes(quote.Condition) - let conditionLength : byte = System.Convert.ToByte(condition.Length) - let messageLength : byte = 23uy + symbolLength + conditionLength - - let bytes : byte[] = Array.zeroCreate (System.Convert.ToInt32(messageLength)) - bytes[0] <- System.Convert.ToByte((int)(if quote.Type = QuoteType.Ask then MessageType.Ask else MessageType.Bid)); - bytes[1] <- messageLength; - bytes[2] <- symbolLength; - Array.Copy(symbolBytes, 0, bytes, 3, symbolLengthInt32); - bytes[3 + symbolLengthInt32] <- System.Convert.ToByte((int)(quote.SubProvider)); - Array.Copy(marketCenterBytes, 0, bytes, 4 + symbolLengthInt32, marketCenterBytes.Length); - Array.Copy(tradePrice, 0, bytes, 6 + symbolLengthInt32, tradePrice.Length); - Array.Copy(tradeSize, 0, bytes, 10 + symbolLengthInt32, tradeSize.Length); - Array.Copy(timeStamp, 0, bytes, 14 + symbolLengthInt32, timeStamp.Length); - bytes[22 + symbolLengthInt32] <- conditionLength; - Array.Copy(condition, 0, bytes, 23 + symbolLengthInt32, System.Convert.ToInt32(conditionLength)); - - // byte 0: message type (hasn't changed) - // byte 1: message length (in bytes, including bytes 0 and 1) - // byte 2: symbol length (in bytes) - // bytes[3...]: symbol string (ascii) - // next byte: source - // next 2 bytes: market center (as 1 char) - // next 4 bytes: ask/bid price (float) - // next 4 bytes: ask/bid size (uint) - // next 8 bytes: timestamp (uint64) - // next byte: condition len - // next bytes: condition string (ascii) - - bytes - - member _.TimeReceived() : DateTime = timeReceived - - member _.IsTrade() : bool = - trade.IsSome - - member _.Trade() : Trade = - trade.Value - - member _.Quote() : Quote = - quote.Value - - member _.GetTimeReceivedBytes() : byte[] = - BitConverter.GetBytes(System.Convert.ToUInt64((timeReceived - DateTime.UnixEpoch).Ticks) * 100UL) - - member _.GetEventBytes() : byte[] = - match trade with - | Some t -> getTradeBytes t - | None -> - match quote with - | Some q -> getQuoteBytes q - | None -> Array.Empty() - -type ClientStats ( - socketDataMessages : uint64, - socketTextMessages : uint64, - queueDepth : int, - eventCount : uint64, - tradeCount : uint64, - quoteCount : uint64) = - - member _.SocketDataMessages() : uint64 = - socketDataMessages - - member _.SocketTextMessages() : uint64 = - socketTextMessages - - member _.QueueDepth() : int = - queueDepth - - member _.EventCount() : uint64 = - eventCount - - member _.TradeCount() : uint64 = - tradeCount - - member _.QuoteCount() : uint64 = - quoteCount - -type public IEquitiesWebSocketClient = - abstract member Join : unit -> unit - abstract member Join : string * bool option -> unit - abstract member Join : string[] * bool option -> unit - abstract member Leave : unit -> unit - abstract member Leave : string -> unit - abstract member Leave : string[] -> unit - abstract member Stop : unit -> unit - abstract member GetStats : unit -> ClientStats - abstract member Log : string * [] propertyValues:obj[] -> unit \ No newline at end of file diff --git a/PUBLISHING.md b/PUBLISHING.md index e62ac63..ecc992b 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -5,11 +5,11 @@ Open a powershell window. Navigate to the ...\intrinio-realtime-csharp-sdk\IntrinioRealTimeSDK folder. Run: ``` -dotnet pack IntrinioRealTimeSDK.csproj -p:NuspecFile=IntrinioRealTimeClient.nuspec +dotnet pack SampleApp.csproj -p:NuspecFile=IntrinioRealTimeClient.nuspec ``` This will create a `IntrinioRealTimeClient.{version}.nupkg` file. The path to this file will be output by the 'pack' command but is likely in: -'...\intrinio-realtime-csharp-sdk\IntrinioRealTimeSDK\bin\Debug\' +'...\intrinio-realtime-csharp-sdk\SampleApp\bin\Release\net8.0\' To publish the file to NuGet, run: # Publishing diff --git a/README.md b/README.md index 18a4747..71277dc 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,14 @@ # intrinio-realtime-dotnet-sdk -SDK for working with Intrinio's realtime Multi-Exchange, delayed SIP, or NASDAQ Basic prices feeds. Get a comprehensive view with increased market volume and enjoy minimized exchange and per user fees. - -[Intrinio](https://intrinio.com/) provides real-time stock prices via a two-way WebSocket connection. To get started, [subscribe to a real-time data feed](https://intrinio.com/real-time-multi-exchange) and follow the instructions below. - -[Documentation for our legacy realtime client](https://github.com/intrinio/intrinio-realtime-csharp-sdk/tree/v2.2.0) +SDK for working with Intrinio's realtime OPRA, IEX, delayed SIP, or NASDAQ Basic prices feeds. Get a comprehensive view with increased market volume and enjoy minimized exchange and per user fees. +[Intrinio](https://intrinio.com/) provides real-time stock and option prices via a two-way WebSocket connection. To get started, [subscribe to a real-time equity feed](https://intrinio.com/real-time-multi-exchange), or [subscribe to a real-time options feed](https://intrinio.com/financial-market-data/options-data) and follow the instructions below. ## Requirements - .NET 8+ ## Docker -Add your API key to the config.json file in IntrinioRealTimeSDK, then +Add your API key to the config.json file in SampleApp as well as select the appropriate provider, comment the appropriate line in Program.cs, then ``` docker compose build docker compose run example @@ -28,30 +25,83 @@ Be sure to update [config.json](https://github.com/intrinio/intrinio-realtime-cs ## Features -* Receive streaming, real-time price quotes (last trade, bid, ask) -* Subscribe to updates from individual securities -* Subscribe to updates for all securities +### Equities + +* Receive streaming, real-time pricing (trades, NBBO bid, ask) +* Subscribe to updates from individual securities, individual contracts, or +* Subscribe to updates for all securities (Lobby/Firehose mode) + +### Options + +* Receive streaming, real-time option price updates: + * every trade + * conflated bid and ask + * open interest, open, close, high, low + * unusual activity(block trades, sweeps, whale trades, unusual sweeps) +* Subscribe to updates from individual options contracts (or option chains) +* Subscribe to updates for the entire universe of option contracts (~1.5M option contracts) ## Example Usage ```csharp -static void Main(string[] _) +static async Task Main(string[] _) { Client.Log("Starting sample app"); - client = new Client(OnTrade, OnQuote); + var client = new EquitiesWebSocketClient(OnTrade, OnQuote); + //var client = new OptionsWebSocketClient(OnTrade, OnQuote, OnRefresh, OnUnusualActivity); + await client.Start(); timer = new Timer(TimerCallback, client, 10000, 10000); - client.Join(); //Load symbols from config.json + await client.Join(); //Load symbols from config.json //client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime + //client.JoinLobby(true); //Join the lobby instead (don't subscribe to anything else) to get everything. Console.CancelKeyPress += new ConsoleCancelEventHandler(Cancel); } + +static void Cancel(object sender, ConsoleCancelEventArgs args) +{ + Log("Stopping sample app"); + timer.Dispose(); + client.Stop(); + Environment.Exit(0); +} ``` ## Handling Quotes -There are thousands of securities, each with their own feed of activity. We highly encourage you to make your trade and quote handlers has short as possible and follow a queue pattern so your app can handle the volume of activity. +There are thousands of securities and millions of options contracts, each with their own feed of activity. We highly encourage you to make your trade and quote handlers as short as possible and follow a queue pattern so your app can handle the volume of activity. Please note that quotes comprise 99% of the volume of the feed. + +## Client Statistics and Performance + +The client is able to report back various statistics about its performance, including the quantity of various events and if and how much it is falling behind. The client has two internal buffers - a main buffer and an overflow buffer. The main buffer will spill into the overflow buffer when the main buffer is full, but if the overflow buffer fills, then messages start to drop until it catches up. The statistics allow you to see how full both buffers are, as well as how many have spilled into overflow, and how many were dropped altogether. If your client is dropping messages, you can try to increase buffer size, but the main problem is most likely that you need more processing power (more hardware cores, and more threads configured in your client config). Please see below for recommended hardware requirements. + +### Minimum Hardware Requirements - Trades only +Equities Client: +* Non-lobby mode: 1 hardware core and 1 thread in your configuration for roughly every 100 symbols, up to the lobby mode settings. Absolute minimum 2 cores and threads. +* Lobby mode: 4 hardware cores and 4 threads in your configuration +* 5 Mbps connection +* 0.5 ms latency + +Options Client: +* Non-lobby mode: 1 hardware core and 1 thread in your configuration for roughly every 250 contracts, up to the lobby mode settings. 3 cores and 3 configured threads for each chain, up to the lobby mode settings. Absolute minimum 3 cores and threads. +* Lobby mode: 6 hardware cores and 6 threads in your configuration +* 25 Mbps connection +* 0.5 ms latency + +### Minimum Hardware Requirements - Trades and Quotes +Equities Client: +* Non-lobby mode: 1 hardware core and 1 thread in your configuration for roughly every 25 symbols, up to the lobby mode settings. Absolute minimum 4 cores and threads. +* Lobby mode: 8 hardware cores and 8 threads in your configuration +* 25 Mbps connection +* 0.5 ms latency + +Options Client: +* Non-lobby mode: 1 hardware core and 1 thread in your configuration for roughly every 100 contracts, up to the lobby mode settings. 4 cores and 4 configured threads for each chain, up to the lobby mode settings. Absolute minimum 4 cores and threads. +* Lobby mode: 12 hardware cores and 12 threads in your configuration +* 100 Mbps connection +* 0.5 ms latency ## Data Format -### Trade Message +### Equity Trade Message ```fsharp type [] Trade = @@ -84,7 +134,7 @@ type [] Trade = * **Condition** - Provides the condition -### Quote Message +### Equity Quote Message ```fsharp type [] Quote = @@ -118,6 +168,126 @@ type [] Quote = * **MarketCenter** - Provides the market center * **Condition** - Provides the condition +### Option Trade Message + +```fsharp +type Trade +``` + +* **Contract** - Identifier for the options contract. This includes the ticker symbol, put/call, expiry, and strike price. +* **Exchange** - Enum identifying the specific exchange through which the trade occurred +* **Price** - the price in USD +* **Size** - the size of the last trade in hundreds (each contract is for 100 shares). +* **TotalVolume** - The number of contracts traded so far today. +* **Timestamp** - a Unix timestamp (with microsecond precision) +* **Qualifiers** - a 4-byte tuple: each byte represents one trade qualifier. See list of possible [Trade Qualifiers](#trade-qualifiers), below. +* **AskPriceAtExecution** - the best last ask price in USD +* **BidPriceAtExecution** - the best last bid price in USD +* **UnderlyingPriceAtExecution** - the price of the underlying security in USD + + +### Option Trade Qualifiers + +| Value | Description | +|-------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0 | Transaction is a regular trade | +| 1 | Out-of-sequence cancellation | +| 2 | Transaction is being reported late and is out-of-sequence | +| 3 | In-sequence cancellation | +| 4 | Transaction is being reported late, but is in correct sequence. | +| 5 | Cancel the first trade of the day | +| 6 | Late report of the opening trade and is out -of-sequence. Send an open price. | +| 7 | Transaction was the only one reported this day for the particular option contract and is now to be cancelled. | +| 8 | Late report of an opening trade and is in correct sequence. Process as regular trade. | +| 9 | Transaction was executed electronically. Process as regular trade. | +| 10 | Re-opening of a contract which was halted earlier. Process as regular trade. | +| 11 | Transaction is a contract for which the terms have been adjusted to reflect stock dividend, stock split or similar event. Process as regular trade. | +| 12 | Transaction represents a trade in two options of same class (a buy and a sell in the same class). Process as regular trade. | +| 13 | Transaction represents a trade in two options of same class (a buy and a sell in a put and a call.). Process as regular trade. | +| 14 | Transaction is the execution of a sale at a price agreed upon by the floor personnel involved, where a condition of the trade is that it reported following a non -stopped trade of the same series at the same price. | +| 15 | Cancel stopped transaction. | +| 16 | Transaction represents the option portion of buy/write (buy stock, sell call options). Process as regular trade. | +| 17 | Transaction represents the buying of a call and selling of a put for same underlying stock or index. Process as regular trade. | +| 18 | Transaction was the execution of an order which was “stopped” at a price that did not constitute a Trade-Through on another market at the time of the stop. Process like a normal transaction. | +| 19 | Transaction was the execution of an order identified as an Intermarket Sweep Order. Updates open, high, low, and last. | +| 20 | Transaction reflects the execution of a “benchmark trade”. A “benchmark trade” is a trade resulting from the matching of “benchmark orders”. A “benchmark order” is an order for which the price is not based, directly or indirectly, on the quoted price of th e option at the time of the order’s execution and for which the material terms were not reasonably determinable at the time a commitment to trade the order was made. Updates open, high, and low, but not last unless the trade is the first of the day. | +| 24 | Transaction is trade through exempt, treat like a regular trade. | +| 27 | “a” (Single leg auction non ISO) | +| 28 | “b” (Single leg auction ISO) | +| 29 | “c” (Single leg cross Non ISO) | +| 30 | “d” (Single leg cross ISO) | +| 31 | “e” (Single leg floor trade) | +| 32 | “f” (Multi leg auto electronic trade) | +| 33 | “g” (Multi leg auction trade) | +| 34 | “h” (Multi leg Cross trade) | +| 35 | “i” (Multi leg floor trade) | +| 36 | “j” (Multi leg auto electronic trade against single leg) | +| 37 | “k” (Stock options Auction) | +| 38 | “l” (Multi leg auction trade against single leg) | +| 39 | “m” (Multi leg floor trade against single leg) | +| 40 | “n” (Stock options auto electronic trade) | +| 41 | “o” (Stock options cross trade) | +| 42 | “p” (Stock options floor trade) | +| 43 | “q” (Stock options auto electronic trade against single leg) | +| 44 | “r” (Stock options auction against single leg) | +| 45 | “s” (Stock options floor trade against single leg) | +| 46 | “t” (Multi leg floor trade of proprietary products) | +| 47 | “u” (Multilateral Compression Trade of Proprietary Data Products)Transaction represents an execution in a proprietary product done as part of a multilateral compression. Trades are executed outside of regular trading hours at prices derived from end of day markets. Trades do not update Open, High, Low, and Closing Prices, but will update total volume. | +| 48 | “v” (Extended Hours Trade )Transaction represents a trade that was executed outside of regular market hours. Trades do not update Open, High, Low, and Closing Prices but will update total volume. | + + + +### Option Quote Message + +```fsharp +type Quote +``` + +* **Contract** - Identifier for the options contract. This includes the ticker symbol, put/call, expiry, and strike price. +* **AskPrice** - the last best ask price in USD +* **AskSize** - the last best ask size in hundreds (each contract is for 100 shares). +* **BidPrice** - the last best bid price in USD +* **BidSize** - the last best bid size in hundreds (each contract is for 100 shares). +* **Timestamp** - a Unix timestamp (with microsecond precision) + + +### Option Refresh Message + +```fsharp +type Refresh +``` + +* **Contract** - Identifier for the options contract. This includes the ticker symbol, put/call, expiry, and strike price. +* **OpenInterest** - the total quantity of opened contracts as reported at the start of the trading day +* **OpenPrice** - the open price price in USD +* **ClosePrice** - the close price in USD +* **HighPrice** - the current high price in USD +* **LowPrice** - the current low price in USD + +### Option Unusual Activity Message + +```fsharp +type UnusualActivity +``` + +* **Contract** - Identifier for the options contract. This includes the ticker symbol, put/call, expiry, and strike price. +* **Type** - The type of unusual activity that was detected + * **`Block`** - represents an 'block' trade + * **`Sweep`** - represents an intermarket sweep + * **`Large`** - represents a trade of at least $100,000 + * **`Unusual Sweep`** - represents an unusually large sweep near market open. +* **Sentiment** - The sentiment of the unusual activity event + * **`Neutral`** - + * **`Bullish`** - + * **`Bearish`** - +* **TotalValue** - The total value of the trade in USD. 'Sweeps' and 'blocks' can be comprised of multiple trades. This is the value of the entire event. +* **TotalSize** - The total size of the trade in number of contracts. 'Sweeps' and 'blocks' can be comprised of multiple trades. This is the total number of contracts exchanged during the event. +* **AveragePrice** - The average price at which the trade was executed. 'Sweeps' and 'blocks' can be comprised of multiple trades. This is the average trade price for the entire event. +* **AskPriceAtExecution** - The 'ask' price of the underlying at execution of the trade event. +* **BidPriceAtExecution** - The 'bid' price of the underlying at execution of the trade event. +* **UnderlyingPriceAtExecution** - The last trade price of the underlying at execution of the trade event. +* **Timestamp** - a Unix timestamp (with microsecond precision). + ## API Keys You will receive your Intrinio API Key after [creating an account](https://intrinio.com/signup). You will need a subscription to a [realtime data feed](https://intrinio.com/real-time-multi-exchange) as well. @@ -128,13 +298,15 @@ You will receive your Intrinio API Key after [creating an account](https://intri * **Parameter** `onTrade`: The Action accepting trades. This function will be invoked when a 'trade' has been received. The trade will be passed as an argument to the callback. * **Parameter** `onQuote`: Optional. The Action accepting quotes. This function will be invoked when a 'quote' has been received. The quote will be passed as an argument to the callback. If 'onQuote' is not provided, the client will NOT request to receive quote updates from the server. --------- +`client.Start();` - Initializes and connects the client. +--------- `client.Join(symbols, tradesOnly);` - Joins the given channels. This can be called at any time. The client will automatically register joined channels and establish the proper subscriptions with the WebSocket connection. If no arguments are provided, this function joins channel(s) configured in config.json. * **Parameter** `symbols` - Optional. A string representing a single ticker symbol (e.g. "AAPL") or an array of ticker symbols (e.g. ["AAPL", "MSFT", "GOOG"]) to join. You can also use the special symbol, "lobby" to join the firehose channel and recieved updates for all ticker symbols. You must have a valid "firehose" subscription. * **Parameter** `tradesOnly` - Optional (default: false). A boolean value indicating whether the server should return trade data only (as opposed to trade and quote data). ```csharp client.Join(["AAPL", "MSFT", "GOOG"]) client.Join("GE", true) -client.Join("lobby") //must have a valid 'firehose' subscription +client.JoinLobby() //must have a valid 'firehose' subscription ``` --------- `client.Leave(symbols)` - Leaves the given channels. @@ -142,11 +314,11 @@ client.Join("lobby") //must have a valid 'firehose' subscription ```csharp client.Leave(["AAPL", "MSFT", "GOOG"]) client.Leave("GE") -client.Leave("lobby") +client.LeaveLobby() client.Leave() ``` --------- -`client.Stop()` - Closes the WebSocket, stops the self-healing and heartbeat intervals. Call this to properly dispose of the client. +`client.Stop()` - Closes the WebSocket, stops the self-healing. Call this to properly dispose of the client. ## Configuration @@ -155,26 +327,33 @@ client.Leave() The application will look for the config file if you don't pass in a config object. ```json { - "Config": { - "ApiKey": "", //Your Intrinio API key. - "NumThreads": 2, //The number of threads to use for processing events. - "Provider": "REALTIME", //or DELAYED_SIP or NASDAQ_BASIC or MANUAL - "Symbols": [ "AAPL", "MSFT", "GOOG" ], //This is a list of individual tickers to subscribe to, or "lobby" to subscribe to all at once (firehose). - "TradesOnly": true //This indicates whether you only want trade events (true) or you want trade, ask, and bid events (false). - }, - "Serilog": { - "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Warning", - "System": "Warning" - } - }, - "WriteTo": [ - { "Name": "Console" } - ] - } + "Config": { + "ApiKey": "API_KEY_HERE", + "NumThreads": 8, + //"Provider": "OPRA", + "Provider": "REALTIME", + //"Provider": "DELAYED_SIP", + //"Provider": "NASDAQ_BASIC", + //"Provider": "MANUAL", + //"IPAddress": "1.2.3.4", + "BufferSize": 4096, + "OverflowBufferSize": 8192, + "Symbols": [ "AAPL", "MSFT", "TSLA" ] + //"Symbols": [ "lobby" ] + }, + "Serilog": { + "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" } + ] + } } ``` To create a config object to pass in instead of the file, do the following. Don't forget to also set up sirilog configuration as well: @@ -185,7 +364,10 @@ config.Provider = Provider.REALTIME; config.ApiKey = ""; config.Symbols = new[] { "AAPL", "MSFT" }; config.NumThreads = 2; +config.BufferSize = 2048; +config.OverflowBufferSize = 4096; client = new Client(onTrade, onQuote, config); +client.Start(); ``` ## Example Replay Client Usage @@ -195,7 +377,7 @@ static void Main(string[] _) Client.Log("Starting sample app"); //You can also simulate a trading day by replaying a particular day's data. You can do this with the actual time between events, or without. DateTime yesterday = DateTime.Today - TimeSpan.FromDays(1); - replayClient = new ReplayClient(onTrade, onQuote, yesterday, true, true); //A client to replay a previous day's data + replayClient = new ReplayClient(onTrade, onQuote, yesterday, true, true, false, String.Empty); //A client to replay a previous day's data timer = new Timer(ReplayTimerCallback, replayClient, 10000, 10000); replayClient.Join(); //Load symbols from your config or config.json //client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime @@ -214,7 +396,7 @@ static void Main(string[] _) //Subscribe the candlestick client to trade and/or quote events as well. It's important any method subscribed this way handles exceptions so as to not cause issues for other subscribers! _useTradeCandleSticks = true; _useQuoteCandleSticks = true; - _candleStickClient = new CandleStickClient(OnTradeCandleStick, OnQuoteCandleStick, IntervalType.OneMinute, true, null, null, 0, false); + _candleStickClient = new CandleStickClient(OnTradeCandleStick, OnQuoteCandleStick, IntervalType.OneMinute, false, null, null, 0, false); onTrade += _candleStickClient.OnTrade; onQuote += _candleStickClient.OnQuote; _candleStickClient.Start(); diff --git a/SampleApp/.DS_Store b/SampleApp/.DS_Store new file mode 100644 index 0000000..5bf14a3 Binary files /dev/null and b/SampleApp/.DS_Store differ diff --git a/SampleApp/EquitiesSampleApp.cs b/SampleApp/EquitiesSampleApp.cs new file mode 100644 index 0000000..19f47df --- /dev/null +++ b/SampleApp/EquitiesSampleApp.cs @@ -0,0 +1,176 @@ +using System; +using System.Threading; +using System.Collections.Concurrent; +using Intrinio.Realtime; +using Intrinio.Realtime.Equities; +using Serilog; +using Serilog.Core; + +namespace SampleApp; + +public class EquitiesSampleApp +{ + private static IEquitiesWebSocketClient client = null; + private static CandleStickClient _candleStickClient = null; + private static Timer timer = null; + private static readonly ConcurrentDictionary trades = new(5, 15_000); + private static readonly ConcurrentDictionary quotes = new(5, 15_000); + private static int maxTradeCount = 0; + private static int maxQuoteCount = 0; + private static Trade maxCountTrade; + private static Quote maxCountQuote; + private static UInt64 _tradeCandleStickCount = 0UL; + private static UInt64 _tradeCandleStickCountIncomplete = 0UL; + private static UInt64 _AskCandleStickCount = 0UL; + private static UInt64 _AskCandleStickCountIncomplete = 0UL; + private static UInt64 _BidCandleStickCount = 0UL; + private static UInt64 _BidCandleStickCountIncomplete = 0UL; + private static bool _useTradeCandleSticks = false; + private static bool _useQuoteCandleSticks = false; + + static void OnQuote(Quote quote) + { + string key = quote.Symbol + ":" + quote.Type; + int updateFunc(string _, int prevValue) + { + if (prevValue + 1 > maxQuoteCount) + { + maxQuoteCount = prevValue + 1; + maxCountQuote = quote; + } + return (prevValue + 1); + } + quotes.AddOrUpdate(key, 1, updateFunc); + } + + static void OnTrade(Trade trade) + { + string key = trade.Symbol; + int updateFunc(string _, int prevValue) + { + if (prevValue + 1 > maxTradeCount) + { + maxTradeCount = prevValue + 1; + maxCountTrade = trade; + } + return (prevValue + 1); + } + trades.AddOrUpdate(key, 1, updateFunc); + } + + static void OnTradeCandleStick(TradeCandleStick tradeCandleStick) + { + if (tradeCandleStick.Complete) + { + Interlocked.Increment(ref _tradeCandleStickCount); + } + else + { + Interlocked.Increment(ref _tradeCandleStickCountIncomplete); + } + } + + static void OnQuoteCandleStick(QuoteCandleStick quoteCandleStick) + { + if (quoteCandleStick.QuoteType == QuoteType.Ask) + if (quoteCandleStick.Complete) + Interlocked.Increment(ref _AskCandleStickCount); + else + Interlocked.Increment(ref _AskCandleStickCountIncomplete); + else + if (quoteCandleStick.Complete) + Interlocked.Increment(ref _BidCandleStickCount); + else + Interlocked.Increment(ref _BidCandleStickCountIncomplete); + } + + static void TimerCallback(object obj) + { + IEquitiesWebSocketClient client = (IEquitiesWebSocketClient) obj; + ClientStats stats = client.GetStats(); + Log("Socket Stats - Grouped Messages: {0}, Text Messages: {1}, Queue Depth: {2}%, Overflow Queue Depth: {3}%, Drops: {4}, Overflow Count: {5}, Individual Events: {6}, Trades: {7}, Quotes: {8}", + stats.SocketDataMessages, + stats.SocketTextMessages, + (stats.QueueDepth * 100) / stats.QueueCapacity, + (stats.OverflowQueueDepth * 100) / stats.OverflowQueueCapacity, + stats.DroppedCount, + stats.OverflowCount, + stats.EventCount, + client.TradeCount, + client.QuoteCount); + if (maxTradeCount > 0) + { + Log("Most active trade: {0} ({1} updates)", maxCountTrade, maxTradeCount); + } + if (maxQuoteCount > 0) + { + Log("Most active quote: {0} ({1} updates)", maxCountQuote, maxQuoteCount); + } + if (_useTradeCandleSticks) + Log("TRADE CANDLESTICK STATS - TradeCandleSticks = {0}, TradeCandleSticksIncomplete = {1}", _tradeCandleStickCount, _tradeCandleStickCountIncomplete); + if (_useQuoteCandleSticks) + Log("QUOTE CANDLESTICK STATS - Asks = {0}, Bids = {1}, AsksIncomplete = {2}, BidsIncomplete = {3}", _AskCandleStickCount, _BidCandleStickCount, _AskCandleStickCountIncomplete, _BidCandleStickCountIncomplete); + } + + static void Cancel(object sender, ConsoleCancelEventArgs args) + { + Log("Stopping sample app"); + timer.Dispose(); + client.Stop(); + if (_useTradeCandleSticks || _useQuoteCandleSticks) + { + _candleStickClient.Stop(); + } + Environment.Exit(0); + } + + [MessageTemplateFormatMethod("messageTemplate")] + static void Log(string messageTemplate, params object[] propertyValues) + { + Serilog.Log.Information(messageTemplate, propertyValues); + } + + public static async Task Run(string[] _) + { + Log("Starting sample app"); + Action onTrade = OnTrade; + Action onQuote = OnQuote; + + // //Subscribe the candlestick client to trade and/or quote events as well. It's important any method subscribed this way handles exceptions so as to not cause issues for other subscribers! + // _useTradeCandleSticks = true; + // _useQuoteCandleSticks = true; + // _candleStickClient = new CandleStickClient(OnTradeCandleStick, OnQuoteCandleStick, IntervalType.OneMinute, true, null, null, 0, false); + // onTrade += _candleStickClient.OnTrade; + // onQuote += _candleStickClient.OnQuote; + // _candleStickClient.Start(); + + // //You can either automatically load the config.json by doing nothing, or you can specify your own config and pass it in. + // //If you don't have a config.json, don't forget to also give Serilog a config so it can write to console + // Log.Logger = new LoggerConfiguration().WriteTo.Console(restrictedToMinimumLevel: Serilog.Events.LogEventLevel.Information).CreateLogger(); + // Config config = new Config(); + // config.Provider = Provider.REALTIME; + // config.ApiKey = "API_KEY_HERE"; + // config.Symbols = new[] { "AAPL", "MSFT" }; + // config.NumThreads = 2; + // config.TradesOnly = false; + // config.BufferSize = 2048; + // config.OverflowBufferSize = 4096; + // client = new Client(onTrade, onQuote, config); + + client = new EquitiesWebSocketClient(onTrade, onQuote); + await client.Start(); + timer = new Timer(TimerCallback, client, 60000, 60000); + await client.Join(); //Load symbols from your config or config.json + // await client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime + + // //You can also simulate a trading day by replaying a particular day's data. You can do this with the actual time between events, or without. + // DateTime yesterday = DateTime.Today - TimeSpan.FromDays(1); + // client = new ReplayClient(onTrade, onQuote, yesterday, false, true, false, "data.csv"); //A client to replay a previous day's data + // await client.Start(); + // timer = new Timer(TimerCallback, replayClient, 10000, 10000); + // await client.Join(); //Load symbols from your config or config.json + // // await client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime + + Console.CancelKeyPress += new ConsoleCancelEventHandler(Cancel); + } +} \ No newline at end of file diff --git a/SampleApp/OptionsSampleApp.cs b/SampleApp/OptionsSampleApp.cs new file mode 100644 index 0000000..bb5d2f4 --- /dev/null +++ b/SampleApp/OptionsSampleApp.cs @@ -0,0 +1,154 @@ +using System; +using System.Threading; +using System.Collections.Concurrent; +using Intrinio.Realtime; +using Intrinio.Realtime.Options; +using Serilog; +using Serilog.Core; + +namespace SampleApp; + +public class OptionsSampleApp +{ + private static IOptionsWebSocketClient client = null; + private static CandleStickClient _candleStickClient = null; + private static Timer timer = null; + private static UInt64 _tradeEventCount = 0UL; + private static UInt64 _quoteEventCount = 0UL; + private static UInt64 _refreshEventCount = 0UL; + private static UInt64 _unusualActivityEventCount = 0UL; + private static UInt64 _tradeCandleStickCount = 0UL; + private static UInt64 _tradeCandleStickCountIncomplete = 0UL; + private static UInt64 _AskCandleStickCount = 0UL; + private static UInt64 _AskCandleStickCountIncomplete = 0UL; + private static UInt64 _BidCandleStickCount = 0UL; + private static UInt64 _BidCandleStickCountIncomplete = 0UL; + private static bool _useTradeCandleSticks = false; + private static bool _useQuoteCandleSticks = false; + + static void OnQuote(Quote quote) + { + Interlocked.Increment(ref _quoteEventCount); + } + + static void OnTrade(Trade trade) + { + Interlocked.Increment(ref _tradeEventCount); + } + + static void OnRefresh(Refresh refresh) + { + Interlocked.Increment(ref _refreshEventCount); + } + + static void OnUnusualActivity(UnusualActivity unusualActivity) + { + Interlocked.Increment(ref _unusualActivityEventCount); + } + + static void OnTradeCandleStick(TradeCandleStick tradeCandleStick) + { + if (tradeCandleStick.Complete) + { + Interlocked.Increment(ref _tradeCandleStickCount); + } + else + { + Interlocked.Increment(ref _tradeCandleStickCountIncomplete); + } + } + + static void OnQuoteCandleStick(QuoteCandleStick quoteCandleStick) + { + if (quoteCandleStick.QuoteType == QuoteType.Ask) + if (quoteCandleStick.Complete) + Interlocked.Increment(ref _AskCandleStickCount); + else + Interlocked.Increment(ref _AskCandleStickCountIncomplete); + else + if (quoteCandleStick.Complete) + Interlocked.Increment(ref _BidCandleStickCount); + else + Interlocked.Increment(ref _BidCandleStickCountIncomplete); + } + + static void TimerCallback(object obj) + { + IOptionsWebSocketClient client = (IOptionsWebSocketClient) obj; + ClientStats stats = client.GetStats(); + Log("Socket Stats - Grouped Messages: {0}, Text Messages: {1}, Queue Depth: {2}%, Overflow Queue Depth: {3}%, Drops: {4}, Overflow Count: {5}, Individual Events: {6}, Trades: {7}, Quotes: {8}, Refreshes: {9}, UnusualActivities: {10}", + stats.SocketDataMessages, + stats.SocketTextMessages, + (stats.QueueDepth * 100) / stats.QueueCapacity, + (stats.OverflowQueueDepth * 100) / stats.OverflowQueueCapacity, + stats.DroppedCount, + stats.OverflowCount, + stats.EventCount, + client.TradeCount, + client.QuoteCount, + client.RefreshCount, + client.UnusualActivityCount); + + if (_useTradeCandleSticks) + Log("TRADE CANDLESTICK STATS - TradeCandleSticks = {0}, TradeCandleSticksIncomplete = {1}", _tradeCandleStickCount, _tradeCandleStickCountIncomplete); + if (_useQuoteCandleSticks) + Log("QUOTE CANDLESTICK STATS - Asks = {0}, Bids = {1}, AsksIncomplete = {2}, BidsIncomplete = {3}", _AskCandleStickCount, _BidCandleStickCount, _AskCandleStickCountIncomplete, _BidCandleStickCountIncomplete); + } + + static void Cancel(object sender, ConsoleCancelEventArgs args) + { + Log("Stopping sample app"); + timer.Dispose(); + client.Stop(); + if (_useTradeCandleSticks || _useQuoteCandleSticks) + { + _candleStickClient.Stop(); + } + Environment.Exit(0); + } + + [MessageTemplateFormatMethod("messageTemplate")] + static void Log(string messageTemplate, params object[] propertyValues) + { + Serilog.Log.Information(messageTemplate, propertyValues); + } + + public static async Task Run(string[] _) + { + Log("Starting sample app"); + Action onTrade = OnTrade; + Action onQuote = OnQuote; + Action onRefresh = OnRefresh; + Action onUnusualActivity = OnUnusualActivity; + + // //Subscribe the candlestick client to trade and/or quote events as well. It's important any method subscribed this way handles exceptions so as to not cause issues for other subscribers! + // _useTradeCandleSticks = true; + // _useQuoteCandleSticks = true; + // _candleStickClient = new CandleStickClient(OnTradeCandleStick, OnQuoteCandleStick, IntervalType.OneMinute, true, null, null, 0); + // onTrade += _candleStickClient.OnTrade; + // onQuote += _candleStickClient.OnQuote; + // _candleStickClient.Start(); + + // //You can either automatically load the config.json by doing nothing, or you can specify your own config and pass it in. + // //If you don't have a config.json, don't forget to also give Serilog a config so it can write to console + // Log.Logger = new LoggerConfiguration().WriteTo.Console(restrictedToMinimumLevel: Serilog.Events.LogEventLevel.Information).CreateLogger(); + // Config config = new Config(); + // config.Provider = Provider.REALTIME; + // config.ApiKey = "API_KEY_HERE"; + // config.Symbols = new[] { "AAPL", "MSFT" }; + // config.NumThreads = 16; + // config.TradesOnly = false; + // config.BufferSize = 2048; + // config.OverflowBufferSize = 4096; + // client = new Client(onTrade, onQuote, onRefresh, onUnusualActivity, config); + + client = new OptionsWebSocketClient(onTrade, onQuote, onRefresh, onUnusualActivity); + await client.Start(); + timer = new Timer(TimerCallback, client, 60000, 60000); + await client.Join(); //Load symbols from your config or config.json + // await client.JoinLobby(false); //Firehose + // await client.Join(new string[] { "AAPL", "GOOG", "MSFT" }, false); //Specify symbols at runtime + + Console.CancelKeyPress += new ConsoleCancelEventHandler(Cancel); + } +} \ No newline at end of file diff --git a/SampleApp/Program.cs b/SampleApp/Program.cs new file mode 100644 index 0000000..e4325f2 --- /dev/null +++ b/SampleApp/Program.cs @@ -0,0 +1,18 @@ +using System; +using System.Threading; +using System.Collections.Concurrent; +using Intrinio.Realtime.Equities; +using Serilog; +using Serilog.Core; + +namespace SampleApp +{ + class Program + { + static async Task Main(string[] args) + { + await EquitiesSampleApp.Run(args); + //await OptionsSampleApp.Run(args); + } + } +} diff --git a/SampleApp/SampleApp.csproj b/SampleApp/SampleApp.csproj new file mode 100644 index 0000000..0142e5e --- /dev/null +++ b/SampleApp/SampleApp.csproj @@ -0,0 +1,31 @@ + + + + Exe + net8.0 + enable + enable + true + true + en + + + + true + + + + + + + + + + + + + Always + + + + diff --git a/IntrinioRealTimeSDK/IntrinioRealTimeSDK.xml b/SampleApp/SampleApp.xml similarity index 64% rename from IntrinioRealTimeSDK/IntrinioRealTimeSDK.xml rename to SampleApp/SampleApp.xml index b581257..1242a00 100644 --- a/IntrinioRealTimeSDK/IntrinioRealTimeSDK.xml +++ b/SampleApp/SampleApp.xml @@ -1,8 +1,8 @@ - IntrinioRealTimeSDK + SampleApp - + \ No newline at end of file diff --git a/IntrinioRealTimeSDK/config.json b/SampleApp/config.json similarity index 68% rename from IntrinioRealTimeSDK/config.json rename to SampleApp/config.json index abec0c4..80149d2 100644 --- a/IntrinioRealTimeSDK/config.json +++ b/SampleApp/config.json @@ -1,14 +1,17 @@ { "Config": { - "ApiKey": "", - "NumThreads": 2, + "ApiKey": "API_KEY_HERE", + "NumThreads": 8, + //"Provider": "OPRA", "Provider": "REALTIME", //"Provider": "DELAYED_SIP", //"Provider": "NASDAQ_BASIC", //"Provider": "MANUAL", - "Symbols": [ "AAPL", "MSFT", "GOOG" ] + //"IPAddress": "1.2.3.4", + "BufferSize": 4096, + "OverflowBufferSize": 8192, + "Symbols": [ "AAPL", "MSFT", "TSLA" ] //"Symbols": [ "lobby" ] - //"IPAddress": "1.2.3.4" }, "Serilog": { "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], diff --git a/IntrinioRealtimeMultiExchange/runtimeconfig.template.json b/SampleApp/runtimeconfig.template.json similarity index 100% rename from IntrinioRealtimeMultiExchange/runtimeconfig.template.json rename to SampleApp/runtimeconfig.template.json