Roll Report Crashing

[t3chap]

I'm running the interactive diagnostics, trying to generate a roll report. For some reason this crashes and the errors would seem to indicate that the system doesn't have data for the MXP/20240300 contract.

Instrument code?(Return for ALL)MXP
1: Print or 2: email or 3: file or 4: email and file? <RETURN for Print> 4
2024-01-29 14:25:03 DEBUG Interactive-Diagnostics Running report Roll report sysproduction.reporting.roll_report.roll_report emailfile {'instrument_code': 'MXP'}
Traceback (most recent call last):
  File "/home/tim/pysystemtrade/syscore/cache.py", line 48, in _get_from_store
    value = self.store[key]
            ~~~~~~~~~~^^^^^
KeyError: "_get_roll_data_dict/('MXP',)/{}"

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/pysystemtrade/syscore/cache.py", line 29, in get
    value_from_store = self._get_from_store(key)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/syscore/cache.py", line 50, in _get_from_store
    raise missingData("Missing cache element %s" % key) from e
syscore.exceptions.missingData: Missing cache element _get_roll_data_dict/('MXP',)/{}

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/tim/pysystemtrade/sysproduction/interactive_diagnostics.py", line 78, in interactive_diagnostics
    menu.run_menu()
  File "/home/tim/pysystemtrade/syscore/interactive/menus.py", line 64, in run_menu
    method_chosen(*self._args, **self._kwargs)
  File "/home/tim/pysystemtrade/sysproduction/interactive_diagnostics.py", line 175, in roll_report
    run_report(report_config, data=data)
  File "/home/tim/pysystemtrade/sysproduction/reporting/reporting_functions.py", line 63, in run_report
    run_report_with_data_blob(report_config, data)
  File "/home/tim/pysystemtrade/sysproduction/reporting/reporting_functions.py", line 75, in run_report_with_data_blob
    report_results = run_report_from_config(report_config=report_config, data=data)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/reporting_functions.py", line 85, in run_report_from_config
    report_results = report_function(data, **report_kwargs)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/roll_report.py", line 40, in roll_report
    formatted_output.append(reporting_api.table_of_roll_data(instrument_code))
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/api.py", line 555, in table_of_roll_data
    result_pd = self._roll_data_as_pd(instrument_code)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/api.py", line 562, in _roll_data_as_pd
    roll_data_dict = self.roll_data_dict(instrument_code)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/api.py", line 571, in roll_data_dict
    return self.cache.get(self._get_roll_data_dict, instrument_code)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/syscore/cache.py", line 31, in get
    value_from_store = self._calculate_and_store(
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/syscore/cache.py", line 38, in _calculate_and_store
    value = function_instance(*args, **kwargs)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/api.py", line 582, in _get_roll_data_dict
    roll_data = get_roll_data_for_instrument(instrument_code, data)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/reporting/data/rolls.py", line 47, in get_roll_data_for_instrument
    price_expiry_days = c_data.days_until_price_expiry(instrument_code)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/data/contracts.py", line 65, in days_until_price_expiry
    price_expiry = self.get_priced_expiry(instrument_code)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/data/contracts.py", line 231, in get_priced_expiry
    return self.get_actual_expiry(instrument_code, contract_id)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/data/contracts.py", line 205, in get_actual_expiry
    contract_object = self.get_contract_from_db_given_code_and_id(
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysproduction/data/contracts.py", line 198, in get_contract_from_db_given_code_and_id
    contract_object = self.db_contract_data.get_contract_object(
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/tim/pysystemtrade/sysdata/futures/contracts.py", line 40, in get_contract_object
    raise ContractNotFound(
syscore.exceptions.ContractNotFound: Contract MXP/20240300 not found

However, I can find this data for the individual contracts using the interactive diagnostics:

Available contract dates ['19950600', '19950900', '19951200', '19960300', '19960600', '19960900', '19961200', '19970300', '19970600', '19970900', '19971200', '19980300', '19980600', '19980900', '19981200', '19990300', '19990600', '19990900', '19991200', '20000300', '20000600', '20000900', '20001200', '20010300', '20010600', '20010900', '20011200', '20020300', '20020600', '20020900', '20021200', '20030300', '20030600', '20030900', '20031200', '20040300', '20040600', '20040900', '20041200', '20050300', '20050600', '20050900', '20051200', '20060300', '20060600', '20060900', '20061200', '20070300', '20070600', '20070900', '20071200', '20080300', '20080600', '20080900', '20081200', '20090300', '20090600', '20090900', '20091200', '20100300', '20100600', '20100900', '20101200', '20110300', '20110600', '20110900', '20111200', '20120300', '20120600', '20120900', '20121200', '20130300', '20130600', '20130900', '20131200', '20140300', '20140600', '20140900', '20141200', '20150300', '20150600', '20150900', '20151200', '20160300', '20160600', '20160900', '20161200', '20170300', '20170600', '20170900', '20171200', '20180300', '20180600', '20180900', '20181200', '20190300', '20190600', '20190900', '20191200', '20200300', '20200600', '20200900', '20201200', '20210300', '20210600', '20210900', '20211200', '20220300', '20220600', '20220900', '20221200', '20230300', '20230600', '20230900', '20231200', '20240300p', '20240600f', '20240900', '20241200', '20250300', '20250600']
p = currently priced, c=current carry, f= current forward
Contract date? [yyyymm or yyyymmdd] (ignore suffixes)

And I'm able to see the data if I proceed further.
Any idea where to start looking into how to fix this?
It appears that the system does have data for this instrument and contract.
Thanks.

[t3chap]

And strangely enough, when I continue to run the interactive diagnostics script, and attempt to see what it shows for MXP, I get the following:

 INTERACTIVE DIAGNOSTICS


0: backtest objects
1: View instrument configuration
2: Emails
3: View prices
4: View capital
5: View positions & orders
6: Reports


Your choice? <RETURN for EXIT> 1
10: View instrument configuration data
11: View contract configuration data
12: View trading hours for all instruments


Your choice? <RETURN for Back> 11
Instrument code?MXP
MXP is not an instrument with contract data
Instrument code?

Clearly something isn't imported or referenced correctly. I'm able to run the backtests in a production environment and generate orders, enter orders manually, etc.

[t3chap]

I'm trying to sort through the layers of abstraction and find out where the contracts are supposed to be stored. Now that we've moved to parquet, I'm assuming they're there somewhere? Or are they still in mongodb?
I'm wondering if something didn't get added to a table somewhere that required it, but I'm not certain of where to look.

The multiple and adjusted files can be pulled up and have data for MXP.
I get the same result for any instrument that I pull up, also.

If I run the following:

from sysdata.data_blob import dataBlob
from sysproduction.data.contracts import dataContracts
with dataBlob(log_name="Interactive-Diagnostics") as data:
    diag_contracts = dataContracts(data)

diag_contracts.db_contract_data.get_all_contract_objects_for_instrument_code('MXP')

Gives me an empty list.

data.db_futures_contract_price.contracts_with_merged_price_data_for_instrument_code('MXP')

Gives me

[MXP/20050900,
 MXP/20240600,
 MXP/20200600,
 MXP/20220900,
 MXP/20080600,
...
...
...
 MXP/20081200]

I'm not sure what the meaning of 'merged' is in this case, I haven't found it in the documentation yet. But it does appear that the MXP data is in the blob, at least in some form.

I've been playing around with the datablob but haven't figured this one out yet, any help or direction appreciated.

[robcarver17]

Contract objects are not time series so they are in mongo still not parquet

You don't need to wade through abstraction, looking at the sysproduction.data.contracts.dataContracts is exactly the right level

Icnidentally the dataBlob doesn't have stuff inside it, it's just a way of collecting together the access to the data layer abstractions like db_futures_contract_price without worrying about where they are or in what format.

Merged prices are just a merge of daily and hourly sampled prices.

This should be fixed if you run update_sampled_contracts which is the script that sets these up

EDIT: Looking at the other thread this won't work unless you connect to IB.

[t3chap]

Thanks for the reply Rob. I have figured out a compromise to be able to access IB when the stock market isn't open so pysystemtrade can get necessary data. I'm using the incompatible framework with ib-insnyc during stock trading hours so I was able to get this working. I'll still have to work something out with a separate account or at least subaccount for trading or some other workaround to have a fully functional PST.

I did run into a problem which I ultimately traced to fileutils.py.
When I first ran update_sampled_contracts I got the following error:

Traceback (most recent call last):
  File "/home/tim/pysystemtrade/sysdata/csv/csv_roll_parameters.py", line 17, in read_from_file
    roll_data = pd.read_csv(filename)
  File "/home/tim/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 948, in read_csv
    return _read(filepath_or_buffer, kwds)
  File "/home/tim/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 611, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "/home/tim/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 1448, in __init__
    self._engine = self._make_engine(f, self.engine)
  File "/home/tim/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 1705, in _make_engine
    self.handles = get_handle(
  File "/home/tim/anaconda3/lib/python3.10/site-packages/pandas/io/common.py", line 863, in get_handle
    handle = open(
FileNotFoundError: [Errno 2] No such file or directory: '/home/tim/pysystemtrade/sysproduction/data/futures/csvconfig/rollconfig.csv'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 453, in <module>
    update_sampled_contracts()
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 45, in update_sampled_contracts
    update_contracts_object.update_sampled_contracts(
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 73, in update_sampled_contracts
    update_active_contracts_with_data(data, instrument_code=instrument_code)
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 86, in update_active_contracts_with_data
    update_active_contracts_for_instrument(instrument_code, data)
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 92, in update_active_contracts_for_instrument
    required_contract_chain = get_contract_chain(data, instrument_code)
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 108, in get_contract_chain
    furthest_out_contract = get_furthest_out_contract_with_roll_parameters(
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 123, in get_furthest_out_contract_with_roll_parameters
    create_furthest_out_contract_with_roll_parameters_from_contract_date(
  File "/home/tim/pysystemtrade/sysproduction/update_sampled_contracts.py", line 146, in create_furthest_out_contract_with_roll_parameters_from_contract_date
    roll_parameters = diag_contracts.get_roll_parameters(instrument_code)
  File "/home/tim/pysystemtrade/sysproduction/data/contracts.py", line 185, in get_roll_parameters
    roll_parameters = self.db_roll_parameters.get_roll_parameters(instrument_code)
  File "/home/tim/pysystemtrade/sysdata/futures/rolls_parameters.py", line 35, in get_roll_parameters
    if self.is_code_in_data(instrument_code):
  File "/home/tim/pysystemtrade/sysdata/futures/rolls_parameters.py", line 87, in is_code_in_data
    if instrument_code in self.get_list_of_instruments():
  File "/home/tim/pysystemtrade/sysdata/csv/csv_roll_parameters.py", line 80, in get_list_of_instruments
    all_roll_parameters = self.get_roll_parameters_all_instruments()
  File "/home/tim/pysystemtrade/sysdata/csv/csv_roll_parameters.py", line 114, in get_roll_parameters_all_instruments
    return allRollParameters.read_from_file(self.config_file)
  File "/home/tim/pysystemtrade/sysdata/csv/csv_roll_parameters.py", line 19, in read_from_file
    raise Exception("Can't read file %s" % filename)
Exception: Can't read file /home/tim/pysystemtrade/sysproduction/data/futures/csvconfig/rollconfig.csv

The problem is the csv files are in the data directory directly under the pysystemtrade "root" directory.
csv_roll_parameters.py has these definitions:

ROLLS_DATAPATH = "data.futures.csvconfig"
ROLLS_CONFIG_FILE = "rollconfig.csv"

However, as pathnames are resolved by fileutils, this code snippet runs:

def get_relative_pathname_from_list(path_as_list: List[str]) -> str:
    """

    >>> get_relative_pathname_from_list(['syscore','tests'])
    '/home/rob/pysystemtrade/syscore/tests'
    """
    package_name = path_as_list[0]
    paths_or_files = path_as_list[1:]

    if len(paths_or_files) == 0:
        directory_name_of_package = os.path.dirname(
            import_module(package_name).__file__
        )
        return directory_name_of_package

    last_item_in_list = path_as_list.pop()
    pathname = os.path.join(
        get_relative_pathname_from_list(path_as_list), last_item_in_list
    )

    return pathname

Since the data directory doesn't appear to be registered as a package, the code above resolves in my case to the home/tim/pysystemtrade/sysproduction/data directory, which isn't where the csv files needed are located.

I wrote a terribly hacky and poorly thought out patch so I could populate the sampled_contracts and catch up on the administrative tasks I'm trying to get to work with PST. I'm certain there is a better approach but this worked for me in the limited time that I had. It did appear to work and didn't appear to break anything else.

The following hacks are in fileutils.py:

def get_pathname_from_list(path_as_list: List[str]) -> str:
    """
    >>> get_pathname_from_list(['C:', 'home', 'rob'])
    'C:\\home\\rob'
    >>> get_pathname_from_list(['','home','rob'])
    '/home/rob'
    >>> get_pathname_from_list(['syscore','tests'])
    '/home/rob/pysystemtrade/syscore/tests'
    """
    if path_as_list[0] == "":
        # path_type_absolute
        resolved_pathname = get_absolute_linux_pathname_from_list(path_as_list[1:])
    elif is_windoze_path_list(path_as_list):
        # windoze
        resolved_pathname = get_absolute_windows_pathname_from_list(path_as_list)
    elif path_as_list[0] =="data":
        #use the data directory, not the sysproduction/data
        resolved_pathname = get_data_pathname_from_list(path_as_list)
    else:
        # relative
        resolved_pathname = get_relative_pathname_from_list(path_as_list)

    return resolved_pathname


def get_data_pathname_from_list(path_as_list: List[str]) -> str:
    """

    >>> get_data_pathname_from_list(['data','futures','csvconfig'])
    '/home/rob/pysystemtrade/data/futures/csvconfig/'
    """
    raw_pathname = os.getcwd()
    # Split the path using 'pysystemtrade' as the delimiter
    left_part, _ = raw_pathname.split('pysystemtrade', 1)
    # Rejoin the left part
    final_path = left_part + 'pysystemtrade'
    
    pathname = os.path.join(
        final_path, *path_as_list
    )
    return pathname