Notebook
In [1]:
from quantopian.pipeline import Pipeline, CustomFactor
from quantopian.pipeline.filters import QTradableStocksUS, StaticSids
from quantopian.pipeline.data import Fundamentals
from quantopian.research import run_pipeline
import numpy as np

Number of unique asof_dates for each Fundamental factor

This notebook reveals the total number of unique asof_dates for each and every Fundamental factor for all assets in the QTradableStocksUS universe. Window_size is set to 10, so we should expect 10 unique dates for factors that change daily, and 1 unique date for most factors that change less frequently. However, the results do not match expectations.

In [33]:
universe = QTradableStocksUS()

class Unique(CustomFactor):
    mask = universe
    def compute(self, today, assets, out, inputs):
        out[:] = np.array([np.unique(a).shape[0] for a in inputs.T])
def make_pipeline():
    return Pipeline(
        columns = {
            x.name[:-10]: Unique(
                inputs=[x,],
                window_length = 10
            )
            for x in Fundamentals.columns if x.name.endswith('_asof_date')
        },
        screen = universe
    )
start_date, end_date = '2018-03-19', '2018-03-19'
df = run_pipeline(make_pipeline(), start_date, end_date)
df.index = df.index.levels[1]
df
Out[33]:
accession_number accession_number_earnings_reports accounts_payable accounts_receivable accretion_on_preferred_stock accrued_interest_receivable accrued_investment_income accrued_liabilities_total accrued_preferred_stock_dividends accruedand_deferred_income ... work_in_process work_performedby_entityand_capitalized working_capital working_capital_per_share working_capital_per_share3_yr_avg working_capital_per_share5_yr_avg working_capital_turnover_ratio write_down write_off yearof_establishment
Equity(2 [ARNC]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(24 [AAPL]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(31 [ABAX]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(41 [ARCB]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(52 [ABM]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(53 [ABMD]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(62 [ABT]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(64 [ABX]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(67 [ADSK]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(76 [TAP]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(110 [ACXM]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(114 [ADBE]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(122 [ADI]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(128 [ADM]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(154 [AEM]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(161 [AEP]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(166 [AES]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(185 [AFL]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(197 [AGCO]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(216 [HES]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(239 [AIG]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(247 [AIN]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(253 [AIR]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(266 [AJG]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(289 [MATX]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(300 [ALK]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(301 [ALKS]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(311 [ALOG]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(337 [AMAT]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(351 [AMD]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Equity(50690 [CJ]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50713 [GOOS]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50716 [DXC]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(50718 [PUMP]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50719 [MULE]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50735 [AYX]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50749 [BTU]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50758 [OKTA]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50763 [SNDR]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50779 [CADE]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50780 [HCC]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50781 [YEXT]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50782 [UPL]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(50789 [WTTR]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 1.0
Equity(50798 [FND]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50807 [CVNA]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50810 [CLDR]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50824 [BCEI]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 10.0 10.0 1.0 1.0 1.0 10.0
Equity(50839 [BHVN]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50860 [VST]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50869 [GDI]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50870 [SOI]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50891 [CARS]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50900 [SGH]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50902 [WOW]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50903 [APPN]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50910 [JHG]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50968 [GPMT]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50970 [ATUS]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0
Equity(50990 [APRN]) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 ... 1.0 1.0 1.0 10.0 1.0 1.0 1.0 1.0 1.0 10.0

2155 rows × 1124 columns

Listed here are all factors that are updated daily for every asset. Clearly, most all of these are all fixed and should never be updated. The daily asof_dates for fixed values creates uncertainty, as will be discussed in the next section.

In [27]:
print '\n'.join(df.sum(0).pipe(lambda s: s[s == 21550]).index)
business_country_id
company_status
conversion_ratio
country_id
currency_id
exchange_id
fiscal_year_end
industry_template_code
investment_id
is_depositary_receipt
is_direct_invest
is_dividend_reinvest
is_limited_partnership
is_primary_share
is_reit
legal_name
legal_name_language_code
mic
primary_exchange_id
primary_mic
primary_share_class_id
primary_symbol
report_style
security_type
share_class_status
short_name
standard_name
symbol

AAPL Daily asof_dates

Problems arise when we have daily asof_dates, yet cannot be sure if the values are supposed to change.

For AAPL we have daily asof_date for all the following factors. Clearly, many of these factors are only set one time, like 'cik'. Some, if they do change, we'd like to easily determine when they changed, like 'exchange_id'. But unfortunately the only way to know when any of these values changed is to scan the actual value for every date and detect changes in the values.

Even worse, values that might change daily must not only be guessed from the daily asof_dates, but also confirmed by checking that the actual values change daily as well. For instance, 'cf_yield' looks like it would update daily, but does it? We'd have to check that the actual values change. What if on some days the values change and others they do not? Does that mean the values are updated only occasionally?

In [39]:
print '\n'.join(df.iloc[1].pipe(lambda s: s[s==10]).index)
advisor
advisor_language_code
auditor
auditor_language_code
book_value_per_share
book_value_yield
business_country_id
buy_back_yield
cape_ratio
cash_return
cf_yield
cfo_per_share
cik
company_status
conversion_ratio
country_id
currency_id
div_yield5_year
e_vto_ebit
e_vto_fcf
e_vto_pre_tax_income
e_vto_revenue
e_vto_total_assets
earning_yield
enterprise_value
ev_to_ebitda
ev_to_forward_ebit
ev_to_forward_ebitda
ev_to_forward_revenue
exchange_id
expected_dividend_growth_rate
fcf_per_share
fcf_ratio
fcf_yield
financial_health_grade
fiscal_year_end
forward_calculation_style
forward_dividend
forward_dividend_yield
forward_earning_yield
forward_pe_ratio
forward_roa
forward_roe
industry_template_code
investment_id
ipo_date
is_depositary_receipt
is_direct_invest
is_dividend_reinvest
is_limited_partnership
is_primary_share
is_reit
legal_name
legal_name_language_code
market_cap
mic
normalized_pe_ratio
p_cash_ratio3_yr_avg
payout_ratio
pb_ratio
pb_ratio10_year_growth
pb_ratio1_year_growth
pb_ratio3_year_growth
pb_ratio3_yr_avg
pb_ratio3_yr_avg_change
pb_ratio5_year_growth
pcf_ratio
pe_ratio
pe_ratio10_year_growth
pe_ratio1_year_growth
pe_ratio3_year_growth
pe_ratio3_yr_avg
pe_ratio3_yr_avg_change
pe_ratio5_year_growth
peg_payback
peg_ratio
pfcf_ratio10_year_growth
pfcf_ratio1_year_growth
pfcf_ratio3_year_growth
pfcf_ratio3_yr_avg
pfcf_ratio3_yr_avg_change
pfcf_ratio5_year_growth
price_change1_m
priceto_cash_ratio
priceto_ebitda
primary_exchange_id
primary_mic
primary_share_class_id
primary_symbol
ps_ratio
ps_ratio10_year_growth
ps_ratio1_year_growth
ps_ratio3_year_growth
ps_ratio3_yr_avg
ps_ratio3_yr_avg_change
ps_ratio5_year_growth
report_style
sales_per_share
sales_yield
security_type
share_class_status
short_name
standard_name
sustainable_growth_rate
symbol
tangible_book_value_per_share
tangible_bv_per_share3_yr_avg
tangible_bv_per_share5_yr_avg
total_asset_per_share
total_yield
trailing_dividend_yield
two_years_forward_earning_yield
two_years_forward_pe_ratio
working_capital_per_share
working_capital_per_share3_yr_avg
working_capital_per_share5_yr_avg
yearof_establishment

To further illustrate the issue, we have here for AAPL a financial ratio determined from the quarterly balance sheet, but it is shown to be updated daily by its asof_dates. This raises the question of what happens with a stock split? I guess the asof_date would continue updating daily, but would the value reflect the stock split? Probably not, since this is a balance sheet ratio. But the why list it as having a daily asof_date.

In [26]:
def make_pipeline():
    return Pipeline(
        columns = {
            'factor': Fundamentals.book_value_per_share.latest,
            '_asof_date': Fundamentals.book_value_per_share_asof_date.latest,
        },
        screen = StaticSids([24])
    )
start_date, end_date = '2018-03-12', '2018-03-19'
run_pipeline(make_pipeline(), start_date, end_date)
Out[26]:
_asof_date factor
2018-03-12 00:00:00+00:00 Equity(24 [AAPL]) 2018-03-09 27.6308
2018-03-13 00:00:00+00:00 Equity(24 [AAPL]) 2018-03-12 27.6308
2018-03-14 00:00:00+00:00 Equity(24 [AAPL]) 2018-03-13 27.6308
2018-03-15 00:00:00+00:00 Equity(24 [AAPL]) 2018-03-14 27.6308
2018-03-16 00:00:00+00:00 Equity(24 [AAPL]) 2018-03-15 27.6308
2018-03-19 00:00:00+00:00 Equity(24 [AAPL]) 2018-03-16 27.6308