From 3062469ee1bd613388c617147b2a1e3edffdc1e6 Mon Sep 17 00:00:00 2001 From: David Stephens Date: Sun, 1 Oct 2017 10:16:07 -0600 Subject: [PATCH 1/5] TST: Update Fama French Test for changes to CRSP See: http://mba.tuck.dartmouth.edu/pages/faculty/ken.french/Data_Library/changes_crsp.html Please note: CRSP has recently completed the Pre62 Daily Data Series Project. The addition of these new daily data results in changes to month-end prices and to dividend ex-dates. These changes have resulted in many small changes to historical returns on my website. --- pandas_datareader/tests/test_famafrench.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas_datareader/tests/test_famafrench.py b/pandas_datareader/tests/test_famafrench.py index 51e3d437..be198c5a 100644 --- a/pandas_datareader/tests/test_famafrench.py +++ b/pandas_datareader/tests/test_famafrench.py @@ -39,10 +39,10 @@ def test_f_f_research(self): exp = pd.DataFrame({'Mkt-RF': [-3.36, 3.4, 6.31, 2., -7.89, -5.56, 6.93, -4.77, 9.54, 3.88, 0.6, 6.82], - 'SMB': [0.37, 1.19, 1.49, 4.99, 0.0, -2.01, 0.21, - -2.99, 3.92, 1.14, 3.68, 0.68], - 'HML': [0.3, 3.18, 2.15, 2.83, -2.41, -4.52, -0.21, - -1.96, -3.12, -2.52, -0.91, 3.78], + 'SMB': [0.38, 1.2, 1.42, 4.98, 0.05, -1.97, 0.16, + -3.00, 3.92, 1.15, 3.70, 0.7], + 'HML': [0.31, 3.16, 2.1, 2.81, -2.38, -4.5, -0.27, + -1.95, -3.12, -2.59, -0.9, 3.81], 'RF': [0., 0., 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01]}, index=pd.period_range('2010-01-01', '2010-12-01', From 7d8803d9707b9bb710982397ad8984773295722b Mon Sep 17 00:00:00 2001 From: David Stephens Date: Sun, 1 Oct 2017 10:22:18 -0600 Subject: [PATCH 2/5] BUG: Update Google URL --- pandas_datareader/google/daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas_datareader/google/daily.py b/pandas_datareader/google/daily.py index b232a072..e80aa703 100644 --- a/pandas_datareader/google/daily.py +++ b/pandas_datareader/google/daily.py @@ -31,7 +31,7 @@ class GoogleDailyReader(_DailyBaseReader): @property def url(self): - return 'http://www.google.com/finance/historical' + return 'http://finance.google.com/finance/historical' def _get_params(self, symbol): params = { From 08a700eba82c1bbc4c47aee64a4e752d15b36337 Mon Sep 17 00:00:00 2001 From: David Stephens Date: Sun, 1 Oct 2017 11:08:21 -0600 Subject: [PATCH 3/5] BUG: Skip tests for google quotes Add note to docs re: status of google quotes rebase --- docs/source/remote_data.rst | 10 ++++++---- pandas_datareader/google/quotes.py | 4 +++- pandas_datareader/tests/google/test_google.py | 16 ++++++++++++---- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/source/remote_data.rst b/docs/source/remote_data.rst index 02470e3f..d8d76f4f 100644 --- a/docs/source/remote_data.rst +++ b/docs/source/remote_data.rst @@ -170,11 +170,13 @@ Google Finance Quotes The GoogleQuotesReader class allows to get quotes data from Google Finance. -.. ipython:: python +OFFLINE AS OF OCT 1, 2017 - import pandas_datareader.data as web - q = web.get_quote_google(['AMZN', 'GOOG']) - q +.. .. ipython:: python + +.. import pandas_datareader.data as web +.. q = web.get_quote_google(['AMZN', 'GOOG']) +.. q .. _remote_data.google_options: diff --git a/pandas_datareader/google/quotes.py b/pandas_datareader/google/quotes.py index 416cdc23..d15bdf10 100644 --- a/pandas_datareader/google/quotes.py +++ b/pandas_datareader/google/quotes.py @@ -9,7 +9,9 @@ class GoogleQuotesReader(_BaseReader): - """Get current google quote""" + """Get current google quote + WARNING: OFFLINE AS OF OCT 1, 2017 + """ @property def url(self): diff --git a/pandas_datareader/tests/google/test_google.py b/pandas_datareader/tests/google/test_google.py index 47913a2c..167785ed 100644 --- a/pandas_datareader/tests/google/test_google.py +++ b/pandas_datareader/tests/google/test_google.py @@ -61,7 +61,7 @@ def test_google(self): for locale in self.locales: with tm.set_locale(locale): - panel = web.DataReader("F", 'google', start, end) + panel = web.DataReader("NYSE:F", 'google', start, end) assert panel.Close[-1] == 13.68 with pytest.raises(Exception): @@ -82,12 +82,14 @@ def assert_option_result(self, df): tm.assert_series_equal(df.dtypes, pd.Series(dtypes, index=exp_columns)) def test_get_quote_string(self): + pytest.skip("Google quote api is offline as of Oct 1, 2017") df = web.get_quote_google('GOOG') assert df.loc['GOOG', 'last'] > 0.0 tm.assert_index_equal(df.index, pd.Index(['GOOG'])) self.assert_option_result(df) def test_get_quote_stringlist(self): + pytest.skip("Google quote api is offline as of Oct 1, 2017") df = web.get_quote_google(['GOOG', 'AMZN', 'GOOG']) assert_series_equal(df.iloc[0], df.iloc[2]) tm.assert_index_equal(df.index, pd.Index(['GOOG', 'AMZN', 'GOOG'])) @@ -143,7 +145,10 @@ def test_get_multi2(self): def test_dtypes(self): # see gh-3995, gh-8980 - data = web.get_data_google('F', start='JAN-01-10', end='JAN-27-13') + data = web.get_data_google( + 'NYSE:F', + start='JAN-01-10', + end='JAN-27-13') assert np.issubdtype(data.Open.dtype, np.number) assert np.issubdtype(data.Close.dtype, np.number) assert np.issubdtype(data.Low.dtype, np.number) @@ -152,7 +157,10 @@ def test_dtypes(self): def test_unicode_date(self): # see gh-8967 - data = web.get_data_google('F', start='JAN-01-10', end='JAN-27-13') + data = web.get_data_google( + 'NYSE:F', + start='JAN-01-10', + end='JAN-27-13') assert data.index.name == 'Date' def test_google_reader_class(self): @@ -167,4 +175,4 @@ def test_google_reader_class(self): def test_bad_retry_count(self): with pytest.raises(ValueError): - web.get_data_google('F', retry_count=-1) + web.get_data_google('NYSE:F', retry_count=-1) From 526eb9322237652033d7691b90dc53546fcdd648 Mon Sep 17 00:00:00 2001 From: David Stephens Date: Sun, 1 Oct 2017 11:32:31 -0600 Subject: [PATCH 4/5] TST: Fix test_yahoo Uses non-weekend date for test, using a weekend date is ambiguous squash --- pandas_datareader/tests/yahoo/test_yahoo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas_datareader/tests/yahoo/test_yahoo.py b/pandas_datareader/tests/yahoo/test_yahoo.py index d579d38c..5eec8c0a 100644 --- a/pandas_datareader/tests/yahoo/test_yahoo.py +++ b/pandas_datareader/tests/yahoo/test_yahoo.py @@ -25,7 +25,7 @@ def setup_class(cls): def test_yahoo(self): # Asserts that yahoo is minimally working start = datetime(2010, 1, 1) - end = datetime(2013, 1, 27) + end = datetime(2013, 1, 25) assert web.DataReader('F', 'yahoo', start, end)['Close'][-1] == 13.68 @@ -205,7 +205,7 @@ def test_yahoo_reader_class(self): r = YahooDailyReader('GOOG') df = r.read() - assert df.Volume.loc['JAN-02-2015'] == 1447500 + assert df.Volume.loc['JAN-02-2015'] == 1447600 session = requests.Session() From 4dfafd7495e77e1dd9dd3cb25e1155163daee8b7 Mon Sep 17 00:00:00 2001 From: David Stephens Date: Sun, 1 Oct 2017 11:33:49 -0600 Subject: [PATCH 5/5] BUG: Drop blank data from yahoo --- pandas_datareader/yahoo/daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas_datareader/yahoo/daily.py b/pandas_datareader/yahoo/daily.py index 87de2e52..971c8fb5 100644 --- a/pandas_datareader/yahoo/daily.py +++ b/pandas_datareader/yahoo/daily.py @@ -117,7 +117,7 @@ def read(self): df['Ret_Index'] = _calc_return_index(df['Adj Close']) if self.adjust_price: df = _adjust_prices(df) - return df.sort_index() + return df.sort_index().dropna(how='all') finally: self.close()