Google Search Volume Combined with Extent of Press News Predicts Stocks’ Returns

1from AlgorithmImports import *
2from dateutil.relativedelta import relativedelta
3from typing import List, Dict
4#endregion
5class GoogleSearchVolumeCombinedWithExtentOfPressNewsPredictsStocksReturns(QCAlgorithm):
6def Initialize(self) -> None:
7 self.SetStartDate(2004, 1, 1) # News data are since 2010 and Google search data are since 2004
8 self.SetCash(100_000)
9 
10 self.leverage: int = 5
11 self.data: Dict[str, SymbolData] = {}
12 self.tickers: List[str] = []
13 self.selected: Dict[str, Symbol] = {}
14 self.symbols_tickers: Dict[Symbol, str] = {}
15 
16 self.stocks_news_last_date: datetime.date = datetime(1,1,1).date()
17 # Data source: https://www.nasdaq.com/market-activity/stocks/aapl/news-headlines
18 # dates in dictionaries aren't sorted in ascending or descending way
19 csv_string_file: str = self.Download('data.quantpedia.com/backtesting_data/index/stocks_news.csv')
20 lines: List[str] = csv_string_file.split('\r\n')
21 
22 columns: List[str] = lines[0].split(';')
23 
24 # subscribe to Quantpedia Google Search data and create SymbolData object for each stock's ticker
25 for ticker in columns[1:]:
26     # convert ticker to upper to match Symbol.Value from quantconnect
27     ticker: str = ticker.upper()
28     
29     # subscribe to QuantpediaGoogleSeach data
30     symbol: Symbol = self.AddData(QuantpediaGoogleSearch, ticker, Resolution.Daily).Symbol
31     self.symbols_tickers[symbol] = ticker
32     
33     # create SymbolData object for current stock
34     self.data[ticker] = SymbolData()
35     
36     self.tickers.append(ticker)
37 
38 for line in lines[1:]: # skip header line
39     if line == '':
40         continue
41     
42     # split line
43     line: List[str] = line.split(';')
44     
45     # convert string to date
46     str_date = line[0]
47     date: datetime.date = datetime.strptime(str_date, '%d.%m.%Y').date()
48     if date > self.stocks_news_last_date:
49         self.stocks_news_last_date = date
50     for index in range(1, len(line)): # skip date as a first value of the row
51         # retrive ticker from list created based on csv header
52         ticker: str = self.tickers[index - 1]
53         # get total stock's news for specific date
54         total_news: str = line[index]
55         
56         # initialize dictionary for total stock's news for specific date
57         self.data[ticker].news[date] = float(total_news)
58 
59 market: Symbol = self.AddEquity('SPY', Resolution.Daily).Symbol
60 
61 self.selection_flag: bool = False
62 self.UniverseSettings.Leverage = self.leverage
63 self.UniverseSettings.Resolution = Resolution.Daily
64 self.AddUniverse(self.FundamentalSelectionFunction)
65 self.Settings.MinimumOrderMarginPortfolioPercentage = 0.
66 self.settings.daily_precise_end_time = False
67 self.Schedule.On(self.DateRules.MonthStart(market), self.TimeRules.AfterMarketOpen(market), self.Selection)
68def OnSecuritiesChanged(self, changes: SecurityChanges) -> None:
69 for security in changes.AddedSecurities:
70     security.SetFeeModel(CustomFeeModel())
71 
72def FundamentalSelectionFunction(self, fundamental: List[Fundamental]) -> List[Symbol]:
73 # rebalance monthly
74 if not self.selection_flag:
75     return Universe.Unchanged
76         
77 # Select S&P 100 stocks
78 self.selected = {x.Symbol.Value: x.Symbol for x in fundamental if x.Symbol.Value in self.data}
79         
80 return list(self.selected.values())
81def OnData(self, slice: Slice) -> None:
82 custom_data_last_update: Dict[Symbol, datetime.date] = QuantpediaGoogleSearch.get_last_update_date()
83 if self.time.date() > self.stocks_news_last_date:
84     self.Liquidate()
85     return
86 # update google search values for each stock
87 for google_search_symbol, stock_ticker in self.symbols_tickers.items():
88     if self.securities[google_search_symbol].get_last_data() and self.time.date() > custom_data_last_update[google_search_symbol]:
89         self.liquidate(stock_ticker)
90         continue
91     # check if google search value exists
92     if slice.contains_key(google_search_symbol) and slice[google_search_symbol]:
93         # retrive and update google search value
94         google_search_value: float = slice[google_search_symbol].Value
95         self.data[stock_ticker].update_google_search_values(google_search_value)
96 
97 # rebalance monthly
98 if not self.selection_flag:
99     return
100 self.selection_flag = False
101 
102 search_increase: List[Symbol] = [] # storing symbol of stocks, which had google search increase 
103 news_increase: List[Symbol] = [] # storing symbol of stocks, which had news increase
104 
105 search_decrease: List[Symbol] = [] # storing symbol of stocks, which had google search decrease
106 news_decrease: List[Symbol] = [] # storing symbol of stocks, which had news decrease
107 
108 # calculate stocks search intensity and news intensity
109 for stock_ticker, symbol_obj in self.data.items():
110     # make sure stock has stock symbol from fundamental function
111     if stock_ticker not in self.selected:
112         continue
113     
114     # make sure google search data are ready
115     if not symbol_obj.is_google_search_ready():
116         continue
117     
118     # get stock symbol based on it's ticker
119     stock_symbol: Symbol = self.selected[stock_ticker]
120     
121     # check if stock is in increase of google search for this month according to last twelve months mean
122     google_search_increase_flag: bool = symbol_obj.google_search_increase() # return False/True/None
123     
124     if google_search_increase_flag is True:
125         search_increase.append(stock_symbol)
126     elif google_search_increase_flag is False:
127         search_decrease.append(stock_symbol)
128     
129     current_date: datetime.date = self.Time.date()
130     twelve_months_before: datetime.date = current_date - relativedelta(months=13) # substract 13 months, because rebalancing is at the start of the month
131     # check if stock is in increase of total_news for this month according to last twelve months mean
132     news_increase_flag: bool = symbol_obj.news_increase(current_date, twelve_months_before) # return False/True/None
133         
134     if news_increase_flag is True:
135         news_increase.append(stock_symbol)
136     elif news_increase_flag is True:
137         news_decrease.append(stock_symbol)
138     
139 # long stocks, which had increase of news and google search
140 long: List[Symbol] = [stock_symbol for stock_symbol in news_increase if stock_symbol in search_increase]
141 
142 # short stocks, which had decrease of news and google search
143 short: List[Symbol] = [stock_symbol for stock_symbol in news_decrease if stock_symbol in search_decrease]
144 
145 long_length = len(long)
146 short_length = len(short)   
147         
148 # trade execution
149 targets: List[PortfolioTarget] = []
150 for i, portfolio in enumerate([long, short]):
151     for symbol in portfolio:
152         if slice.contains_key(symbol) and slice[symbol]:
153             targets.append(PortfolioTarget(symbol, ((-1) ** i) / len(portfolio)))
154 
155 self.SetHoldings(targets, True)
156 
157def Selection(self) -> None:
158 self.selection_flag = True
159 
160class SymbolData():
161def __init__(self) -> None:
162 self.google_search_values: List[float] = []
163 self.news: Dict[datetime.date, float] = {}
164 
165def update_google_search_values(self, google_search_value: float) -> None:
166 self.google_search_values.append(google_search_value)
167 
168def is_google_search_ready(self) -> bool:
169 # return True if there are atleast twelve months of google search values
170 if len(self.google_search_values) >= 12:
171     return True
172 else:
173     return False
174     
175def google_search_increase(self) -> bool:
176 # check if stock has no search volume
177 if self.google_search_values[-1] == 0:
178     # stock has no search volume
179     return None
180 
181 max_value: float = max(self.google_search_values)
182 
183 twelve_months_values: List[float] = []
184 
185 # normalize and store google search values for last 12 months
186 for google_search_value in self.google_search_values[-12:]:
187     if google_search_value != 0:
188         twelve_months_values.append(google_search_value / max_value)
189     else:
190         twelve_months_values.append(0)
191 
192 twelve_months_mean: float = np.mean(twelve_months_values)
193 
194 last_value: float = self.google_search_values[-1] / max_value
195 
196 # check if google search volume increased
197 if last_value > twelve_months_mean:
198     # google search volume increased
199     return True
200 else:
201     # google search volume decreased
202     return False
203     
204def news_increase(self, current_date: datetime.date, twelve_months_before: datetime.date) -> bool:
205 news_data: Dict[str, np.ndarray]  = {} # storing numpy array of total news values keyed by string of it's month and year
206 prev_month_year_string: str = str(current_date.month - 1) + str(current_date.year)
207 current_month_year_string: str = str(current_date.month) + str(current_date.year)
208 
209 for date, total_news in self.news.items():
210     # check if date is in period last twelve months from current date
211     if date >= twelve_months_before and date <= current_date:
212         year_month_string: str = str(date.month) + str(date.year)
213         
214         # initialize list if dictionary doesn't contain string of month and year as a key
215         if year_month_string not in news_data:
216             news_data[year_month_string] = np.array([])
217         
218         # append new total news value to numpy array    
219         news_data[year_month_string] = np.append(news_data[year_month_string], total_news)
220 
221 if current_month_year_string in news_data:     
222     # delete current month data, because only prev 12 months are included in calculation
223     del news_data[current_month_year_string]
224         
225 # stock doesn't have twelve months data of news or stock doesn't have any news volume for current month
226 if len(news_data) < 12 or prev_month_year_string not in news_data or sum(news_data[prev_month_year_string]) == 0:
227     return None
228 
229 # create list of total news for each month in twelve months period    
230 twelve_months_news_list: List[float] = [sum(month_news_array) for _, month_news_array in news_data.items()]
231 
232 # calculate mean of twelve months news
233 twelve_months_news_mean: float = np.mean(twelve_months_news_list)
234    
235 # retrieve total number of last month news
236 last_month_news_value: float = sum(news_data[prev_month_year_string])
237 
238 # check if news volume increased
239 if last_month_news_value > twelve_months_news_mean:
240     # news volume increased
241     return True
242 else:
243     # news volume decreased
244     return False
245 
246# Custom fee model.
247class CustomFeeModel(FeeModel):
248def GetOrderFee(self, parameters: OrderFeeParameters) -> OrderFee:
249 fee: float = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
250 return OrderFee(CashAmount(fee, "USD"))
251 
252# Quantpedia data.
253# NOTE: IMPORTANT: Data order must be ascending (datewise)
254class QuantpediaGoogleSearch(PythonData):
255_last_update_date:Dict[Symbol, datetime.date] = {}
256@staticmethod
257def get_last_update_date() -> Dict[Symbol, datetime.date]:
258return QuantpediaGoogleSearch._last_update_date
259def GetSource(self, config, date, isLiveMode):
260 return SubscriptionDataSource("data.quantpedia.com/backtesting_data/google_search/{0}.csv".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)
261def Reader(self, config, line, date, isLiveMode):
262 data = QuantpediaGoogleSearch()
263 data.Symbol = config.Symbol
264 
265 if not line[0].isdigit(): return None
266 split = line.split(';')
267 
268 data.Time = datetime.strptime(split[0], "%d.%m.%Y") + timedelta(days=1)
269 data['value'] = float(split[1])
270 data.Value = float(split[1])
271 if config.Symbol not in QuantpediaGoogleSearch._last_update_date:
272     QuantpediaGoogleSearch._last_update_date[config.Symbol] = datetime(1,1,1).date()
273 if data.Time.date() > QuantpediaGoogleSearch._last_update_date[config.Symbol]:
274     QuantpediaGoogleSearch._last_update_date[config.Symbol] = data.Time.date()
275 return data