As of at least September 12, 2023, it was possible to use pd.read_html
to easily read Markdown tables used to create minimal reproducible examples.
However, as of this posting, it no longer seems possible to read reproducible data tables directly from Stack Overflow.
Why were changes made to Stack Overflow/Stack Exchange which break accessing markdown tables for reproducible data, or is it an (un)intended side effect of changes made on Cloudflare?
This change, after many years of working, breaks existing answers, which relied on reading the table from the question.
The point of an easy to use minimal reproducible example is to make it simple for non-expert users of SO to copy and execute the code in the question, and in the answer. Running the code in the answer was easy, scraping the body of a question with the API, is not easy for casual users trying to run the given code.
For example, the following table might be posted in a question about how to plot a dataframe.
timestamp | cnt |
---|---|
2015-01-04 | 9234 |
2015-01-05 | 20372 |
2015-01-06 | 20613 |
2015-01-07 | 21064 |
2015-01-08 | 15601 |
2016-12-27 | 10842 |
2016-12-28 | 12428 |
2016-12-29 | 14052 |
2016-12-30 | 11566 |
2016-12-31 | 11424 |
And the following code, which can easily be copied and executed by anyone with the same question, would be posted in the answer.
import pandas as pd
# load the data from the markdown table in the OP, into a dataframe
df = pd.read_html('https://meta.stackoverflow.com/q/426625/7758804')[0]
# plot bars
ax = df.plot(kind='bar', x='timestamp')
Now results in HTTPError: HTTP Error 403: Forbidden
:
---------------------------------------------------------------------------
HTTPError Traceback (most recent call last)
Cell In[15], line 1
----> 1 df = pd.read_html('https://meta.stackoverflow.com/q/426625/7758804')[0]
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\html.py:1245, in read_html(io, match, flavor, header, index_col, skiprows, attrs, parse_dates, thousands, encoding, decimal, converters, na_values, keep_default_na, displayed_only, extract_links, dtype_backend, storage_options)
1229 if isinstance(io, str) and not any(
1230 [
1231 is_file_like(io),
(...)
1235 ]
1236 ):
1237 warnings.warn(
1238 "Passing literal html to 'read_html' is deprecated and "
1239 "will be removed in a future version. To read from a "
(...)
1242 stacklevel=find_stack_level(),
1243 )
-> 1245 return _parse(
1246 flavor=flavor,
1247 io=io,
1248 match=match,
1249 header=header,
1250 index_col=index_col,
1251 skiprows=skiprows,
1252 parse_dates=parse_dates,
1253 thousands=thousands,
1254 attrs=attrs,
1255 encoding=encoding,
1256 decimal=decimal,
1257 converters=converters,
1258 na_values=na_values,
1259 keep_default_na=keep_default_na,
1260 displayed_only=displayed_only,
1261 extract_links=extract_links,
1262 dtype_backend=dtype_backend,
1263 storage_options=storage_options,
1264 )
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\html.py:988, in _parse(flavor, io, match, attrs, encoding, displayed_only, extract_links, storage_options, **kwargs)
977 p = parser(
978 io,
979 compiled_match,
(...)
984 storage_options,
985 )
987 try:
--> 988 tables = p.parse_tables()
989 except ValueError as caught:
990 # if `io` is an io-like object, check if it's seekable
991 # and try to rewind it before trying the next parser
992 if hasattr(io, "seekable") and io.seekable():
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\html.py:248, in _HtmlFrameParser.parse_tables(self)
240 def parse_tables(self):
241 """
242 Parse and return all tables from the DOM.
243
(...)
246 list of parsed (header, body, footer) tuples from tables.
247 """
--> 248 tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
249 return (self._parse_thead_tbody_tfoot(table) for table in tables)
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\html.py:811, in _LxmlFrameParser._build_doc(self)
809 pass
810 else:
--> 811 raise e
812 else:
813 if not hasattr(r, "text_content"):
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\html.py:790, in _LxmlFrameParser._build_doc(self)
788 try:
789 if is_url(self.io):
--> 790 with get_handle(
791 self.io, "r", storage_options=self.storage_options
792 ) as f:
793 r = parse(f.handle, parser=parser)
794 else:
795 # try to parse the input in the simplest way
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\common.py:718, in get_handle(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)
715 codecs.lookup_error(errors)
717 # open URLs
--> 718 ioargs = _get_filepath_or_buffer(
719 path_or_buf,
720 encoding=encoding,
721 compression=compression,
722 mode=mode,
723 storage_options=storage_options,
724 )
726 handle = ioargs.filepath_or_buffer
727 handles: list[BaseBuffer]
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\common.py:372, in _get_filepath_or_buffer(filepath_or_buffer, encoding, compression, mode, storage_options)
370 # assuming storage_options is to be interpreted as headers
371 req_info = urllib.request.Request(filepath_or_buffer, headers=storage_options)
--> 372 with urlopen(req_info) as req:
373 content_encoding = req.headers.get("Content-Encoding", None)
374 if content_encoding == "gzip":
375 # Override compression based on Content-Encoding header
File ~\anaconda3\envs\py11\Lib\site-packages\pandas\io\common.py:274, in urlopen(*args, **kwargs)
268 """
269 Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of
270 the stdlib.
271 """
272 import urllib.request
--> 274 return urllib.request.urlopen(*args, **kwargs)
File ~\anaconda3\envs\py11\Lib\urllib\request.py:216, in urlopen(url, data, timeout, cafile, capath, cadefault, context)
214 else:
215 opener = _opener
--> 216 return opener.open(url, data, timeout)
File ~\anaconda3\envs\py11\Lib\urllib\request.py:525, in OpenerDirector.open(self, fullurl, data, timeout)
523 for processor in self.process_response.get(protocol, []):
524 meth = getattr(processor, meth_name)
--> 525 response = meth(req, response)
527 return response
File ~\anaconda3\envs\py11\Lib\urllib\request.py:634, in HTTPErrorProcessor.http_response(self, request, response)
631 # According to RFC 2616, "2xx" code indicates that the client's
632 # request was successfully received, understood, and accepted.
633 if not (200 <= code < 300):
--> 634 response = self.parent.error(
635 'http', request, response, code, msg, hdrs)
637 return response
File ~\anaconda3\envs\py11\Lib\urllib\request.py:563, in OpenerDirector.error(self, proto, *args)
561 if http_err:
562 args = (dict, 'default', 'http_error_default') + orig_args
--> 563 return self._call_chain(*args)
File ~\anaconda3\envs\py11\Lib\urllib\request.py:496, in OpenerDirector._call_chain(self, chain, kind, meth_name, *args)
494 for handler in handlers:
495 func = getattr(handler, meth_name)
--> 496 result = func(*args)
497 if result is not None:
498 return result
File ~\anaconda3\envs\py11\Lib\urllib\request.py:643, in HTTPDefaultErrorHandler.http_error_default(self, req, fp, code, msg, hdrs)
642 def http_error_default(self, req, fp, code, msg, hdrs):
--> 643 raise HTTPError(req.full_url, code, msg, hdrs, fp)
HTTPError: HTTP Error 403: Forbidden
HTTPError: HTTP Error 403: Forbidden
is extremely explicit. This is Stack Overflow saying that it does not want to provide the requested information to your program. That is not a bug unless you can find something in the site documentation that supports a claim that it's supposed to be provided in this specific way.my $qxString = "wget -O - \"$URL\" "; my $content = qx/$qxString/;
(the URL must be quoted, at least if it contains "?" (GET parameters)). And capture standard output (or through a temporary file).