Skip to content

Commit

Permalink
indices functions created & indices.csv generated
Browse files Browse the repository at this point in the history 
- indices functions have been created in order to retrieve index data
- `indices.csv` file has been generated via `investpy.indices.retrieve_indices()`
- codecov target has been modified in order to avoid build failing since indices are not covered in the tests/ yet
  • Loading branch information
Álvaro Bartolomé del Canto committed Sep 23, 2019
1 parent 831eded commit d78a5ef
Show file tree
Hide file tree
Showing 4 changed files with 711 additions and 4 deletions.
2 changes: 1 addition & 1 deletion .codecov.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ coverage:
project:
default:
# basic
target: 80%
target: 70%
threshold: 10%
base: auto
# advanced
Expand Down
6 changes: 3 additions & 3 deletions investpy/funds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,12 +400,12 @@ def funds_as_dict(country=None, columns=None, as_json=False):
"""
This function retrieves all the available funds on Investing.com and returns them as a :obj:`dict` containing the
`asset_class`, `id`, `issuer`, `name`, `symbol` and `tag`. All the available funds can be found at:
https://es.investing.com/etfs/
https://es.investing.com/funds/
Args:
country (:obj:`str`, optional): name of the country to retrieve all its available funds from.
columns (:obj:`list` of :obj:`str`, optional): description
a `list` containing the column names from which the data is going to be retrieved.
a :obj:`list` containing the column names from which the data is going to be retrieved.
as_json (:obj:`bool`, optional): description
value to determine the format of the output data (:obj:`dict` or :obj:`json`).
Expand All @@ -428,7 +428,7 @@ def funds_as_dict(country=None, columns=None, as_json=False):
Raises:
ValueError: raised when the introduced arguments are not correct.
IOError: if the funds file from `investpy` is missing or errored.
IOError: raised if the funds file from `investpy` is missing or errored.
"""

if country is not None and not isinstance(country, str):
Expand Down
322 changes: 322 additions & 0 deletions investpy/indices.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,174 @@

# Copyright 2018-2019 Alvaro Bartolome
# See LICENSE for details.
import json

import pandas as pd
import pkg_resources

import requests
import unidecode
from lxml.html import fromstring

from investpy import user_agent as ua


def retrieve_indices(test_mode=False):
"""
This function retrieves all the available `equities` indexed on Investing.com, so to
retrieve data from them which will be used later for inner functions for data retrieval.
All the equities available can be found at: https://es.investing.com/equities/. Additionally,
when equities are retrieved all the meta-information is both returned as a :obj:`pandas.DataFrame`
and stored on a CSV file on a package folder containing all the available resources.
Note that maybe some of the information contained in the resulting :obj:`pandas.DataFrame` is useless as it is
just used for inner function purposes.
Args:
test_mode (:obj:`bool`):
variable to avoid time waste on travis-ci since it just needs to test the basics in order to improve code
coverage.
Returns:
:obj:`pandas.DataFrame` - indices:
The resulting :obj:`pandas.DataFrame` contains all the indices meta-information if found, if not, an
empty :obj:`pandas.DataFrame` will be returned and no CSV file will be stored.
In the case that the retrieval process of indices was successfully completed, the resulting
:obj:`pandas.DataFrame` will look like::
country | name | full_name | tag | id | currency
--------|------|-----------|-----|----|----------
xxxxxxx | xxxx | xxxxxxxxx | xxx | xx | xxxxxxxx
Raises:
ValueError: raised if any of the introduced arguments is not valid.
FileNotFoundError: raised if `index_countries.csv` file does not exists or is empty.
ConnectionError: raised if GET requests did not return 200 status code.
IndexError: raised if indices information was unavailable or not found.
"""

if not isinstance(test_mode, bool):
raise ValueError('ERR#0041: test_mode can just be either True or False')

results = list()

for country in index_countries_as_list():
head = {
"User-Agent": ua.get_random(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}

url = "https://www.investing.com/indices/" + country.replace(' ', '-') + "-indices"

req = requests.get(url, headers=head)

if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")

root_ = fromstring(req.text)
path_ = root_.xpath(".//table[@id='cr1']/tbody/tr")

if path_:
for elements_ in path_:
id_ = elements_.get('id').replace('pair_', '')

for element_ in elements_.xpath('.//a'):
tag_ = element_.get('href')

if str(tag_).__contains__('/indices/'):
tag_ = tag_.replace('/indices/', '')
full_name_ = element_.get('title').replace(' (CFD)', '').strip()
name = element_.text.strip()

info = retrieve_index_info(tag_)

data = {
'country': country,
'name': name,
'full_name': full_name_,
'tag': tag_,
'id': id_,
'currency': info['currency'],
}

results.append(data)

if test_mode is True:
break

if test_mode is True:
break

resource_package = __name__
resource_path = '/'.join(('resources', 'indices', 'indices.csv'))
file = pkg_resources.resource_filename(resource_package, resource_path)

df = pd.DataFrame(results)

if test_mode is False:
df.to_csv(file, index=False)

return df


def retrieve_index_info(tag):
"""
This function retrieves additional information from an index as listed in Investing.com. Every index data is
retrieved and stored in a CSV in order to get all the possible information from it.
Args:
tag (:obj:`str`): is the identifying tag of the specified index.
Returns:
:obj:`dict` - index_data:
The resulting :obj:`dict` contains the retrieved data if found, if not, the corresponding
fields are filled with `None` values.
In case the information was successfully retrieved, the :obj:`dict` will look like::
{
'currency': currency
}
Raises:
ConnectionError: raised if GET requests does not return 200 status code.
IndexError: raised if fund information was unavailable or not found.
"""

url = "https://www.investing.com/indices/" + tag

head = {
"User-Agent": ua.get_random(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}

req = requests.get(url, headers=head)

if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")

result = {
'currency': None
}

root_ = fromstring(req.text)

path_ = root_.xpath(".//div[contains(@class, 'bottom')]"
"/span[@class='bold']")

for element_ in path_:
if element_.text_content():
result['currency'] = element_.text_content()

return result


def retrieve_index_countries(test_mode=False):
"""
This function retrieves all the country names indexed in Investing.com with available equities to retrieve data
Expand Down Expand Up @@ -108,3 +266,167 @@ def index_countries_as_list():
raise IOError("ERR#0036: equity countries list not found or unable to retrieve.")
else:
return countries['country'].tolist()


def indices_as_df(country=None):
"""
This function retrieves all the available `indices` from Investing.com and returns them as a :obj:`pandas.DataFrame`,
which contains not just the index names, but all the fields contained on the indices file.
All the available indices can be found at: https://es.investing.com/indices/
Args:
country (:obj:`str`, optional): name of the country to retrieve all its available indices from.
Returns:
:obj:`pandas.DataFrame` - indices_df:
The resulting :obj:`pandas.DataFrame` contains all the indices basic information retrieved from Investing.com,
some of which is not useful for the user, but for the inner package functions, such as the `tag` field,
for example.
In case the information was successfully retrieved, the :obj:`pandas.DataFrame` will look like::
country | name | full_name | tag | id | currency
--------|------|-----------|-----|----|----------
xxxxxxx | xxxx | xxxxxxxxx | xxx | xx | xxxxxxxx
Just like `investpy.indices.retrieve_indices()`, the output of this function is a :obj:`pandas.DataFrame`,
but instead of generating the CSV file, this function just reads it and loads it into a
:obj:`pandas.DataFrame` object.
Raises:
IOError: raised if the indices file from `investpy` is missing or errored.
"""

if country is not None and not isinstance(country, str):
raise ValueError("ERR#0025: specified country value not valid.")

resource_package = __name__
resource_path = '/'.join(('resources', 'indices', 'indices.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
indices = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
indices = retrieve_indices()

if indices is None:
raise IOError("ERR#0037: indices not found or unable to retrieve.")

if country is None:
indices.reset_index(drop=True, inplace=True)
return indices
elif unidecode.unidecode(country.lower()) in index_countries_as_list():
indices = indices[indices['country'] == unidecode.unidecode(country.lower())]
indices.reset_index(drop=True, inplace=True)
return indices


def indices_as_list(country=None):
"""
This function retrieves all the available indices and returns a list of each one of them.
All the available indices can be found at: https://es.investing.com/indices/
Args:
country (:obj:`str`, optional): name of the country to retrieve all its available indices from.
Returns:
:obj:`list` - indices_list:
The resulting :obj:`list` contains the retrieved data, which corresponds to the index names of
every index listed on Investing.com.
In case the information was successfully retrieved from the CSV file, the :obj:`list` will look like::
indices = [...]
Raises:
ValueError: raised when the introduced arguments are not correct.
IOError: raised if the indices file from `investpy` is missing or errored.
"""

if country is not None and not isinstance(country, str):
raise ValueError("ERR#0025: specified country value not valid.")

resource_package = __name__
resource_path = '/'.join(('resources', 'indices', 'indices.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
indices = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
indices = retrieve_indices()

if indices is None:
raise IOError("ERR#0037: indices not found or unable to retrieve.")

if country is None:
return indices['name'].tolist()
elif unidecode.unidecode(country.lower()) in index_countries_as_list():
return indices[indices['country'] == unidecode.unidecode(country.lower())]['name'].tolist()


def indices_as_dict(country=None, columns=None, as_json=False):
"""
This function retrieves all the available indices on Investing.com and returns them as a :obj:`dict` containing the
`country`, `name`, `full_name`, `symbol`, `tag` and `currency`. All the available indices can be found at:
https://es.investing.com/indices/
Args:
country (:obj:`str`, optional): name of the country to retrieve all its available indices from.
columns (:obj:`list` of :obj:`str`, optional): description
a :obj:`list` containing the column names from which the data is going to be retrieved.
as_json (:obj:`bool`, optional): description
value to determine the format of the output data (:obj:`dict` or :obj:`json`).
Returns:
:obj:`dict` or :obj:`json` - indices_dict:
The resulting :obj:`dict` contains the retrieved data if found, if not, the corresponding
fields are filled with `None` values.
In case the information was successfully retrieved, the :obj:`dict` will look like::
{
'country': country,
'name': name,
'full_name': full_name,
'symbol': symbol,
'tag': tag
'currency': currency
}
Raises:
ValueError: raised when the introduced arguments are not correct.
IOError: raised if the indices file from `investpy` is missing or errored.
"""

if country is not None and not isinstance(country, str):
raise ValueError("ERR#0025: specified country value not valid.")

if not isinstance(as_json, bool):
raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.")

resource_package = __name__
resource_path = '/'.join(('resources', 'indices', 'indices.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
indices = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
indices = retrieve_indices()

if indices is None:
raise IOError("ERR#0037: indices not found or unable to retrieve.")

if columns is None:
columns = indices.columns.tolist()
else:
if not isinstance(columns, list):
raise ValueError("ERR#0020: specified columns argument is not a list, it can just be list type.")

if not all(column in indices.columns.tolist() for column in columns):
raise ValueError("ERR#0023: specified columns does not exist, available columns are "
"<country, name, full_name, symbol, tag, currency>")

if country is None:
if as_json:
return json.dumps(indices[columns].to_dict(orient='records'))
else:
return indices[columns].to_dict(orient='records')
elif country in index_countries_as_list():
if as_json:
return json.dumps(indices[indices['country'] == unidecode.unidecode(country.lower())][columns].to_dict(orient='records'))
else:
return indices[indices['country'] == unidecode.unidecode(country.lower())][columns].to_dict(orient='records')
Loading

0 comments on commit d78a5ef

Please sign in to comment.