update version to 1.0.8 - empty headline solved

CHANGELOG.rst

@@ -1,6 +1,10 @@
 Change Log
 ===========
 
+1.0.8
+-------
+- Solve empty headlines
+
 1.0.7
 -------
 - Add markets data to ycnbc

meta.yaml

@@ -1,13 +1,13 @@
 {% set name = "ycnbc" %}
-{% set version = "1.0.7" %}
+{% set version = "1.0.8" %}
 
 package:
   name: "{{ name|lower }}"
   version: "{{ version }}"
 
 source:
   url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz"
-  sha256: "670D54B9688CC95B4776CF887CFA2E723D642A7A174B86F33B0D629868A1A39C"
+  sha256: "B66FD76CB7706458976358192FAEDA527CEEA264E81CF6EE648C8328B056B165"
 
 build:
   noarch: python
@@ -18,12 +18,14 @@ requirements:
   host:
     - requests >=2.23.0
     - lxml >=4.5.1
+    - cssselect >=1.2.0
     - pip
     - python
 
   run:
     - requests >=2.23.0
     - lxml >=4.5.1
+    - cssselect >=1.2.0
     - python
 
 test:

requirements.txt

@@ -1,2 +1,3 @@
 requests>=2.23.0
-lxml>=4.5.1
+lxml>=4.5.1
+cssselect>=1.2.0

setup.py

@@ -51,7 +51,7 @@
     platforms=['any'],
     keywords='scrape news, cnbc library, cnbc python, cnbc api',
     packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']),
-    install_requires=['requests>=2.23.0', 'lxml>=4.5.1'],
+    install_requires=['requests>=2.23.0', 'lxml>=4.5.1', 'cssselect>=1.2.0'],
     entry_points={
         'console_scripts': [
             # 'sample=sample:main',

test.py

@@ -55,6 +55,7 @@ def test_cnbc_news(self):
             with self.subTest(method=method_name):
                 method = getattr(self.news, method_name)
                 response = method()
+                print(json.dumps(response))
                 self.assertNotIn("error", response, f"{method_name} returned an error")
 
     def test_cnbc_markets(self):

ycnbc/news/news_utils.py

@@ -52,7 +52,7 @@ def trending(self):
                 source.append(link)
 
             return {
-                'title': title,
+                'headline': title,
                 'link': source
             }
         except Exception as e:
@@ -116,30 +116,19 @@ def by_category(self, category):
             if "error" in tree:
                 return tree
 
-            source, title, posttime = [], [], []
-
-            news = tree.xpath("//div[contains(@class, 'Card-titleContainer')]")
-            if not news:
-                return {"error": "No news items found"}
-
-            posttime_news = tree.xpath("//span[contains(@class, 'Card-time')]")
-            if not posttime_news:
-                return {"error": "No post time found"}
-
-            for i in posttime_news:
-                text = i.xpath(".//text()")
-                posttime.append(' '.join(text))
-
-            for i in news:
-                text = i.xpath("..//div/text()")
-                link = list(i.iterlinks())[0][2] if list(i.iterlinks()) else None
-                source.append(link)
-                title.append(' '.join(text))
+            headlines, post_times, links = [], [], []
+            news_elements = tree.cssselect('.Card-titleContainer a.Card-title')
+            post_time_elements = tree.cssselect('span.Card-time')
+            for news, post_time in zip(news_elements, post_time_elements):
+                headlines.append(news.text.strip())
+                links.append(news.get('href'))
+                post_times.append(post_time.text.strip())
 
             return {
-                'headline': title,
-                'time': posttime,
-                'link': source
+                'headline': headlines,
+                'time': post_times,
+                'link': links,
             }
+
         except Exception as e:
             return {"error": str(e)}

ycnbc/version.py

@@ -1 +1 @@
-version = "1.0.7"
+version = "1.0.8"