Best Articles
Travel Articles
Best Places to Visit in Montreal | Canada
Tech Articles
Understanding 0x0 0x0: A Deep Dive into the Null Pointer Understanding Windows Error Code 0x0 0x0: Reasons, Solutions, and Avoidance Decoding the Secrets of 0x0 0x0
import textwrap import feedparser from bs4 import BeautifulSoup from datetime import datetime import os
DEFAULT_IMAGE_URL = 'https://cdn.hashnode.com/res/hashnode/image/upload/v1704026789016/QS9k8VMZb.jpg'
def clean_html(html, max_description_words=None): if not isinstance(html, str) or not html.strip().startswith('<'): return html
soup = BeautifulSoup(html, 'html.parser') cleaned_text = soup.get_text()
if max_description_words: words = cleaned_text.split()[:max_description_words] cleaned_text = ' '.join(words)
return cleaned_text
def save_to_markdown(data, output_folder='markdown_articles'): os.makedirs(output_folder, exist_ok=True)
for i, item in enumerate(data, start=1): today_date = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ') url_parts = item['Link'].split('/') slug = None
for part in reversed(url_parts): if part and not part.startswith(('www.', 'http', 'https')): slug = part break
if slug is None: slug = f"article-{i}"
wrapped_description = textwrap.fill(item['Description'], width=80)
Include the image URL or the default URL in the Markdown content
cover_photo_url = item.get('CoverPhoto') image_url = cover_photo_url if cover_photo_url else DEFAULT_IMAGE_URL markdown_content = f"""--- title: "{item['Title']}" date: "{today_date}" slug: "{slug}"
image: "{image_url}" # Update this line to use 'image'
{wrapped_description}
filename = os.path.join(output_folder, f'a{i}.md') with open(filename, 'w', encoding='utf-8') as md_file: md_file.write(markdown_content)
def get_cover_photo_url(entry): if hasattr(entry, 'media_thumbnail') and 'url' in entry.media_thumbnail[0]: return entry.media_thumbnail[0]['url'] elif hasattr(entry, 'enclosures') and entry.enclosures: return entry.enclosures[0]['url'] else: return None
def aggregate_rss_feeds(feed_urls, max_description_words=None, num_articles=3): aggregated_items = []
for feed_url in feed_urls: try: feed = feedparser.parse(feed_url) except Exception as e: print(f"Error parsing feed {feed_url}: {e}") continue
for entry in feed.entries[:num_articles]: try: title = entry.title link = entry.link description = entry.summary if hasattr( entry, 'summary') else entry.description
cleaned_description = clean_html( description, max_description_words)
Include cover photo URL or default URL in the item dictionary
item = { 'Title': title, 'Link': link, 'Description': cleaned_description, 'CoverPhoto': get_cover_photo_url(entry) }
aggregated_items.append(item) except Exception as e: print(f"Error processing entry in feed {feed_url}: {e}") continue
return aggregated_items
Example usage
feed_urls = [ 'https://www.techmeme.com/feed.xml', 'https://magazine.sebastianraschka.com/feed', 'https://aiacceleratorinstitute.com/rss/', 'https://ai-techpark.com/category/ai/feed/', 'https://knowtechie.com/category/ai/feed/', 'https://aibusiness.com/rss.xml', 'https://www.artificialintelligence-news.com/feed/rss/', 'https://venturebeat.com/category/ai/feed/', 'https://www.reddit.com/r/aipromptprogramming', 'https://siliconangle.com/category/ai/feed', 'https://aisnakeoil.substack.com/feed', 'https://eng.uber.com/category/articles/ai/feed', 'https://www.anaconda.com/blog/feed', 'https://analyticsindiamag.com/feed/', 'https://stability.ai/blog?format=rss', 'https://feeds.arstechnica.com/arstechnica/index', 'https://www.reddit.com/r/artificial', 'https://theconversation.com/europe/topics/artificial-intelligence-ai-90/articles.atom', 'https://www.theguardian.com/technology/artificialintelligenceai/rss', 'https://spacenews.com/tag/artificial-intelligence/feed/', 'https://futurism.com/categories/ai-artificial-intelligence/feed', 'https://www.wired.com/feed/tag/ai/latest/rss', 'https://www.sciencedaily.com/rss/computers_math/artificial_intelligence.xml', 'https://www.techrepublic.com/rssfeeds/topic/artificial-intelligence/', 'https://medium.com/feed/artificialis', 'https://siliconangle.com/category/big-data/feed', 'https://machinelearningmastery.com/blog/feed', 'https://davidstutz.de/category/blog/feed', 'https://www.together.xyz/blog?format=rss', 'https://neptune.ai/blog/feed', 'https://blog.eleuther.ai/index.xml', 'https://pyimagesearch.com/blog/feed', 'https://feeds.bloomberg.com/technology/news.rss', 'https://feeds.businessinsider.com/custom/all', 'https://www.wired.com/feed/category/business/latest/rss', 'https://every.to/chain-of-thought/feed.xml', 'https://huyenchip.com/feed', 'https://www.reddit.com/r/computervision', 'http://www.computerworld.com/index.rss', 'https://txt.cohere.ai/rss/', 'https://news.crunchbase.com/feed', 'https://arxiv.org/rss/cs.CL', 'https://arxiv.org/rss/cs.CV', 'https://arxiv.org/rss/cs.LG', 'https://dagshub.com/blog/rss/', 'https://www.darkreading.com/rss_simple.asp', 'https://www.databricks.com/feed', 'https://datafloq.com/feed/?post_type=post', 'https://datamachina.substack.com/feed', 'https://www.datanami.com/feed/', 'https://www.reddit.com/r/datascience', 'https://debuggercafe.com/feed/', 'https://deephaven.io/blog/rss.xml', 'https://www.reddit.com/r/deeplearning', 'https://deepmind.com/blog/feed/basic/', 'https://tech.eu/category/deep-tech/feed', 'https://dev.to/feed', 'https://www.eetimes.com/feed', 'https://www.engadget.com/rss.xml', 'https://eugeneyan.com/rss/', 'https://explosion.ai/feed', 'https://www.freethink.com/feed/all'
] max_description_words = 100 num_articles_to_import = 3 aggregated_data = aggregate_rss_feeds( feed_urls, max_description_words, num_articles_to_import)
Specify the output folder here
output_folder = 'markdown_articles' save_to_markdown(aggregated_data, output_folder)