468 lines
17 KiB
Python
468 lines
17 KiB
Python
# python3 -m pip install --user virtualenv
|
|
# python3 -m venv venv
|
|
# source venv/bin/activate
|
|
# pip install langgraph langchain langchain-community langchain-ollama
|
|
# source venv/bin/activate
|
|
|
|
|
|
from datetime import timedelta
|
|
import sys
|
|
from typing import TypedDict, List
|
|
from urllib import response
|
|
#from asyncio import tools
|
|
from langgraph.graph import START, StateGraph, END
|
|
from langchain_core.prompts import ChatPromptTemplate
|
|
from langchain_core.messages import HumanMessage, SystemMessage
|
|
from langchain_core.tools import tool
|
|
from langgraph.prebuilt import ToolNode
|
|
import requests, json, re, string, os
|
|
from html.parser import HTMLParser
|
|
from langchain_ollama import ChatOllama
|
|
import operator
|
|
from typing_extensions import TypedDict, Annotated
|
|
|
|
class HTMLCrawler(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.skip_content = False # Skip content inside <script> and <style>
|
|
self.crawled_text = ''
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
# Skip content inside script and style tags
|
|
if tag in ['script', 'style']:
|
|
self.skip_content = True
|
|
|
|
def handle_endtag(self, tag):
|
|
# Resume collecting after closing script and style tags
|
|
if tag in ['script', 'style']:
|
|
self.skip_content = False
|
|
|
|
def handle_data(self, data):
|
|
# Only process data if not inside script/style tags
|
|
if self.skip_content:
|
|
return
|
|
|
|
# Clean up the data: strip whitespace and filter empty lines
|
|
text = data.strip()
|
|
|
|
# Skip if empty or too short
|
|
if not text or len(text) < 2:
|
|
return
|
|
|
|
# Skip common junk: JSON-like patterns, URLs in certain contexts, etc.
|
|
if text.startswith('{') or text.startswith('['):
|
|
return
|
|
|
|
# Print meaningful text
|
|
#print("data :", text)
|
|
self.crawled_text += text + '\n'
|
|
|
|
|
|
|
|
# Configuration
|
|
LLM_BASE_URL = "http://192.168.50.215:11434"
|
|
MODEL_NAME = "qwen3:8b"
|
|
#MODEL_NAME = "gpt-oss:20b"
|
|
LINKDING_API_URL = "https://linkding.hal.se/api/bookmarks/"
|
|
LINKDING_API_TOKEN = "fa54dee2ccbcad80a0c6259bdbbed896581e1423"
|
|
|
|
|
|
llm = ChatOllama(
|
|
model=MODEL_NAME,
|
|
base_url=LLM_BASE_URL,
|
|
num_ctx=4096, # Increase context window to handle longer content
|
|
temperature=0.3 # Lower temperature for more focused summaries
|
|
)
|
|
|
|
@tool
|
|
def todays_date() -> str:
|
|
"""Use this tool whenever you need to know today's date in ISO 8601 format."""
|
|
from datetime import datetime
|
|
return datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
@tool
|
|
def calculate_date(dat: str, days: int) -> str:
|
|
"""Use this tool whenever you need to calculate the date in ISO 8601 format given a date and the number of days to deduce from that date."""
|
|
from datetime import datetime
|
|
date1 = datetime.strptime(dat, "%Y-%m-%dT%H:%M:%SZ")
|
|
date2 = date1 - timedelta(days=days)
|
|
return date2.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
@tool
|
|
def crawl_homepage(url:str) -> str:
|
|
"""Use this tool whenever you need to read the content of a home page on the internet.
|
|
|
|
Args:
|
|
url: home page url to crawl
|
|
|
|
Returns:
|
|
Formatted text from homepage
|
|
"""
|
|
htmlParser = HTMLCrawler()
|
|
try:
|
|
raw_html = requests.get(url.strip()).text
|
|
htmlParser.feed(raw_html)
|
|
except Exception as e:
|
|
return f"Error occurred while parsing HTML: {e}"
|
|
|
|
return htmlParser.crawled_text
|
|
|
|
@tool
|
|
def fetch_bookmarks(date_added: str = "2026-03-01T00:00:00Z") -> str:
|
|
"""Use this tool whenever you need to fetch bookmarks.
|
|
|
|
Args:
|
|
date_added: The date from which to fetch bookmarks, in ISO 8601 format (e.g., "2026-04-01T00:00:00Z").
|
|
|
|
Returns:
|
|
Formatted bookmark with titles, descriptions, URL, date_added, tags and ID.
|
|
"""
|
|
|
|
_url = f"{LINKDING_API_URL}?added_since={date_added}"
|
|
|
|
_headers = {
|
|
"Authorization": "Token " + LINKDING_API_TOKEN
|
|
}
|
|
|
|
response = requests.get(_url, headers=_headers)
|
|
data = response.json()
|
|
|
|
if not data:
|
|
return f"No bookmarks found"
|
|
|
|
formatted_results = [f"Bookmarks:"]
|
|
for i, bookmark in enumerate(data['results'], 1):
|
|
title = bookmark.get('title', 'No title')
|
|
id = bookmark.get('id', 'No ID')
|
|
url = bookmark.get('url', 'No URL')
|
|
date_added = bookmark.get('date_added', 'No date added')
|
|
description = bookmark.get('description', 'No description')
|
|
tag_names = bookmark.get('tag_names', 'No tags')
|
|
|
|
text = f" {title}\n {description}\n {url}\n {date_added}\n {tag_names}\n {id}\n"
|
|
|
|
formatted_results.append(text)
|
|
|
|
return "\n\n".join(formatted_results)
|
|
|
|
@tool
|
|
def add_tag_to_bookmark(bookmark_id: int, tag: str) -> str:
|
|
"""Use this tool whenever you need to add a tag to a bookmark.
|
|
|
|
Args:
|
|
bookmark_id: The ID of the bookmark to which the tag should be added.
|
|
tag: The tag to add to the bookmark.
|
|
|
|
Returns:
|
|
A message indicating whether the tag was successfully added or if an error occurred.
|
|
"""
|
|
_url = f"{LINKDING_API_URL}{bookmark_id}/"
|
|
_headers = {
|
|
"Authorization": "Token " + LINKDING_API_TOKEN,
|
|
"Content-Type": "application/json"
|
|
}
|
|
payload = {
|
|
"tag_names": [tag]
|
|
}
|
|
|
|
try:
|
|
response = requests.patch(_url, headers=_headers, data=json.dumps(payload))
|
|
response.raise_for_status() # Raise an exception for HTTP errors
|
|
return f"Tag '{tag}' successfully added to bookmark with ID {bookmark_id}."
|
|
except requests.exceptions.RequestException as e:
|
|
return f"Error occurred while adding tag: {e}"
|
|
|
|
@tool
|
|
def write_to_file(filename: str, content: str) -> str:
|
|
"""Use this tool whenever you need to write content to a file.
|
|
|
|
Args:
|
|
filename: The name of the file to which the content should be written.
|
|
content: The content to write to the file.
|
|
|
|
Returns:
|
|
A message indicating whether the content was successfully written or if an error occurred.
|
|
"""
|
|
try:
|
|
with open(filename, 'a') as f:
|
|
f.write(content)
|
|
return f"Content successfully written to {filename}."
|
|
except Exception as e:
|
|
return f"Error occurred while writing to file: {e}"
|
|
|
|
# ----- Helper function to fetch raw bookmark data -----
|
|
def fetch_raw_bookmarks(date_added: str) -> List[dict]:
|
|
"""Internal helper to fetch raw bookmark JSON data"""
|
|
_url = f"{LINKDING_API_URL}?added_since={date_added}"
|
|
_headers = {
|
|
"Authorization": "Token " + LINKDING_API_TOKEN
|
|
}
|
|
try:
|
|
response = requests.get(_url, headers=_headers)
|
|
data = response.json()
|
|
return data.get('results', [])
|
|
except Exception as e:
|
|
print(f"Error fetching bookmarks: {e}")
|
|
return []
|
|
|
|
# ----- Shared State -----
|
|
class AgentState(TypedDict):
|
|
messages: Annotated[list, operator.add]
|
|
bookmarks: list # Raw bookmark data
|
|
current_index: int # Index of bookmark being processed
|
|
target_date: str # Date used for fetching bookmarks. in ISO 8601 format (e.g., "2026-04-01T00:00:00Z").
|
|
path_to_file: str # Path to the file where summaries will be written
|
|
|
|
# ----- Agent Nodes -----
|
|
def initialization_node(state: AgentState):
|
|
"""Phase 1: Fetch relevant bookmarks"""
|
|
messages = state['messages']
|
|
|
|
# Fetch bookmarks using the tool (for logging)
|
|
bookmarks_result = fetch_bookmarks.invoke({'date_added': state['target_date']})
|
|
messages.append(HumanMessage(f"[INIT] {bookmarks_result}"))
|
|
|
|
# Also fetch raw bookmark data for processing
|
|
raw_bookmarks = fetch_raw_bookmarks(state['target_date'])
|
|
messages.append(HumanMessage(f"[INIT] Found {len(raw_bookmarks)} bookmarks to process"))
|
|
|
|
if not raw_bookmarks:
|
|
messages.append(HumanMessage("[INIT] No bookmarks found. Stopping."))
|
|
|
|
return {
|
|
'messages': messages,
|
|
'bookmarks': raw_bookmarks,
|
|
'current_index': 0
|
|
}
|
|
|
|
def process_bookmarks_node(state: AgentState):
|
|
"""Phase 2: Process each bookmark by crawling, writing, and tagging"""
|
|
messages = state['messages']
|
|
bookmarks = state['bookmarks']
|
|
|
|
if not bookmarks:
|
|
messages.append(HumanMessage("[PROCESS] No bookmarks to process."))
|
|
return {
|
|
'messages': messages,
|
|
'current_index': 0
|
|
}
|
|
|
|
# Step 1: Crawl all bookmarks
|
|
crawled_data = []
|
|
for i, bookmark in enumerate(bookmarks, 1):
|
|
messages.append(HumanMessage(f"\n[PROCESS] Crawling bookmark {i}/{len(bookmarks)}: {bookmark['title']}"))
|
|
|
|
try:
|
|
content = crawl_homepage.invoke({'url': bookmark['url']})
|
|
messages.append(HumanMessage(f" ✓ Crawled: {len(content)} chars"))
|
|
crawled_data.append({
|
|
'bookmark': bookmark,
|
|
'content': content
|
|
})
|
|
except Exception as e:
|
|
messages.append(HumanMessage(f" ✗ Error crawling: {e}"))
|
|
crawled_data.append({
|
|
'bookmark': bookmark,
|
|
'content': ''
|
|
})
|
|
|
|
# Step 2: Batch-analyze all content in a single LLM call for summaries
|
|
messages.append(HumanMessage(f"\n[PROCESS] Generating summaries for all {len(crawled_data)} bookmarks..."))
|
|
summaries_by_bookmark = {}
|
|
try:
|
|
# Prepare content for LLM summary analysis
|
|
summary_text = ""
|
|
for i, item in enumerate(crawled_data, 1):
|
|
title = item['bookmark']['title']
|
|
content = item['content'][:1000] # Reduce to 1000 chars to keep LLM focused
|
|
summary_text += f"{i}. Title: {title}\nContent: {content}\n\n"
|
|
|
|
# Use LLM to summarize all bookmarks at once - with a very simple, explicit prompt
|
|
summary_prompt = SystemMessage("""For each numbered item (1-10), write a 5-10 sentence summary. Output format: "N. summary text"
|
|
|
|
Example:
|
|
1. Travel guides website offering Lonely Planet destination collections and expert travel advice.
|
|
2. Cloudflare security page blocking access to the website.""")
|
|
|
|
content_message = HumanMessage(f"Summarize these items:\n\n{summary_text}")
|
|
|
|
response = llm.invoke([summary_prompt, content_message])
|
|
summary_response = response.content.strip()
|
|
messages.append(HumanMessage(f" ✓ Generated summaries for all bookmarks"))
|
|
|
|
# Parse the simple format: "1. text", "2. text", etc.
|
|
lines = summary_response.split('\n')
|
|
for line in lines:
|
|
line = line.strip()
|
|
if not line or len(line) < 3:
|
|
continue
|
|
|
|
# Look for pattern "N. text" or "N) text" or "N- text"
|
|
match = re.match(r'^(\d+)[.\)\-]\s+(.*)', line)
|
|
if match:
|
|
try:
|
|
num = int(match.group(1))
|
|
summary_content = match.group(2).strip()
|
|
if 1 <= num <= len(crawled_data) and len(summary_content) > 10:
|
|
summaries_by_bookmark[num] = summary_content
|
|
except (ValueError, IndexError):
|
|
pass
|
|
|
|
# Log how many summaries were extracted
|
|
messages.append(HumanMessage(f" ✓ Extracted {len(summaries_by_bookmark)} summaries from LLM response"))
|
|
except Exception as e:
|
|
messages.append(HumanMessage(f" ✗ Error generating summaries: {e}"))
|
|
summaries_by_bookmark = {}
|
|
|
|
# Step 3: Batch-analyze all content in a single LLM call for tags
|
|
messages.append(HumanMessage(f"\n[PROCESS] Analyzing all {len(crawled_data)} bookmarks for tags..."))
|
|
try:
|
|
# Prepare content for LLM analysis
|
|
analysis_text = ""
|
|
for i, item in enumerate(crawled_data, 1):
|
|
title = item['bookmark']['title']
|
|
content_snippet = item['content'][:500] # Use first 500 chars per bookmark
|
|
analysis_text += f"{i}. Title: {title}\nContent snippet: {content_snippet}\n---\n"
|
|
|
|
# Use LLM to analyze all bookmarks at once
|
|
analysis_prompt = SystemMessage("""You are a bookmark tagging assistant. For each bookmark, suggest 1-3 relevant tags.
|
|
Output format: For each numbered bookmark, respond with:
|
|
N: tag1, tag2, tag3
|
|
|
|
Tags should be lowercase and hyphenated if multi-word. Be concise.""")
|
|
|
|
content_message = HumanMessage(f"Analyze these bookmarks and suggest relevant tags:\n\n{analysis_text}")
|
|
|
|
response = llm.invoke([analysis_prompt, content_message])
|
|
tags_response = response.content.strip()
|
|
messages.append(HumanMessage(f" ✓ Generated tags for all bookmarks"))
|
|
|
|
# Parse the response to extract tags for each bookmark
|
|
tags_by_bookmark = {}
|
|
for line in tags_response.split('\n'):
|
|
if line.strip() and ':' in line:
|
|
try:
|
|
num_str = line.split(':')[0].strip()
|
|
num = int(num_str)
|
|
if 1 <= num <= len(crawled_data):
|
|
tags_text = line.split(':', 1)[1].strip()
|
|
tags = [tag.strip() for tag in tags_text.split(',')]
|
|
tags_by_bookmark[num] = tags
|
|
except (ValueError, IndexError):
|
|
pass
|
|
except Exception as e:
|
|
messages.append(HumanMessage(f" ✗ Error analyzing bookmarks: {e}"))
|
|
tags_by_bookmark = {}
|
|
|
|
# Step 4: Write to file and add tags for each bookmark
|
|
for i, item in enumerate(crawled_data, 1):
|
|
bookmark = item['bookmark']
|
|
suggested_tags = tags_by_bookmark.get(i, [])
|
|
suggested_summary = summaries_by_bookmark.get(i, "No summary available")
|
|
|
|
messages.append(HumanMessage(f"\n[WRITE] Bookmark {i}/{len(crawled_data)}: {bookmark['title']}"))
|
|
|
|
# Write to file
|
|
try:
|
|
output_content = f"## {bookmark['title']}\n- URL: {bookmark['url']}\n- Description: {bookmark['description']}\n- Summary: {suggested_summary}\n- Tags: {', '.join(suggested_tags)}\n- Date: {bookmark['date_added']}\n\n"
|
|
|
|
filename = os.path.expanduser(state['path_to_file'] + '/bookmark_summaries_' + state['target_date'][:10] + '.md')
|
|
write_to_file.invoke({'filename': filename, 'content': output_content})
|
|
messages.append(HumanMessage(f" ✓ Wrote to file"))
|
|
except Exception as e:
|
|
messages.append(HumanMessage(f" ✗ Error writing file: {e}"))
|
|
continue
|
|
|
|
# Add tags to bookmark
|
|
try:
|
|
if suggested_tags:
|
|
for tag in suggested_tags:
|
|
add_tag_to_bookmark.invoke({
|
|
'bookmark_id': bookmark['id'],
|
|
'tag': tag
|
|
})
|
|
messages.append(HumanMessage(f" ✓ Added tags: {', '.join(suggested_tags)}"))
|
|
else:
|
|
messages.append(HumanMessage(f" ⊘ No tags generated"))
|
|
except Exception as e:
|
|
messages.append(HumanMessage(f" ✗ Error tagging: {e}"))
|
|
|
|
messages.append(HumanMessage(f"\n[PROCESS] ✓ ALL {len(bookmarks)} BOOKMARKS PROCESSED"))
|
|
|
|
return {
|
|
'messages': messages,
|
|
'current_index': len(bookmarks)
|
|
}
|
|
|
|
def create_agent():
|
|
|
|
builder = StateGraph(AgentState)
|
|
|
|
builder.add_node("initialize", initialization_node)
|
|
builder.add_node("process", process_bookmarks_node)
|
|
builder.set_entry_point("initialize")
|
|
|
|
builder.add_edge("initialize", "process")
|
|
builder.add_edge("process", END)
|
|
|
|
graph = builder.compile()
|
|
|
|
return graph
|
|
|
|
def main():
|
|
if len(sys.argv) == 3:
|
|
days = int(sys.argv[1])
|
|
file_path = sys.argv[2]
|
|
else:
|
|
days = 7 # Default to 7 days if not provided
|
|
file_path = '~'
|
|
|
|
agent = create_agent()
|
|
|
|
# Get today's date
|
|
today = todays_date.invoke({})
|
|
# Calculate date 7 days ago
|
|
target_date = calculate_date.invoke({'dat': today, 'days': days})
|
|
human_prompt = HumanMessage(f"Process all bookmarks from the last {days} days: fetch them, summarize their content, write summaries to a file, and add relevant tags.")
|
|
|
|
result = agent.invoke({'messages': [human_prompt], 'bookmarks': [], 'current_index': 0, 'target_date': target_date, 'path_to_file': file_path})
|
|
|
|
print("\n" + "="*80)
|
|
print("EXECUTION COMPLETE")
|
|
print("="*80)
|
|
for msg in result['messages']:
|
|
if isinstance(msg, HumanMessage):
|
|
print(msg.content)
|
|
print("="*80)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
|
|
"""
|
|
result = fetch_bookmarks.invoke({})
|
|
print(result)
|
|
|
|
|
|
result = fetch_bookmarks.invoke({'date_added': "2026-04-09T20:26:31Z"})
|
|
print(result)
|
|
|
|
|
|
result = crawl_homepage.invoke({'url': "http://example.com"})
|
|
print(result)
|
|
|
|
result = todays_date.invoke({})
|
|
print(result)
|
|
|
|
#result = add_tag_to_bookmark.invoke({'bookmark_id': 123, 'tag': 'newtag'})
|
|
#print(result)
|
|
|
|
#result = add_tag_to_bookmark.invoke({'bookmark_id': 4, 'tag': 'another_tag'})
|
|
#print(result)
|
|
|
|
result = calculate_date.invoke({'dat': "2026-03-01T00:00:00Z", 'days': 3})
|
|
print(result)
|
|
|
|
result = write_to_file.invoke({'filename': 'test.txt', 'content': 'This is a test.'})
|
|
print(result)
|
|
""" |