Master 4.0 (#2210)

* stage academic conversation

* stage document conversation

* fix buggy gradio version

* file dynamic load

* merge more academic plugins

* accelerate nltk

* feat: 为predict函数添加文件和URL读取功能
- 添加URL检测和网页内容提取功能,支持自动提取网页文本
- 添加文件路径识别和文件内容读取功能,支持private_upload路径格式
- 集成WebTextExtractor处理网页内容提取
- 集成TextContentLoader处理本地文件读取
- 支持文件路径与问题组合的智能处理

* back

* block unstable

---------

Co-authored-by: XiaoBoAI <liuboyin2019@ia.ac.cn>
这个提交包含在:
binary-husky
2025-08-23 15:59:22 +08:00
提交者 GitHub
父节点 65a4cf59c2
当前提交 8042750d41
共有 79 个文件被更改,包括 20850 次插入57 次删除

查看文件

@@ -0,0 +1,386 @@
from abc import ABC, abstractmethod
from typing import List, Dict, Any
from ..query_analyzer import SearchCriteria
from ..sources.github_source import GitHubSource
import asyncio
import re
from datetime import datetime
class BaseHandler(ABC):
"""处理器基类"""
def __init__(self, github: GitHubSource, llm_kwargs: Dict = None):
self.github = github
self.llm_kwargs = llm_kwargs or {}
self.ranked_repos = [] # 存储排序后的仓库列表
def _get_search_params(self, plugin_kwargs: Dict) -> Dict:
"""获取搜索参数"""
return {
'max_repos': plugin_kwargs.get('max_repos', 150), # 最大仓库数量,从30改为150
'max_details': plugin_kwargs.get('max_details', 80), # 最多展示详情的仓库数量,新增参数
'search_multiplier': plugin_kwargs.get('search_multiplier', 3), # 检索倍数
'min_stars': plugin_kwargs.get('min_stars', 0), # 最少星标数
}
@abstractmethod
async def handle(
self,
criteria: SearchCriteria,
chatbot: List[List[str]],
history: List[List[str]],
system_prompt: str,
llm_kwargs: Dict[str, Any],
plugin_kwargs: Dict[str, Any],
) -> str:
"""处理查询"""
pass
async def _search_repositories(self, query: str, language: str = None, min_stars: int = 0,
sort: str = "stars", per_page: int = 30) -> List[Dict]:
"""搜索仓库"""
try:
# 构建查询字符串
if min_stars > 0 and "stars:>" not in query:
query += f" stars:>{min_stars}"
if language and "language:" not in query:
query += f" language:{language}"
# 执行搜索
result = await self.github.search_repositories(
query=query,
sort=sort,
per_page=per_page
)
if result and "items" in result:
return result["items"]
return []
except Exception as e:
print(f"仓库搜索出错: {str(e)}")
return []
async def _search_bilingual_repositories(self, english_query: str, chinese_query: str, language: str = None, min_stars: int = 0,
sort: str = "stars", per_page: int = 30) -> List[Dict]:
"""同时搜索中英文仓库并合并结果"""
try:
# 搜索英文仓库
english_results = await self._search_repositories(
query=english_query,
language=language,
min_stars=min_stars,
sort=sort,
per_page=per_page
)
# 搜索中文仓库
chinese_results = await self._search_repositories(
query=chinese_query,
language=language,
min_stars=min_stars,
sort=sort,
per_page=per_page
)
# 合并结果,去除重复项
merged_results = []
seen_repos = set()
# 优先添加英文结果
for repo in english_results:
repo_id = repo.get('id')
if repo_id and repo_id not in seen_repos:
seen_repos.add(repo_id)
merged_results.append(repo)
# 添加中文结果(排除重复)
for repo in chinese_results:
repo_id = repo.get('id')
if repo_id and repo_id not in seen_repos:
seen_repos.add(repo_id)
merged_results.append(repo)
# 按星标数重新排序
merged_results.sort(key=lambda x: x.get('stargazers_count', 0), reverse=True)
return merged_results[:per_page] # 返回合并后的前per_page个结果
except Exception as e:
print(f"双语仓库搜索出错: {str(e)}")
return []
async def _search_code(self, query: str, language: str = None, per_page: int = 30) -> List[Dict]:
"""搜索代码"""
try:
# 构建查询字符串
if language and "language:" not in query:
query += f" language:{language}"
# 执行搜索
result = await self.github.search_code(
query=query,
per_page=per_page
)
if result and "items" in result:
return result["items"]
return []
except Exception as e:
print(f"代码搜索出错: {str(e)}")
return []
async def _search_bilingual_code(self, english_query: str, chinese_query: str, language: str = None, per_page: int = 30) -> List[Dict]:
"""同时搜索中英文代码并合并结果"""
try:
# 搜索英文代码
english_results = await self._search_code(
query=english_query,
language=language,
per_page=per_page
)
# 搜索中文代码
chinese_results = await self._search_code(
query=chinese_query,
language=language,
per_page=per_page
)
# 合并结果,去除重复项
merged_results = []
seen_files = set()
# 优先添加英文结果
for item in english_results:
# 使用文件URL作为唯一标识
file_url = item.get('html_url', '')
if file_url and file_url not in seen_files:
seen_files.add(file_url)
merged_results.append(item)
# 添加中文结果(排除重复)
for item in chinese_results:
file_url = item.get('html_url', '')
if file_url and file_url not in seen_files:
seen_files.add(file_url)
merged_results.append(item)
# 对结果进行排序,优先显示匹配度高的结果
# 由于无法直接获取匹配度,这里使用仓库的星标数作为替代指标
merged_results.sort(key=lambda x: x.get('repository', {}).get('stargazers_count', 0), reverse=True)
return merged_results[:per_page] # 返回合并后的前per_page个结果
except Exception as e:
print(f"双语代码搜索出错: {str(e)}")
return []
async def _search_users(self, query: str, per_page: int = 30) -> List[Dict]:
"""搜索用户"""
try:
result = await self.github.search_users(
query=query,
per_page=per_page
)
if result and "items" in result:
return result["items"]
return []
except Exception as e:
print(f"用户搜索出错: {str(e)}")
return []
async def _search_bilingual_users(self, english_query: str, chinese_query: str, per_page: int = 30) -> List[Dict]:
"""同时搜索中英文用户并合并结果"""
try:
# 搜索英文用户
english_results = await self._search_users(
query=english_query,
per_page=per_page
)
# 搜索中文用户
chinese_results = await self._search_users(
query=chinese_query,
per_page=per_page
)
# 合并结果,去除重复项
merged_results = []
seen_users = set()
# 优先添加英文结果
for user in english_results:
user_id = user.get('id')
if user_id and user_id not in seen_users:
seen_users.add(user_id)
merged_results.append(user)
# 添加中文结果(排除重复)
for user in chinese_results:
user_id = user.get('id')
if user_id and user_id not in seen_users:
seen_users.add(user_id)
merged_results.append(user)
# 按关注者数量进行排序
merged_results.sort(key=lambda x: x.get('followers', 0), reverse=True)
return merged_results[:per_page] # 返回合并后的前per_page个结果
except Exception as e:
print(f"双语用户搜索出错: {str(e)}")
return []
async def _search_topics(self, query: str, per_page: int = 30) -> List[Dict]:
"""搜索主题"""
try:
result = await self.github.search_topics(
query=query,
per_page=per_page
)
if result and "items" in result:
return result["items"]
return []
except Exception as e:
print(f"主题搜索出错: {str(e)}")
return []
async def _search_bilingual_topics(self, english_query: str, chinese_query: str, per_page: int = 30) -> List[Dict]:
"""同时搜索中英文主题并合并结果"""
try:
# 搜索英文主题
english_results = await self._search_topics(
query=english_query,
per_page=per_page
)
# 搜索中文主题
chinese_results = await self._search_topics(
query=chinese_query,
per_page=per_page
)
# 合并结果,去除重复项
merged_results = []
seen_topics = set()
# 优先添加英文结果
for topic in english_results:
topic_name = topic.get('name')
if topic_name and topic_name not in seen_topics:
seen_topics.add(topic_name)
merged_results.append(topic)
# 添加中文结果(排除重复)
for topic in chinese_results:
topic_name = topic.get('name')
if topic_name and topic_name not in seen_topics:
seen_topics.add(topic_name)
merged_results.append(topic)
# 可以按流行度进行排序(如果有)
if merged_results and 'featured' in merged_results[0]:
merged_results.sort(key=lambda x: x.get('featured', False), reverse=True)
return merged_results[:per_page] # 返回合并后的前per_page个结果
except Exception as e:
print(f"双语主题搜索出错: {str(e)}")
return []
async def _get_repo_details(self, repos: List[Dict]) -> List[Dict]:
"""获取仓库详细信息"""
enhanced_repos = []
for repo in repos:
try:
# 获取README信息
owner = repo.get('owner', {}).get('login') if repo.get('owner') is not None else None
repo_name = repo.get('name')
if owner and repo_name:
readme = await self.github.get_repo_readme(owner, repo_name)
if readme and "decoded_content" in readme:
# 提取README的前1000个字符作为摘要
repo['readme_excerpt'] = readme["decoded_content"][:1000] + "..."
# 获取语言使用情况
languages = await self.github.get_repository_languages(owner, repo_name)
if languages:
repo['languages_detail'] = languages
# 获取最新发布版本
releases = await self.github.get_repo_releases(owner, repo_name, per_page=1)
if releases and len(releases) > 0:
repo['latest_release'] = releases[0]
# 获取主题标签
topics = await self.github.get_repo_topics(owner, repo_name)
if topics and "names" in topics:
repo['topics'] = topics["names"]
enhanced_repos.append(repo)
except Exception as e:
print(f"获取仓库 {repo.get('full_name')} 详情时出错: {str(e)}")
enhanced_repos.append(repo) # 添加原始仓库信息
return enhanced_repos
def _format_repos(self, repos: List[Dict]) -> str:
"""格式化仓库列表"""
formatted = []
for i, repo in enumerate(repos, 1):
# 构建仓库URL
repo_url = repo.get('html_url', '')
# 构建完整的引用
reference = (
f"{i}. **{repo.get('full_name', '')}**\n"
f" - 描述: {repo.get('description', 'N/A')}\n"
f" - 语言: {repo.get('language', 'N/A')}\n"
f" - 星标: {repo.get('stargazers_count', 0)}\n"
f" - Fork数: {repo.get('forks_count', 0)}\n"
f" - 更新时间: {repo.get('updated_at', 'N/A')[:10]}\n"
f" - 创建时间: {repo.get('created_at', 'N/A')[:10]}\n"
f" - URL: <a href='{repo_url}' target='_blank'>{repo_url}</a>\n"
)
# 添加主题标签(如果有)
if repo.get('topics'):
topics_str = ", ".join(repo.get('topics'))
reference += f" - 主题标签: {topics_str}\n"
# 添加最新发布版本(如果有)
if repo.get('latest_release'):
release = repo.get('latest_release')
reference += f" - 最新版本: {release.get('tag_name', 'N/A')} ({release.get('published_at', 'N/A')[:10]})\n"
# 添加README摘要(如果有)
if repo.get('readme_excerpt'):
# 截断README,只取前300个字符
readme_short = repo.get('readme_excerpt')[:300].replace('\n', ' ')
reference += f" - README摘要: {readme_short}...\n"
formatted.append(reference)
return "\n".join(formatted)
def _generate_apology_prompt(self, criteria: SearchCriteria) -> str:
"""生成道歉提示"""
return f"""很抱歉,我们未能找到与"{criteria.main_topic}"相关的GitHub项目。
可能的原因:
1. 搜索词过于具体或冷门
2. 星标数要求过高
3. 编程语言限制过于严格
建议解决方案:
1. 尝试使用更通用的关键词
2. 降低最低星标数要求
3. 移除或更改编程语言限制
请根据以上建议调整后重试。"""
def _get_current_time(self) -> str:
"""获取当前时间信息"""
now = datetime.now()
return now.strftime("%Y年%m月%d")

查看文件

@@ -0,0 +1,156 @@
from typing import List, Dict, Any
from .base_handler import BaseHandler
from ..query_analyzer import SearchCriteria
import asyncio
class CodeSearchHandler(BaseHandler):
"""代码搜索处理器"""
def __init__(self, github, llm_kwargs=None):
super().__init__(github, llm_kwargs)
async def handle(
self,
criteria: SearchCriteria,
chatbot: List[List[str]],
history: List[List[str]],
system_prompt: str,
llm_kwargs: Dict[str, Any],
plugin_kwargs: Dict[str, Any],
) -> str:
"""处理代码搜索请求,返回最终的prompt"""
search_params = self._get_search_params(plugin_kwargs)
# 搜索代码
code_results = await self._search_bilingual_code(
english_query=criteria.github_params["query"],
chinese_query=criteria.github_params["chinese_query"],
language=criteria.language,
per_page=search_params['max_repos']
)
if not code_results:
return self._generate_apology_prompt(criteria)
# 获取代码文件内容
enhanced_code_results = await self._get_code_details(code_results[:search_params['max_details']])
self.ranked_repos = [item["repository"] for item in enhanced_code_results if "repository" in item]
if not enhanced_code_results:
return self._generate_apology_prompt(criteria)
# 构建最终的prompt
current_time = self._get_current_time()
final_prompt = f"""当前时间: {current_time}
基于用户对{criteria.main_topic}的查询,我找到了以下代码示例。
代码搜索结果:
{self._format_code_results(enhanced_code_results)}
请提供:
1. 对于搜索的"{criteria.main_topic}"主题的综合解释:
- 概念和原理介绍
- 常见实现方法和技术
- 最佳实践和注意事项
2. 对每个代码示例:
- 解释代码的主要功能和实现方式
- 分析代码质量、可读性和效率
- 指出代码中的亮点和潜在改进空间
- 说明代码的适用场景
3. 代码实现比较:
- 不同实现方法的优缺点
- 性能和可维护性分析
- 适用不同场景的实现建议
4. 学习建议:
- 理解和使用这些代码需要的背景知识
- 如何扩展或改进所展示的代码
- 进一步学习相关技术的资源
重要提示:
- 深入解释代码的核心逻辑和实现思路
- 提供专业、技术性的分析
- 优先关注代码的实现质量和技术价值
- 当代码实现有问题时,指出并提供改进建议
- 对于复杂代码,分解解释其组成部分
- 根据用户查询的具体问题提供针对性答案
- 所有链接请使用<a href='链接地址' target='_blank'>链接文本</a>格式,确保链接在新窗口打开
使用markdown格式提供清晰的分节回复。
"""
return final_prompt
async def _get_code_details(self, code_results: List[Dict]) -> List[Dict]:
"""获取代码详情"""
enhanced_results = []
for item in code_results:
try:
repo = item.get('repository', {})
file_path = item.get('path', '')
repo_name = repo.get('full_name', '')
if repo_name and file_path:
owner, repo_name = repo_name.split('/')
# 获取文件内容
file_content = await self.github.get_file_content(owner, repo_name, file_path)
if file_content and "decoded_content" in file_content:
item['code_content'] = file_content["decoded_content"]
# 获取仓库基本信息
repo_details = await self.github.get_repo(owner, repo_name)
if repo_details:
item['repository'] = repo_details
enhanced_results.append(item)
except Exception as e:
print(f"获取代码详情时出错: {str(e)}")
enhanced_results.append(item) # 添加原始信息
return enhanced_results
def _format_code_results(self, code_results: List[Dict]) -> str:
"""格式化代码搜索结果"""
formatted = []
for i, item in enumerate(code_results, 1):
# 构建仓库信息
repo = item.get('repository', {})
repo_name = repo.get('full_name', 'N/A')
repo_url = repo.get('html_url', '')
stars = repo.get('stargazers_count', 0)
language = repo.get('language', 'N/A')
# 构建文件信息
file_path = item.get('path', 'N/A')
file_url = item.get('html_url', '')
# 构建代码内容
code_content = item.get('code_content', '')
if code_content:
# 只显示前30行代码
code_lines = code_content.split("\n")
if len(code_lines) > 30:
displayed_code = "\n".join(code_lines[:30]) + "\n... (代码太长已截断) ..."
else:
displayed_code = code_content
else:
displayed_code = "(代码内容获取失败)"
reference = (
f"### {i}. {file_path} (在 {repo_name} 中)\n\n"
f"- **仓库**: <a href='{repo_url}' target='_blank'>{repo_name}</a> (⭐ {stars}, 语言: {language})\n"
f"- **文件路径**: <a href='{file_url}' target='_blank'>{file_path}</a>\n\n"
f"```{language.lower()}\n{displayed_code}\n```\n\n"
)
formatted.append(reference)
return "\n".join(formatted)

查看文件

@@ -0,0 +1,192 @@
from typing import List, Dict, Any
from .base_handler import BaseHandler
from ..query_analyzer import SearchCriteria
import asyncio
class RepositoryHandler(BaseHandler):
"""仓库搜索处理器"""
def __init__(self, github, llm_kwargs=None):
super().__init__(github, llm_kwargs)
async def handle(
self,
criteria: SearchCriteria,
chatbot: List[List[str]],
history: List[List[str]],
system_prompt: str,
llm_kwargs: Dict[str, Any],
plugin_kwargs: Dict[str, Any],
) -> str:
"""处理仓库搜索请求,返回最终的prompt"""
search_params = self._get_search_params(plugin_kwargs)
# 如果是特定仓库查询
if criteria.repo_id:
try:
owner, repo = criteria.repo_id.split('/')
repo_details = await self.github.get_repo(owner, repo)
if repo_details:
# 获取推荐的相似仓库
similar_repos = await self.github.get_repo_recommendations(criteria.repo_id, limit=5)
# 添加详细信息
all_repos = [repo_details] + similar_repos
enhanced_repos = await self._get_repo_details(all_repos)
self.ranked_repos = enhanced_repos
# 构建最终的prompt
current_time = self._get_current_time()
final_prompt = self._build_repo_detail_prompt(enhanced_repos[0], enhanced_repos[1:], current_time)
return final_prompt
else:
return self._generate_apology_prompt(criteria)
except Exception as e:
print(f"处理特定仓库时出错: {str(e)}")
return self._generate_apology_prompt(criteria)
# 一般仓库搜索
repos = await self._search_bilingual_repositories(
english_query=criteria.github_params["query"],
chinese_query=criteria.github_params["chinese_query"],
language=criteria.language,
min_stars=criteria.min_stars,
per_page=search_params['max_repos']
)
if not repos:
return self._generate_apology_prompt(criteria)
# 获取仓库详情
enhanced_repos = await self._get_repo_details(repos[:search_params['max_details']]) # 使用max_details参数
self.ranked_repos = enhanced_repos
if not enhanced_repos:
return self._generate_apology_prompt(criteria)
# 构建最终的prompt
current_time = self._get_current_time()
final_prompt = f"""当前时间: {current_time}
基于用户对{criteria.main_topic}的兴趣,以下是相关的GitHub仓库。
可供推荐的GitHub仓库:
{self._format_repos(enhanced_repos)}
请提供:
1. 按功能、用途或成熟度对仓库进行分组
2. 对每个仓库:
- 简要描述其主要功能和用途
- 分析其技术特点和优势
- 说明其适用场景和使用难度
- 指出其与同类产品相比的独特优势
- 解释其星标数量和活跃度代表的意义
3. 使用建议:
- 新手最适合入门的仓库
- 生产环境中最稳定可靠的选择
- 最新技术栈或创新方案的代表
- 学习特定技术的最佳资源
4. 相关资源:
- 学习这些项目需要的前置知识
- 项目间的关联和技术栈兼容性
- 可能的使用组合方案
重要提示:
- 重点解释为什么每个仓库值得关注
- 突出项目间的关联性和差异性
- 考虑用户不同水平的需求(初学者vs专业人士)
- 在介绍项目时,使用<a href='链接' target='_blank'>文本</a>格式,确保链接在新窗口打开
- 根据仓库的活跃度、更新频率、维护状态提供使用建议
- 仅基于提供的信息,不要做无根据的猜测
- 在信息缺失或不明确时,坦诚说明
使用markdown格式提供清晰的分节回复。
"""
return final_prompt
def _build_repo_detail_prompt(self, main_repo: Dict, similar_repos: List[Dict], current_time: str) -> str:
"""构建仓库详情prompt"""
# 提取README摘要
readme_content = "未提供"
if main_repo.get('readme_excerpt'):
readme_content = main_repo.get('readme_excerpt')
# 构建语言分布
languages = main_repo.get('languages_detail', {})
lang_distribution = []
if languages:
total = sum(languages.values())
for lang, bytes_val in languages.items():
percentage = (bytes_val / total) * 100
lang_distribution.append(f"{lang}: {percentage:.1f}%")
lang_str = "未知"
if lang_distribution:
lang_str = ", ".join(lang_distribution)
# 构建最终prompt
prompt = f"""当前时间: {current_time}
## 主要仓库信息
### {main_repo.get('full_name')}
- **描述**: {main_repo.get('description', '未提供')}
- **星标数**: {main_repo.get('stargazers_count', 0)}
- **Fork数**: {main_repo.get('forks_count', 0)}
- **Watch数**: {main_repo.get('watchers_count', 0)}
- **Issues数**: {main_repo.get('open_issues_count', 0)}
- **语言分布**: {lang_str}
- **许可证**: {main_repo.get('license', {}).get('name', '未指定') if main_repo.get('license') is not None else '未指定'}
- **创建时间**: {main_repo.get('created_at', '')[:10]}
- **最近更新**: {main_repo.get('updated_at', '')[:10]}
- **主题标签**: {', '.join(main_repo.get('topics', ['']))}
- **GitHub链接**: <a href='{main_repo.get('html_url')}' target='_blank'>链接</a>
### README摘要:
{readme_content}
## 类似仓库:
{self._format_repos(similar_repos)}
请提供以下内容:
1. **项目概述**
- 详细解释{main_repo.get('name', '')}项目的主要功能和用途
- 分析其技术特点、架构和实现原理
- 讨论其在所属领域的地位和影响力
- 评估项目成熟度和稳定性
2. **优势与特点**
- 与同类项目相比的独特优势
- 显著的技术创新或设计模式
- 值得学习或借鉴的代码实践
3. **使用场景**
- 最适合的应用场景
- 潜在的使用限制和注意事项
- 入门门槛和学习曲线评估
- 产品级应用的可行性分析
4. **资源与生态**
- 相关学习资源推荐
- 配套工具和库的建议
- 社区支持和活跃度评估
5. **类似项目对比**
- 与列出的类似项目的详细对比
- 不同场景下的最佳选择建议
- 潜在的互补使用方案
提示:所有链接请使用<a href='链接地址' target='_blank'>链接文本</a>格式,确保链接在新窗口打开。
请以专业、客观的技术分析角度回答,使用markdown格式提供结构化信息。
"""
return prompt

查看文件

@@ -0,0 +1,217 @@
from typing import List, Dict, Any
from .base_handler import BaseHandler
from ..query_analyzer import SearchCriteria
import asyncio
class TopicHandler(BaseHandler):
"""主题搜索处理器"""
def __init__(self, github, llm_kwargs=None):
super().__init__(github, llm_kwargs)
async def handle(
self,
criteria: SearchCriteria,
chatbot: List[List[str]],
history: List[List[str]],
system_prompt: str,
llm_kwargs: Dict[str, Any],
plugin_kwargs: Dict[str, Any],
) -> str:
"""处理主题搜索请求,返回最终的prompt"""
search_params = self._get_search_params(plugin_kwargs)
# 搜索主题
topics = await self._search_bilingual_topics(
english_query=criteria.github_params["query"],
chinese_query=criteria.github_params["chinese_query"],
per_page=search_params['max_repos']
)
if not topics:
# 尝试用主题搜索仓库
search_query = criteria.github_params["query"]
chinese_search_query = criteria.github_params["chinese_query"]
if "topic:" not in search_query:
search_query += " topic:" + criteria.main_topic.replace(" ", "-")
if "topic:" not in chinese_search_query:
chinese_search_query += " topic:" + criteria.main_topic.replace(" ", "-")
repos = await self._search_bilingual_repositories(
english_query=search_query,
chinese_query=chinese_search_query,
language=criteria.language,
min_stars=criteria.min_stars,
per_page=search_params['max_repos']
)
if not repos:
return self._generate_apology_prompt(criteria)
# 获取仓库详情
enhanced_repos = await self._get_repo_details(repos[:10])
self.ranked_repos = enhanced_repos
if not enhanced_repos:
return self._generate_apology_prompt(criteria)
# 构建基于主题的仓库列表prompt
current_time = self._get_current_time()
final_prompt = f"""当前时间: {current_time}
基于用户对主题"{criteria.main_topic}"的查询,我找到了以下相关GitHub仓库。
主题相关仓库:
{self._format_repos(enhanced_repos)}
请提供:
1. 主题综述:
- "{criteria.main_topic}"主题的概述和重要性
- 该主题在技术领域中的应用和发展趋势
- 主题相关的主要技术栈和知识体系
2. 仓库分析:
- 按功能、技术栈或应用场景对仓库进行分类
- 每个仓库在该主题领域的定位和贡献
- 不同仓库间的技术路线对比
3. 学习路径建议:
- 初学者入门该主题的推荐仓库和学习顺序
- 进阶学习的关键仓库和技术要点
- 实际应用中的最佳实践选择
4. 技术生态分析:
- 该主题下的主流工具和库
- 社区活跃度和维护状况
- 与其他相关技术的集成方案
重要提示:
- 主题"{criteria.main_topic}"是用户查询的核心,请围绕此主题展开分析
- 注重仓库质量评估和使用建议
- 提供基于事实的客观技术分析
- 在介绍仓库时使用<a href='链接地址' target='_blank'>链接文本</a>格式,确保链接在新窗口打开
- 考虑不同技术水平用户的需求
使用markdown格式提供清晰的分节回复。
"""
return final_prompt
# 如果找到了主题,则获取主题下的热门仓库
topic_repos = []
for topic in topics[:5]: # 增加到5个主题
topic_name = topic.get('name', '')
if topic_name:
# 搜索该主题下的仓库
repos = await self._search_repositories(
query=f"topic:{topic_name}",
language=criteria.language,
min_stars=criteria.min_stars,
per_page=20 # 每个主题最多20个仓库
)
if repos:
for repo in repos:
repo['topic_source'] = topic_name
topic_repos.append(repo)
if not topic_repos:
return self._generate_apology_prompt(criteria)
# 获取前N个仓库的详情
enhanced_repos = await self._get_repo_details(topic_repos[:search_params['max_details']])
self.ranked_repos = enhanced_repos
if not enhanced_repos:
return self._generate_apology_prompt(criteria)
# 构建最终的prompt
current_time = self._get_current_time()
final_prompt = f"""当前时间: {current_time}
基于用户对"{criteria.main_topic}"主题的查询,我找到了以下相关GitHub主题和仓库。
主题相关仓库:
{self._format_topic_repos(enhanced_repos)}
请提供:
1. 主题概述:
- 对"{criteria.main_topic}"相关主题的介绍和技术背景
- 这些主题在软件开发中的重要性和应用范围
- 主题间的关联性和技术演进路径
2. 精选仓库分析:
- 每个主题下最具代表性的仓库详解
- 仓库的技术亮点和创新点
- 使用场景和技术成熟度评估
3. 技术趋势分析:
- 基于主题和仓库活跃度的技术发展趋势
- 新兴解决方案和传统方案的对比
- 未来可能的技术方向预测
4. 实践建议:
- 不同应用场景下的最佳仓库选择
- 学习路径和资源推荐
- 实际项目中的应用策略
重要提示:
- 将分析重点放在主题的技术内涵和价值上
- 突出主题间的关联性和技术演进脉络
- 提供基于数据(星标数、更新频率等)的客观分析
- 考虑不同技术背景用户的需求
- 所有链接请使用<a href='链接地址' target='_blank'>链接文本</a>格式,确保链接在新窗口打开
使用markdown格式提供清晰的分节回复。
"""
return final_prompt
def _format_topic_repos(self, repos: List[Dict]) -> str:
"""按主题格式化仓库列表"""
# 按主题分组
topics_dict = {}
for repo in repos:
topic = repo.get('topic_source', '其他')
if topic not in topics_dict:
topics_dict[topic] = []
topics_dict[topic].append(repo)
# 格式化输出
formatted = []
for topic, topic_repos in topics_dict.items():
formatted.append(f"## 主题: {topic}\n")
for i, repo in enumerate(topic_repos, 1):
# 构建仓库URL
repo_url = repo.get('html_url', '')
# 构建引用
reference = (
f"{i}. **{repo.get('full_name', '')}**\n"
f" - 描述: {repo.get('description', 'N/A')}\n"
f" - 语言: {repo.get('language', 'N/A')}\n"
f" - 星标: {repo.get('stargazers_count', 0)}\n"
f" - Fork数: {repo.get('forks_count', 0)}\n"
f" - 更新时间: {repo.get('updated_at', 'N/A')[:10]}\n"
f" - URL: <a href='{repo_url}' target='_blank'>{repo_url}</a>\n"
)
# 添加主题标签(如果有)
if repo.get('topics'):
topics_str = ", ".join(repo.get('topics'))
reference += f" - 主题标签: {topics_str}\n"
# 添加README摘要(如果有)
if repo.get('readme_excerpt'):
# 截断README,只取前200个字符
readme_short = repo.get('readme_excerpt')[:200].replace('\n', ' ')
reference += f" - README摘要: {readme_short}...\n"
formatted.append(reference)
formatted.append("\n") # 主题之间添加空行
return "\n".join(formatted)

查看文件

@@ -0,0 +1,164 @@
from typing import List, Dict, Any
from .base_handler import BaseHandler
from ..query_analyzer import SearchCriteria
import asyncio
class UserSearchHandler(BaseHandler):
"""用户搜索处理器"""
def __init__(self, github, llm_kwargs=None):
super().__init__(github, llm_kwargs)
async def handle(
self,
criteria: SearchCriteria,
chatbot: List[List[str]],
history: List[List[str]],
system_prompt: str,
llm_kwargs: Dict[str, Any],
plugin_kwargs: Dict[str, Any],
) -> str:
"""处理用户搜索请求,返回最终的prompt"""
search_params = self._get_search_params(plugin_kwargs)
# 搜索用户
users = await self._search_bilingual_users(
english_query=criteria.github_params["query"],
chinese_query=criteria.github_params["chinese_query"],
per_page=search_params['max_repos']
)
if not users:
return self._generate_apology_prompt(criteria)
# 获取用户详情和仓库
enhanced_users = await self._get_user_details(users[:search_params['max_details']])
self.ranked_repos = [] # 添加用户top仓库进行展示
for user in enhanced_users:
if user.get('top_repos'):
self.ranked_repos.extend(user.get('top_repos'))
if not enhanced_users:
return self._generate_apology_prompt(criteria)
# 构建最终的prompt
current_time = self._get_current_time()
final_prompt = f"""当前时间: {current_time}
基于用户对{criteria.main_topic}的查询,我找到了以下GitHub用户。
GitHub用户搜索结果:
{self._format_users(enhanced_users)}
请提供:
1. 用户综合分析:
- 各开发者的专业领域和技术专长
- 他们在GitHub开源社区的影响力
- 技术实力和项目质量评估
2. 对每位开发者:
- 其主要贡献领域和技术栈
- 代表性项目及其价值
- 编程风格和技术特点
- 在相关领域的影响力
3. 项目推荐:
- 针对用户查询的最有价值项目
- 值得学习和借鉴的代码实践
- 不同用户项目的相互补充关系
4. 如何学习和使用:
- 如何从这些开发者项目中学习
- 最适合入门学习的项目
- 进阶学习的路径建议
重要提示:
- 关注开发者的技术专长和核心贡献
- 分析其开源项目的技术价值
- 根据用户的原始查询提供相关建议
- 避免过度赞美或主观评价
- 基于事实数据(项目数、星标数等)进行客观分析
- 所有链接请使用<a href='链接地址' target='_blank'>链接文本</a>格式,确保链接在新窗口打开
使用markdown格式提供清晰的分节回复。
"""
return final_prompt
async def _get_user_details(self, users: List[Dict]) -> List[Dict]:
"""获取用户详情和仓库"""
enhanced_users = []
for user in users:
try:
username = user.get('login')
if username:
# 获取用户详情
user_details = await self.github.get_user(username)
if user_details:
user.update(user_details)
# 获取用户仓库
repos = await self.github.get_user_repos(
username,
sort="stars",
per_page=10 # 增加到10个仓库
)
if repos:
user['top_repos'] = repos
enhanced_users.append(user)
except Exception as e:
print(f"获取用户 {user.get('login')} 详情时出错: {str(e)}")
enhanced_users.append(user) # 添加原始信息
return enhanced_users
def _format_users(self, users: List[Dict]) -> str:
"""格式化用户列表"""
formatted = []
for i, user in enumerate(users, 1):
# 构建用户信息
username = user.get('login', 'N/A')
name = user.get('name', username)
profile_url = user.get('html_url', '')
bio = user.get('bio', '无简介')
followers = user.get('followers', 0)
public_repos = user.get('public_repos', 0)
company = user.get('company', '未指定')
location = user.get('location', '未指定')
blog = user.get('blog', '')
user_info = (
f"### {i}. {name} (@{username})\n\n"
f"- **简介**: {bio}\n"
f"- **关注者**: {followers} | **公开仓库**: {public_repos}\n"
f"- **公司**: {company} | **地点**: {location}\n"
f"- **个人网站**: {blog}\n"
f"- **GitHub**: <a href='{profile_url}' target='_blank'>{username}</a>\n\n"
)
# 添加用户的热门仓库
top_repos = user.get('top_repos', [])
if top_repos:
user_info += "**热门仓库**:\n\n"
for repo in top_repos:
repo_name = repo.get('name', '')
repo_url = repo.get('html_url', '')
repo_desc = repo.get('description', '无描述')
repo_stars = repo.get('stargazers_count', 0)
repo_language = repo.get('language', '未指定')
user_info += (
f"- <a href='{repo_url}' target='_blank'>{repo_name}</a> - ⭐ {repo_stars}, {repo_language}\n"
f" {repo_desc}\n\n"
)
formatted.append(user_info)
return "\n".join(formatted)

查看文件

@@ -0,0 +1,356 @@
from typing import Dict, List
from dataclasses import dataclass
import re
@dataclass
class SearchCriteria:
"""搜索条件"""
query_type: str # 查询类型: repo/code/user/topic
main_topic: str # 主题
sub_topics: List[str] # 子主题列表
language: str # 编程语言
min_stars: int # 最少星标数
github_params: Dict # GitHub搜索参数
original_query: str = "" # 原始查询字符串
repo_id: str = "" # 特定仓库ID或名称
class QueryAnalyzer:
"""查询分析器"""
# 响应索引常量
BASIC_QUERY_INDEX = 0
GITHUB_QUERY_INDEX = 1
def __init__(self):
self.valid_types = {
"repo": ["repository", "project", "library", "framework", "tool"],
"code": ["code", "snippet", "implementation", "function", "class", "algorithm"],
"user": ["user", "developer", "organization", "contributor", "maintainer"],
"topic": ["topic", "category", "tag", "field", "area", "domain"]
}
def analyze_query(self, query: str, chatbot: List, llm_kwargs: Dict):
"""分析查询意图"""
from crazy_functions.crazy_utils import \
request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency as request_gpt
# 1. 基本查询分析
type_prompt = f"""请分析这个与GitHub相关的查询,并严格按照以下XML格式回答
查询: {query}
说明:
1. 你的回答必须使用下面显示的XML标签,不要有任何标签外的文本
2. 从以下选项中选择查询类型: repo/code/user/topic
- repo: 用于查找仓库、项目、框架或库
- code: 用于查找代码片段、函数实现或算法
- user: 用于查找用户、开发者或组织
- topic: 用于查找主题、类别或领域相关项目
3. 识别主题和子主题
4. 识别首选编程语言(如果有)
5. 确定最低星标数(如果适用)
必需格式:
<query_type>此处回答</query_type>
<main_topic>此处回答</main_topic>
<sub_topics>子主题1, 子主题2, ...</sub_topics>
<language>此处回答</language>
<min_stars>此处回答</min_stars>
示例回答:
1. 仓库查询:
查询: "查找有至少1000颗星的Python web框架"
<query_type>repo</query_type>
<main_topic>web框架</main_topic>
<sub_topics>后端开发, HTTP服务器, ORM</sub_topics>
<language>Python</language>
<min_stars>1000</min_stars>
2. 代码查询:
查询: "如何用JavaScript实现防抖函数"
<query_type>code</query_type>
<main_topic>防抖函数</main_topic>
<sub_topics>事件处理, 性能优化, 函数节流</sub_topics>
<language>JavaScript</language>
<min_stars>0</min_stars>"""
# 2. 生成英文搜索条件
github_prompt = f"""Optimize the following GitHub search query:
Query: {query}
Task: Convert the natural language query into an optimized GitHub search query.
Please use English, regardless of the language of the input query.
Available search fields and filters:
1. Basic fields:
- in:name - Search in repository names
- in:description - Search in repository descriptions
- in:readme - Search in README files
- in:topic - Search in topics
- language:X - Filter by programming language
- user:X - Repositories from a specific user
- org:X - Repositories from a specific organization
2. Code search fields:
- extension:X - Filter by file extension
- path:X - Filter by path
- filename:X - Filter by filename
3. Metric filters:
- stars:>X - Has more than X stars
- forks:>X - Has more than X forks
- size:>X - Size greater than X KB
- created:>YYYY-MM-DD - Created after a specific date
- pushed:>YYYY-MM-DD - Updated after a specific date
4. Other filters:
- is:public/private - Public or private repositories
- archived:true/false - Archived or not archived
- license:X - Specific license
- topic:X - Contains specific topic tag
Examples:
1. Query: "Find Python machine learning libraries with at least 1000 stars"
<query>machine learning in:description language:python stars:>1000</query>
2. Query: "Recently updated React UI component libraries"
<query>UI components library in:readme in:description language:javascript topic:react pushed:>2023-01-01</query>
3. Query: "Open source projects developed by Facebook"
<query>org:facebook is:public</query>
4. Query: "Depth-first search implementation in JavaScript"
<query>depth first search in:file language:javascript</query>
Please analyze the query and answer using only the XML tag:
<query>Provide the optimized GitHub search query, using appropriate fields and operators</query>"""
# 3. 生成中文搜索条件
chinese_github_prompt = f"""优化以下GitHub搜索查询:
查询: {query}
任务: 将自然语言查询转换为优化的GitHub搜索查询语句。
为了搜索中文内容,请提取原始查询的关键词并使用中文形式,同时保留GitHub特定的搜索语法为英文。
可用的搜索字段和过滤器:
1. 基本字段:
- in:name - 在仓库名称中搜索
- in:description - 在仓库描述中搜索
- in:readme - 在README文件中搜索
- in:topic - 在主题中搜索
- language:X - 按编程语言筛选
- user:X - 特定用户的仓库
- org:X - 特定组织的仓库
2. 代码搜索字段:
- extension:X - 按文件扩展名筛选
- path:X - 按路径筛选
- filename:X - 按文件名筛选
3. 指标过滤器:
- stars:>X - 有超过X颗星
- forks:>X - 有超过X个分支
- size:>X - 大小超过X KB
- created:>YYYY-MM-DD - 在特定日期后创建
- pushed:>YYYY-MM-DD - 在特定日期后更新
4. 其他过滤器:
- is:public/private - 公开或私有仓库
- archived:true/false - 已归档或未归档
- license:X - 特定许可证
- topic:X - 含特定主题标签
示例:
1. 查询: "找有关机器学习的Python库,至少1000颗星"
<query>机器学习 in:description language:python stars:>1000</query>
2. 查询: "最近更新的React UI组件库"
<query>UI 组件库 in:readme in:description language:javascript topic:react pushed:>2023-01-01</query>
3. 查询: "微信小程序开发框架"
<query>微信小程序 开发框架 in:name in:description in:readme</query>
请分析查询并仅使用XML标签回答:
<query>提供优化的GitHub搜索查询,使用适当的字段和运算符,保留中文关键词</query>"""
try:
# 构建提示数组
prompts = [
type_prompt,
github_prompt,
chinese_github_prompt,
]
show_messages = [
"分析查询类型...",
"优化英文GitHub搜索参数...",
"优化中文GitHub搜索参数...",
]
sys_prompts = [
"你是一个精通GitHub生态系统的专家,擅长分析与GitHub相关的查询。",
"You are a GitHub search expert, specialized in converting natural language queries into optimized GitHub search queries in English.",
"你是一个GitHub搜索专家,擅长处理查询并保留中文关键词进行搜索。",
]
# 使用同步方式调用LLM
responses = yield from request_gpt(
inputs_array=prompts,
inputs_show_user_array=show_messages,
llm_kwargs=llm_kwargs,
chatbot=chatbot,
history_array=[[] for _ in prompts],
sys_prompt_array=sys_prompts,
max_workers=3
)
# 从收集的响应中提取我们需要的内容
extracted_responses = []
for i in range(len(prompts)):
if (i * 2 + 1) < len(responses):
response = responses[i * 2 + 1]
if response is None:
raise Exception(f"Response {i} is None")
if not isinstance(response, str):
try:
response = str(response)
except:
raise Exception(f"Cannot convert response {i} to string")
extracted_responses.append(response)
else:
raise Exception(f"未收到第 {i + 1} 个响应")
# 解析基本信息
query_type = self._extract_tag(extracted_responses[self.BASIC_QUERY_INDEX], "query_type")
if not query_type:
print(
f"Debug - Failed to extract query_type. Response was: {extracted_responses[self.BASIC_QUERY_INDEX]}")
raise Exception("无法提取query_type标签内容")
query_type = query_type.lower()
main_topic = self._extract_tag(extracted_responses[self.BASIC_QUERY_INDEX], "main_topic")
if not main_topic:
print(f"Debug - Failed to extract main_topic. Using query as fallback.")
main_topic = query
query_type = self._normalize_query_type(query_type, query)
# 提取子主题
sub_topics = []
sub_topics_text = self._extract_tag(extracted_responses[self.BASIC_QUERY_INDEX], "sub_topics")
if sub_topics_text:
sub_topics = [topic.strip() for topic in sub_topics_text.split(",")]
# 提取语言
language = self._extract_tag(extracted_responses[self.BASIC_QUERY_INDEX], "language")
# 提取最低星标数
min_stars = 0
min_stars_text = self._extract_tag(extracted_responses[self.BASIC_QUERY_INDEX], "min_stars")
if min_stars_text and min_stars_text.isdigit():
min_stars = int(min_stars_text)
# 解析GitHub搜索参数 - 英文
english_github_query = self._extract_tag(extracted_responses[self.GITHUB_QUERY_INDEX], "query")
# 解析GitHub搜索参数 - 中文
chinese_github_query = self._extract_tag(extracted_responses[2], "query")
# 构建GitHub参数
github_params = {
"query": english_github_query,
"chinese_query": chinese_github_query,
"sort": "stars", # 默认按星标排序
"order": "desc", # 默认降序
"per_page": 30, # 默认每页30条
"page": 1 # 默认第1页
}
# 检查是否为特定仓库查询
repo_id = ""
if "repo:" in english_github_query or "repository:" in english_github_query:
repo_match = re.search(r'(repo|repository):([a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)', english_github_query)
if repo_match:
repo_id = repo_match.group(2)
print(f"Debug - 提取的信息:")
print(f"查询类型: {query_type}")
print(f"主题: {main_topic}")
print(f"子主题: {sub_topics}")
print(f"语言: {language}")
print(f"最低星标数: {min_stars}")
print(f"英文GitHub参数: {english_github_query}")
print(f"中文GitHub参数: {chinese_github_query}")
print(f"特定仓库: {repo_id}")
# 更新返回的 SearchCriteria,包含中英文查询
return SearchCriteria(
query_type=query_type,
main_topic=main_topic,
sub_topics=sub_topics,
language=language,
min_stars=min_stars,
github_params=github_params,
original_query=query,
repo_id=repo_id
)
except Exception as e:
raise Exception(f"分析查询失败: {str(e)}")
def _normalize_query_type(self, query_type: str, query: str) -> str:
"""规范化查询类型"""
if query_type in ["repo", "code", "user", "topic"]:
return query_type
query_lower = query.lower()
for type_name, keywords in self.valid_types.items():
for keyword in keywords:
if keyword in query_lower:
return type_name
query_type_lower = query_type.lower()
for type_name, keywords in self.valid_types.items():
for keyword in keywords:
if keyword in query_type_lower:
return type_name
return "repo" # 默认返回repo类型
def _extract_tag(self, text: str, tag: str) -> str:
"""提取标记内容"""
if not text:
return ""
# 标准XML格式处理多行和特殊字符
pattern = f"<{tag}>(.*?)</{tag}>"
match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
if match:
content = match.group(1).strip()
if content:
return content
# 备用模式
patterns = [
rf"<{tag}>\s*([\s\S]*?)\s*</{tag}>", # 标准XML格式
rf"<{tag}>([\s\S]*?)(?:</{tag}>|$)", # 未闭合的标签
rf"[{tag}]([\s\S]*?)[/{tag}]", # 方括号格式
rf"{tag}:\s*(.*?)(?=\n\w|$)", # 冒号格式
rf"<{tag}>\s*(.*?)(?=<|$)" # 部分闭合
]
# 尝试所有模式
for pattern in patterns:
match = re.search(pattern, text, re.IGNORECASE | re.DOTALL)
if match:
content = match.group(1).strip()
if content: # 确保提取的内容不为空
return content
# 如果所有模式都失败,返回空字符串
return ""

查看文件

@@ -0,0 +1,701 @@
import aiohttp
import asyncio
import base64
import json
import random
from datetime import datetime
from typing import List, Dict, Optional, Union, Any
class GitHubSource:
"""GitHub API实现"""
# 默认API密钥列表 - 可以放置多个GitHub令牌
API_KEYS = [
"github_pat_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"github_pat_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
# "your_github_token_1",
# "your_github_token_2",
# "your_github_token_3"
]
def __init__(self, api_key: Optional[Union[str, List[str]]] = None):
"""初始化GitHub API客户端
Args:
api_key: GitHub个人访问令牌或令牌列表
"""
if api_key is None:
self.api_keys = self.API_KEYS
elif isinstance(api_key, str):
self.api_keys = [api_key]
else:
self.api_keys = api_key
self._initialize()
def _initialize(self) -> None:
"""初始化客户端,设置默认参数"""
self.base_url = "https://api.github.com"
self.headers = {
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "GitHub-API-Python-Client"
}
# 如果有可用的API密钥,随机选择一个
if self.api_keys:
selected_key = random.choice(self.api_keys)
self.headers["Authorization"] = f"Bearer {selected_key}"
print(f"已随机选择API密钥进行认证")
else:
print("警告: 未提供API密钥,将受到GitHub API请求限制")
async def _request(self, method: str, endpoint: str, params: Dict = None, data: Dict = None) -> Any:
"""发送API请求
Args:
method: HTTP方法 (GET, POST, PUT, DELETE等)
endpoint: API端点
params: URL参数
data: 请求体数据
Returns:
解析后的响应JSON
"""
async with aiohttp.ClientSession(headers=self.headers) as session:
url = f"{self.base_url}{endpoint}"
# 为调试目的打印请求信息
print(f"请求: {method} {url}")
if params:
print(f"参数: {params}")
# 发送请求
request_kwargs = {}
if params:
request_kwargs["params"] = params
if data:
request_kwargs["json"] = data
async with session.request(method, url, **request_kwargs) as response:
response_text = await response.text()
# 检查HTTP状态码
if response.status >= 400:
print(f"API请求失败: HTTP {response.status}")
print(f"响应内容: {response_text}")
return None
# 解析JSON响应
try:
return json.loads(response_text)
except json.JSONDecodeError:
print(f"JSON解析错误: {response_text}")
return None
# ===== 用户相关方法 =====
async def get_user(self, username: Optional[str] = None) -> Dict:
"""获取用户信息
Args:
username: 指定用户名,不指定则获取当前授权用户
Returns:
用户信息字典
"""
endpoint = "/user" if username is None else f"/users/{username}"
return await self._request("GET", endpoint)
async def get_user_repos(self, username: Optional[str] = None, sort: str = "updated",
direction: str = "desc", per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取用户的仓库列表
Args:
username: 指定用户名,不指定则获取当前授权用户
sort: 排序方式 (created, updated, pushed, full_name)
direction: 排序方向 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
仓库列表
"""
endpoint = "/user/repos" if username is None else f"/users/{username}/repos"
params = {
"sort": sort,
"direction": direction,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def get_user_starred(self, username: Optional[str] = None,
per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取用户星标的仓库
Args:
username: 指定用户名,不指定则获取当前授权用户
per_page: 每页结果数量
page: 页码
Returns:
星标仓库列表
"""
endpoint = "/user/starred" if username is None else f"/users/{username}/starred"
params = {
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
# ===== 仓库相关方法 =====
async def get_repo(self, owner: str, repo: str) -> Dict:
"""获取仓库信息
Args:
owner: 仓库所有者
repo: 仓库名
Returns:
仓库信息
"""
endpoint = f"/repos/{owner}/{repo}"
return await self._request("GET", endpoint)
async def get_repo_branches(self, owner: str, repo: str, per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取仓库的分支列表
Args:
owner: 仓库所有者
repo: 仓库名
per_page: 每页结果数量
page: 页码
Returns:
分支列表
"""
endpoint = f"/repos/{owner}/{repo}/branches"
params = {
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def get_repo_commits(self, owner: str, repo: str, sha: Optional[str] = None,
path: Optional[str] = None, per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取仓库的提交历史
Args:
owner: 仓库所有者
repo: 仓库名
sha: 特定提交SHA或分支名
path: 文件路径筛选
per_page: 每页结果数量
page: 页码
Returns:
提交列表
"""
endpoint = f"/repos/{owner}/{repo}/commits"
params = {
"per_page": per_page,
"page": page
}
if sha:
params["sha"] = sha
if path:
params["path"] = path
return await self._request("GET", endpoint, params=params)
async def get_commit_details(self, owner: str, repo: str, commit_sha: str) -> Dict:
"""获取特定提交的详情
Args:
owner: 仓库所有者
repo: 仓库名
commit_sha: 提交SHA
Returns:
提交详情
"""
endpoint = f"/repos/{owner}/{repo}/commits/{commit_sha}"
return await self._request("GET", endpoint)
# ===== 内容相关方法 =====
async def get_file_content(self, owner: str, repo: str, path: str, ref: Optional[str] = None) -> Dict:
"""获取文件内容
Args:
owner: 仓库所有者
repo: 仓库名
path: 文件路径
ref: 分支名、标签名或提交SHA
Returns:
文件内容信息
"""
endpoint = f"/repos/{owner}/{repo}/contents/{path}"
params = {}
if ref:
params["ref"] = ref
response = await self._request("GET", endpoint, params=params)
if response and isinstance(response, dict) and "content" in response:
try:
# 解码Base64编码的文件内容
content = base64.b64decode(response["content"].encode()).decode()
response["decoded_content"] = content
except Exception as e:
print(f"解码文件内容时出错: {str(e)}")
return response
async def get_directory_content(self, owner: str, repo: str, path: str, ref: Optional[str] = None) -> List[Dict]:
"""获取目录内容
Args:
owner: 仓库所有者
repo: 仓库名
path: 目录路径
ref: 分支名、标签名或提交SHA
Returns:
目录内容列表
"""
# 注意此方法与get_file_content使用相同的端点,但对于目录会返回列表
endpoint = f"/repos/{owner}/{repo}/contents/{path}"
params = {}
if ref:
params["ref"] = ref
return await self._request("GET", endpoint, params=params)
# ===== Issues相关方法 =====
async def get_issues(self, owner: str, repo: str, state: str = "open",
sort: str = "created", direction: str = "desc",
per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取仓库的Issues列表
Args:
owner: 仓库所有者
repo: 仓库名
state: Issue状态 (open, closed, all)
sort: 排序方式 (created, updated, comments)
direction: 排序方向 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
Issues列表
"""
endpoint = f"/repos/{owner}/{repo}/issues"
params = {
"state": state,
"sort": sort,
"direction": direction,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def get_issue(self, owner: str, repo: str, issue_number: int) -> Dict:
"""获取特定Issue的详情
Args:
owner: 仓库所有者
repo: 仓库名
issue_number: Issue编号
Returns:
Issue详情
"""
endpoint = f"/repos/{owner}/{repo}/issues/{issue_number}"
return await self._request("GET", endpoint)
async def get_issue_comments(self, owner: str, repo: str, issue_number: int) -> List[Dict]:
"""获取Issue的评论
Args:
owner: 仓库所有者
repo: 仓库名
issue_number: Issue编号
Returns:
评论列表
"""
endpoint = f"/repos/{owner}/{repo}/issues/{issue_number}/comments"
return await self._request("GET", endpoint)
# ===== Pull Requests相关方法 =====
async def get_pull_requests(self, owner: str, repo: str, state: str = "open",
sort: str = "created", direction: str = "desc",
per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取仓库的Pull Request列表
Args:
owner: 仓库所有者
repo: 仓库名
state: PR状态 (open, closed, all)
sort: 排序方式 (created, updated, popularity, long-running)
direction: 排序方向 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
Pull Request列表
"""
endpoint = f"/repos/{owner}/{repo}/pulls"
params = {
"state": state,
"sort": sort,
"direction": direction,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def get_pull_request(self, owner: str, repo: str, pr_number: int) -> Dict:
"""获取特定Pull Request的详情
Args:
owner: 仓库所有者
repo: 仓库名
pr_number: Pull Request编号
Returns:
Pull Request详情
"""
endpoint = f"/repos/{owner}/{repo}/pulls/{pr_number}"
return await self._request("GET", endpoint)
async def get_pull_request_files(self, owner: str, repo: str, pr_number: int) -> List[Dict]:
"""获取Pull Request中修改的文件
Args:
owner: 仓库所有者
repo: 仓库名
pr_number: Pull Request编号
Returns:
修改文件列表
"""
endpoint = f"/repos/{owner}/{repo}/pulls/{pr_number}/files"
return await self._request("GET", endpoint)
# ===== 搜索相关方法 =====
async def search_repositories(self, query: str, sort: str = "stars",
order: str = "desc", per_page: int = 30, page: int = 1) -> Dict:
"""搜索仓库
Args:
query: 搜索关键词
sort: 排序方式 (stars, forks, updated)
order: 排序顺序 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
搜索结果
"""
endpoint = "/search/repositories"
params = {
"q": query,
"sort": sort,
"order": order,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def search_code(self, query: str, sort: str = "indexed",
order: str = "desc", per_page: int = 30, page: int = 1) -> Dict:
"""搜索代码
Args:
query: 搜索关键词
sort: 排序方式 (indexed)
order: 排序顺序 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
搜索结果
"""
endpoint = "/search/code"
params = {
"q": query,
"sort": sort,
"order": order,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def search_issues(self, query: str, sort: str = "created",
order: str = "desc", per_page: int = 30, page: int = 1) -> Dict:
"""搜索Issues和Pull Requests
Args:
query: 搜索关键词
sort: 排序方式 (created, updated, comments)
order: 排序顺序 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
搜索结果
"""
endpoint = "/search/issues"
params = {
"q": query,
"sort": sort,
"order": order,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def search_users(self, query: str, sort: str = "followers",
order: str = "desc", per_page: int = 30, page: int = 1) -> Dict:
"""搜索用户
Args:
query: 搜索关键词
sort: 排序方式 (followers, repositories, joined)
order: 排序顺序 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
搜索结果
"""
endpoint = "/search/users"
params = {
"q": query,
"sort": sort,
"order": order,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
# ===== 组织相关方法 =====
async def get_organization(self, org: str) -> Dict:
"""获取组织信息
Args:
org: 组织名称
Returns:
组织信息
"""
endpoint = f"/orgs/{org}"
return await self._request("GET", endpoint)
async def get_organization_repos(self, org: str, type: str = "all",
sort: str = "created", direction: str = "desc",
per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取组织的仓库列表
Args:
org: 组织名称
type: 仓库类型 (all, public, private, forks, sources, member, internal)
sort: 排序方式 (created, updated, pushed, full_name)
direction: 排序方向 (asc, desc)
per_page: 每页结果数量
page: 页码
Returns:
仓库列表
"""
endpoint = f"/orgs/{org}/repos"
params = {
"type": type,
"sort": sort,
"direction": direction,
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
async def get_organization_members(self, org: str, per_page: int = 30, page: int = 1) -> List[Dict]:
"""获取组织成员列表
Args:
org: 组织名称
per_page: 每页结果数量
page: 页码
Returns:
成员列表
"""
endpoint = f"/orgs/{org}/members"
params = {
"per_page": per_page,
"page": page
}
return await self._request("GET", endpoint, params=params)
# ===== 更复杂的操作 =====
async def get_repository_languages(self, owner: str, repo: str) -> Dict:
"""获取仓库使用的编程语言及其比例
Args:
owner: 仓库所有者
repo: 仓库名
Returns:
语言使用情况
"""
endpoint = f"/repos/{owner}/{repo}/languages"
return await self._request("GET", endpoint)
async def get_repository_stats_contributors(self, owner: str, repo: str) -> List[Dict]:
"""获取仓库的贡献者统计
Args:
owner: 仓库所有者
repo: 仓库名
Returns:
贡献者统计信息
"""
endpoint = f"/repos/{owner}/{repo}/stats/contributors"
return await self._request("GET", endpoint)
async def get_repository_stats_commit_activity(self, owner: str, repo: str) -> List[Dict]:
"""获取仓库的提交活动
Args:
owner: 仓库所有者
repo: 仓库名
Returns:
提交活动统计
"""
endpoint = f"/repos/{owner}/{repo}/stats/commit_activity"
return await self._request("GET", endpoint)
async def example_usage():
"""GitHubSource使用示例"""
# 创建客户端实例可选传入API令牌
# github = GitHubSource(api_key="your_github_token")
github = GitHubSource()
try:
# 示例1搜索热门Python仓库
print("\n=== 示例1搜索热门Python仓库 ===")
repos = await github.search_repositories(
query="language:python stars:>1000",
sort="stars",
order="desc",
per_page=5
)
if repos and "items" in repos:
for i, repo in enumerate(repos["items"], 1):
print(f"\n--- 仓库 {i} ---")
print(f"名称: {repo['full_name']}")
print(f"描述: {repo['description']}")
print(f"星标数: {repo['stargazers_count']}")
print(f"Fork数: {repo['forks_count']}")
print(f"最近更新: {repo['updated_at']}")
print(f"URL: {repo['html_url']}")
# 示例2获取特定仓库的详情
print("\n=== 示例2获取特定仓库的详情 ===")
repo_details = await github.get_repo("microsoft", "vscode")
if repo_details:
print(f"名称: {repo_details['full_name']}")
print(f"描述: {repo_details['description']}")
print(f"星标数: {repo_details['stargazers_count']}")
print(f"Fork数: {repo_details['forks_count']}")
print(f"默认分支: {repo_details['default_branch']}")
print(f"开源许可: {repo_details.get('license', {}).get('name', '')}")
print(f"语言: {repo_details['language']}")
print(f"Open Issues数: {repo_details['open_issues_count']}")
# 示例3获取仓库的提交历史
print("\n=== 示例3获取仓库的最近提交 ===")
commits = await github.get_repo_commits("tensorflow", "tensorflow", per_page=5)
if commits:
for i, commit in enumerate(commits, 1):
print(f"\n--- 提交 {i} ---")
print(f"SHA: {commit['sha'][:7]}")
print(f"作者: {commit['commit']['author']['name']}")
print(f"日期: {commit['commit']['author']['date']}")
print(f"消息: {commit['commit']['message'].splitlines()[0]}")
# 示例4搜索代码
print("\n=== 示例4搜索代码 ===")
code_results = await github.search_code(
query="filename:README.md language:markdown pytorch in:file",
per_page=3
)
if code_results and "items" in code_results:
print(f"共找到: {code_results['total_count']} 个结果")
for i, item in enumerate(code_results["items"], 1):
print(f"\n--- 代码 {i} ---")
print(f"仓库: {item['repository']['full_name']}")
print(f"文件: {item['path']}")
print(f"URL: {item['html_url']}")
# 示例5获取文件内容
print("\n=== 示例5获取文件内容 ===")
file_content = await github.get_file_content("python", "cpython", "README.rst")
if file_content and "decoded_content" in file_content:
content = file_content["decoded_content"]
print(f"文件名: {file_content['name']}")
print(f"大小: {file_content['size']} 字节")
print(f"内容预览: {content[:200]}...")
# 示例6获取仓库使用的编程语言
print("\n=== 示例6获取仓库使用的编程语言 ===")
languages = await github.get_repository_languages("facebook", "react")
if languages:
print(f"React仓库使用的编程语言:")
for lang, bytes_of_code in languages.items():
print(f"- {lang}: {bytes_of_code} 字节")
# 示例7获取组织信息
print("\n=== 示例7获取组织信息 ===")
org_info = await github.get_organization("google")
if org_info:
print(f"名称: {org_info['name']}")
print(f"描述: {org_info.get('description', '')}")
print(f"位置: {org_info.get('location', '未指定')}")
print(f"公共仓库数: {org_info['public_repos']}")
print(f"成员数: {org_info.get('public_members', 0)}")
print(f"URL: {org_info['html_url']}")
# 示例8获取用户信息
print("\n=== 示例8获取用户信息 ===")
user_info = await github.get_user("torvalds")
if user_info:
print(f"名称: {user_info['name']}")
print(f"公司: {user_info.get('company', '')}")
print(f"博客: {user_info.get('blog', '')}")
print(f"位置: {user_info.get('location', '未指定')}")
print(f"公共仓库数: {user_info['public_repos']}")
print(f"关注者数: {user_info['followers']}")
print(f"URL: {user_info['html_url']}")
except Exception as e:
print(f"发生错误: {str(e)}")
import traceback
print(traceback.format_exc())
if __name__ == "__main__":
import asyncio
# 运行示例
asyncio.run(example_usage())