first commit

This commit is contained in:
zhu327 2024-08-22 22:15:08 +08:00
commit 0ee735299e
7 changed files with 1774 additions and 0 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
target
node_modules
.wrangler

1397
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

30
Cargo.toml Normal file
View File

@ -0,0 +1,30 @@
[package]
name = "miniflux-ai"
version = "0.1.0"
edition = "2021"
authors = [ "zhu327" ]
[package.metadata.release]
release = false
# https://github.com/rustwasm/wasm-pack/issues/1247
[package.metadata.wasm-pack.profile.release]
wasm-opt = false
[profile.release]
lto = true
strip = true
codegen-units = 1
[lib]
crate-type = ["cdylib"]
[dependencies]
base64 = "0.21"
worker = { version="0.3.4" }
worker-macros = { version="0.3.4" }
console_error_panic_hook = { version = "0.1.7" }
reqwest = { version = "0.11", features = ["json"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
futures = "0.3"

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 Timmy
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

57
README.md Normal file
View File

@ -0,0 +1,57 @@
# Miniflux AI Summarizer
This Cloudflare Workers tool automatically adds AI-generated summaries to articles in your Miniflux RSS reader. The summaries are generated using the OpenAI API and appended to articles in a user-friendly format.
## Features
- **Automated Summarization**: Fetches unread articles from Miniflux, generates concise summaries using AI, and updates the articles with the summaries.
- **Customizable**: Configure the list of whitelisted websites, API endpoints, and AI model parameters through environment variables.
- **Concurrency**: Uses asynchronous Rust features to handle multiple articles concurrently, ensuring quick processing.
- **Cloudflare Integration**: Deployed as a serverless function on Cloudflare Workers, leveraging the scalability and performance of Cloudflare's global network.
- **Recommended Model**: Uses the Cloudflare Workers AI model `@cf/qwen/qwen1.5-14b-chat-awq` for generating high-quality, concise summaries.
## Getting Started
### Prerequisites
- [Rust](https://www.rust-lang.org/tools/install) installed
- A Miniflux instance with API access
- An OpenAI account with access to the model endpoint
- A Cloudflare account
### Installation
1. Clone the repository:
```bash
git clone https://github.com/yourusername/miniflux-ai.git
cd miniflux-ai
```
2. Deploy to Cloudflare Workers:
```bash
npx wrangler deploy
```
### Configuration
The tool is configured using environment variables, which are set in the `wrangler.toml` file:
- `MINIFLUX_URL`: Your Miniflux instance URL.
- `MINIFLUX_USERNAME`: Your Miniflux username.
- `MINIFLUX_PASSWORD`: Your Miniflux password.
- `OPENAI_URL`: The endpoint for the OpenAI API.
- `OPENAI_TOKEN`: Your OpenAI API token.
- `OPENAI_MODEL`: The model ID to use for generating summaries. We recommend using the `@cf/qwen/qwen1.5-14b-chat-awq` model for best results.
- `WHITELIST_URL`: A comma-separated list of website URLs that should be summarized.
### Usage
The tool runs as a scheduled Cloudflare Worker, querying for unread articles every 5 minutes. If an article is from a whitelisted site and does not contain code blocks, it generates a summary and updates the article.
### Contributing
Contributions are welcome! Please feel free to submit issues, feature requests, or pull requests.
### License
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

248
src/lib.rs Normal file
View File

@ -0,0 +1,248 @@
use base64;
use futures::future::join_all;
use reqwest::header::{AUTHORIZATION, CONTENT_TYPE};
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use worker::{event, Env, ScheduleContext, ScheduledEvent};
#[derive(Debug, Deserialize)]
struct Feed {
site_url: String,
}
#[derive(Debug, Deserialize)]
struct Entry {
id: u64,
content: String,
feed: Feed,
}
#[derive(Debug, Deserialize)]
struct ApiResponse {
entries: Vec<Entry>,
}
#[derive(Serialize)]
struct UpdateRequest {
content: String,
}
async fn get_entries(
base_url: &str,
username: &str,
password: &str,
) -> Result<ApiResponse, Box<dyn std::error::Error>> {
// 创建 HTTP 客户端
let client = reqwest::Client::new();
// 使用 Basic Auth 进行身份验证
let auth = format!(
"Basic {}",
base64::encode(format!("{}:{}", username, password))
);
// 发送 GET 请求
let response = client
.get(&format!("{}/v1/entries?status=unread&limit=100", base_url))
.header(AUTHORIZATION, auth)
.header(CONTENT_TYPE, "application/json")
.send()
.await?
.json::<ApiResponse>()
.await?;
Ok(response)
}
async fn update_entry(
base_url: &str,
username: &str,
password: &str,
id: u64,
content: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let client = reqwest::Client::new();
let auth = format!(
"Basic {}",
base64::encode(format!("{}:{}", username, password))
);
let url = format!("{}/v1/entries/{}", base_url, id);
let update_request = UpdateRequest {
content: content.to_string(),
};
client
.put(&url)
.header(AUTHORIZATION, auth)
.header(CONTENT_TYPE, "application/json")
.json(&update_request) // 将请求体序列化为 JSON
.send()
.await?
.error_for_status()?;
Ok(())
}
#[derive(Serialize)]
struct ChatCompletionRequest {
model: String,
messages: Vec<Message>,
}
#[derive(Serialize, Deserialize)]
struct Message {
role: String,
content: String,
}
#[derive(Deserialize)]
struct ChatCompletionChoice {
message: Message,
}
#[derive(Deserialize)]
struct ChatCompletionResponse {
choices: Vec<ChatCompletionChoice>,
}
async fn request_openai_chat_completion(
base_url: &str,
api_key: &str,
model: &str,
messages: Vec<Message>,
) -> Result<String, Box<dyn std::error::Error>> {
let client = reqwest::Client::new();
let request_body = ChatCompletionRequest {
model: model.to_string(),
messages,
};
let response = client
.post(&format!("{}/v1/chat/completions", base_url))
.header(AUTHORIZATION, format!("Bearer {}", api_key))
.header(CONTENT_TYPE, "application/json")
.json(&request_body)
.send()
.await?;
if response.status().is_success() {
let completion_response: ChatCompletionResponse = response.json().await?;
Ok(completion_response.choices[0].message.content.clone())
} else {
let error_message = response.text().await?;
Err(format!("Error: {:?}", error_message).into())
}
}
struct Miniflux {
url: String,
username: String,
password: String,
}
struct OpenAi {
url: String,
token: String,
model: String,
}
struct Config {
miniflux: Miniflux,
openai: OpenAi,
whitelist: HashSet<String>,
}
async fn generate_and_update_entry(
config: &Config,
entry: Entry,
) -> Result<(), Box<dyn std::error::Error>> {
let content: &str = &entry.content;
// Check if the content should be summarized and if the site is whitelisted
if content.starts_with("<pre") || !config.whitelist.contains(&entry.feed.site_url) {
return Ok(());
}
let messages = vec![
Message {
role: "system".to_string(),
content: "Please summarize the content of the article under 50 words in Chinese. Do not add any additional Character、markdown language to the result text. 请用不超过50个汉字概括文章内容。结果文本中不要添加任何额外的字符、Markdown语言。".to_string(),
},
Message {
role: "user".to_string(),
content: format!(
"The following is the input content:\n---\n {}",
content,
),
},
];
// Generate summary
if let Ok(summary) = request_openai_chat_completion(
&config.openai.url,
&config.openai.token,
&config.openai.model,
messages,
)
.await
{
let updated_content = format!(
"<pre style=\"white-space: pre-wrap;\"><code>\n💡AI 摘要:\n{}</code></pre><hr><br />{}",
summary, content
);
// Update the entry
update_entry(
&config.miniflux.url,
&config.miniflux.username,
&config.miniflux.password,
entry.id,
&updated_content,
)
.await?;
}
Ok(())
}
#[event(scheduled)]
async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) {
let config = &Config {
whitelist: env
.var("WHITELIST_URL")
.unwrap()
.to_string()
.split(",")
.map(|s| s.to_string())
.collect(),
openai: OpenAi {
url: env.var("OPENAI_URL").unwrap().to_string(),
token: env.var("OPENAI_TOKEN").unwrap().to_string(),
model: env.var("OPENAI_MODEL").unwrap().to_string(),
},
miniflux: Miniflux {
url: env.var("MINIFLUX_URL").unwrap().to_string(),
username: env.var("MINIFLUX_USERNAME").unwrap().to_string(),
password: env.var("MINIFLUX_PASSWORD").unwrap().to_string(),
},
};
// 查询未读文章
let entries = get_entries(
&config.miniflux.url,
&config.miniflux.username,
&config.miniflux.password,
)
.await
.unwrap();
// 生成摘要并更新的并发任务
let tasks: Vec<_> = entries
.entries
.into_iter()
.map(|entry| generate_and_update_entry(&config, entry))
.collect();
// 执行所有任务并等待结果
join_all(tasks).await;
}

18
wrangler.toml Normal file
View File

@ -0,0 +1,18 @@
name = "miniflux-ai"
main = "build/worker/shim.mjs"
compatibility_date = "2024-08-06"
[build]
command = "cargo install -q worker-build && worker-build --release"
[triggers]
crons = ["*/5 * * * *"]
[vars]
MINIFLUX_URL = "your miniflux url"
MINIFLUX_USERNAME = "your miniflux username"
MINIFLUX_PASSWORD = "your miniflux password"
OPENAI_URL = "https://api.cloudflare.com/client/v4/accounts/{your cloudflare account}/ai"
OPENAI_TOKEN = "your cloudflare workers AI token"
OPENAI_MODEL = "@cf/qwen/qwen1.5-14b-chat-awq"
WHITELIST_URL = "https://www.zaobao.com/news/china,https://t.me/s/theinitium_rss,https://cn.nytimes.com,https://www.latepost.com/news/get-news-data,https://t.me/s/wsj_rss,http://www.zhihu.com,https://new.qq.com/omn/author/5157372,https://www.huxiu.com"